diff --git a/PKGBUILD b/PKGBUILD index acf50b9..442a857 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -339,7 +339,7 @@ case $_basever in '19661ec0d39f9663452b34433214c755179894528bf73a42f6ba52ccf572832a' 'b302ba6c5bbe8ed19b20207505d513208fae1e678cf4d8e7ac0b154e5fe3f456' 'f46ed0f026490b11b6a6cfb21e78cd253f0d7c308dc5a34e93971659a4eaa19e' - 'c5dd103953b8830640538ba30ff511028bd93310f95e4f5587a6ed5e6414a60d' + '377d0eb1df251808b8280d1aec598b4a2986f7d167306cdec9048c337cdcf2e1' '9fad4a40449e09522899955762c8928ae17f4cdaa16e01239fd12592e9d58177' 'a557b342111849a5f920bbe1c129f3ff1fc1eff62c6bd6685e0972fc88e39911' '77e71048389e3e1d6bbdfaf384f62120d3f984257b758d75366c7bebe6be64fa' diff --git a/linux-tkg-patches/5.10/0008-5.10-bcachefs.patch b/linux-tkg-patches/5.10/0008-5.10-bcachefs.patch index 69f697b..249b48e 100644 --- a/linux-tkg-patches/5.10/0008-5.10-bcachefs.patch +++ b/linux-tkg-patches/5.10/0008-5.10-bcachefs.patch @@ -69801,19 +69801,6 @@ index bfd00320c7f3..0af6ca0e3b2e 100644 iter->offset += obj_size; if (!is_power_of_2(obj_size) && -diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h -index 6cdd0152c253..ef41a609640e 100644 ---- a/include/linux/rcupdate.h -+++ b/include/linux/rcupdate.h -@@ -33,6 +33,8 @@ - #define ULONG_CMP_GE(a, b) (ULONG_MAX / 2 >= (a) - (b)) - #define ULONG_CMP_LT(a, b) (ULONG_MAX / 2 < (a) - (b)) - #define ulong2long(a) (*(long *)(&(a))) -+#define USHORT_CMP_GE(a, b) (USHRT_MAX / 2 >= (unsigned short)((a) - (b))) -+#define USHORT_CMP_LT(a, b) (USHRT_MAX / 2 < (unsigned short)((a) - (b))) - - /* Exported common interfaces */ - void call_rcu(struct rcu_head *head, rcu_callback_t func); diff --git a/include/linux/sched.h b/include/linux/sched.h index 76cd21fa5501..f7719e7d5fa7 100644 --- a/include/linux/sched.h @@ -70029,52 +70016,6 @@ index 000000000000..a16e94f482e9 +void six_lock_wakeup_all(struct six_lock *); + +#endif /* _LINUX_SIX_H */ -diff --git a/include/linux/srcu.h b/include/linux/srcu.h -index e432cc92c73d..a0895bbf71ce 100644 ---- a/include/linux/srcu.h -+++ b/include/linux/srcu.h -@@ -60,6 +60,9 @@ void cleanup_srcu_struct(struct srcu_struct *ssp); - int __srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp); - void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases(ssp); - void synchronize_srcu(struct srcu_struct *ssp); -+unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp); -+unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp); -+bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie); - - #ifdef CONFIG_DEBUG_LOCK_ALLOC - -diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h -index 5a5a1941ca15..0e0cf4d6a72a 100644 ---- a/include/linux/srcutiny.h -+++ b/include/linux/srcutiny.h -@@ -15,7 +15,8 @@ - - struct srcu_struct { - short srcu_lock_nesting[2]; /* srcu_read_lock() nesting depth. */ -- short srcu_idx; /* Current reader array element. */ -+ unsigned short srcu_idx; /* Current reader array element in bit 0x2. */ -+ unsigned short srcu_idx_max; /* Furthest future srcu_idx request. */ - u8 srcu_gp_running; /* GP workqueue running? */ - u8 srcu_gp_waiting; /* GP waiting for readers? */ - struct swait_queue_head srcu_wq; -@@ -59,7 +60,7 @@ static inline int __srcu_read_lock(struct srcu_struct *ssp) - { - int idx; - -- idx = READ_ONCE(ssp->srcu_idx); -+ idx = ((READ_ONCE(ssp->srcu_idx) + 1) & 0x2) >> 1; - WRITE_ONCE(ssp->srcu_lock_nesting[idx], ssp->srcu_lock_nesting[idx] + 1); - return idx; - } -@@ -80,7 +81,7 @@ static inline void srcu_torture_stats_print(struct srcu_struct *ssp, - { - int idx; - -- idx = READ_ONCE(ssp->srcu_idx) & 0x1; -+ idx = ((READ_ONCE(ssp->srcu_idx) + 1) & 0x2) >> 1; - pr_alert("%s%s Tiny SRCU per-CPU(idx=%d): (%hd,%hd)\n", - tt, tf, idx, - READ_ONCE(ssp->srcu_lock_nesting[!idx]), diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index 938eaf9517e2..b16872f983ed 100644 --- a/include/linux/vmalloc.h @@ -71459,302 +71400,6 @@ index a4fa44a652a7..55d2a85a2100 100644 } bool __weak module_init_section(const char *name) -diff --git a/kernel/rcu/srcutiny.c b/kernel/rcu/srcutiny.c -index 6208c1dae5c9..26344dc6483b 100644 ---- a/kernel/rcu/srcutiny.c -+++ b/kernel/rcu/srcutiny.c -@@ -34,6 +34,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp) - ssp->srcu_gp_running = false; - ssp->srcu_gp_waiting = false; - ssp->srcu_idx = 0; -+ ssp->srcu_idx_max = 0; - INIT_WORK(&ssp->srcu_work, srcu_drive_gp); - INIT_LIST_HEAD(&ssp->srcu_work.entry); - return 0; -@@ -84,6 +85,8 @@ void cleanup_srcu_struct(struct srcu_struct *ssp) - WARN_ON(ssp->srcu_gp_waiting); - WARN_ON(ssp->srcu_cb_head); - WARN_ON(&ssp->srcu_cb_head != ssp->srcu_cb_tail); -+ WARN_ON(ssp->srcu_idx != ssp->srcu_idx_max); -+ WARN_ON(ssp->srcu_idx & 0x1); - } - EXPORT_SYMBOL_GPL(cleanup_srcu_struct); - -@@ -114,7 +117,7 @@ void srcu_drive_gp(struct work_struct *wp) - struct srcu_struct *ssp; - - ssp = container_of(wp, struct srcu_struct, srcu_work); -- if (ssp->srcu_gp_running || !READ_ONCE(ssp->srcu_cb_head)) -+ if (ssp->srcu_gp_running || USHORT_CMP_GE(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max))) - return; /* Already running or nothing to do. */ - - /* Remove recently arrived callbacks and wait for readers. */ -@@ -124,11 +127,12 @@ void srcu_drive_gp(struct work_struct *wp) - ssp->srcu_cb_head = NULL; - ssp->srcu_cb_tail = &ssp->srcu_cb_head; - local_irq_enable(); -- idx = ssp->srcu_idx; -- WRITE_ONCE(ssp->srcu_idx, !ssp->srcu_idx); -+ idx = (ssp->srcu_idx & 0x2) / 2; -+ WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1); - WRITE_ONCE(ssp->srcu_gp_waiting, true); /* srcu_read_unlock() wakes! */ - swait_event_exclusive(ssp->srcu_wq, !READ_ONCE(ssp->srcu_lock_nesting[idx])); - WRITE_ONCE(ssp->srcu_gp_waiting, false); /* srcu_read_unlock() cheap. */ -+ WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1); - - /* Invoke the callbacks we removed above. */ - while (lh) { -@@ -146,11 +150,27 @@ void srcu_drive_gp(struct work_struct *wp) - * straighten that out. - */ - WRITE_ONCE(ssp->srcu_gp_running, false); -- if (READ_ONCE(ssp->srcu_cb_head)) -+ if (USHORT_CMP_LT(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max))) - schedule_work(&ssp->srcu_work); - } - EXPORT_SYMBOL_GPL(srcu_drive_gp); - -+static void srcu_gp_start_if_needed(struct srcu_struct *ssp) -+{ -+ unsigned short cookie; -+ -+ cookie = get_state_synchronize_srcu(ssp); -+ if (USHORT_CMP_GE(READ_ONCE(ssp->srcu_idx_max), cookie)) -+ return; -+ WRITE_ONCE(ssp->srcu_idx_max, cookie); -+ if (!READ_ONCE(ssp->srcu_gp_running)) { -+ if (likely(srcu_init_done)) -+ schedule_work(&ssp->srcu_work); -+ else if (list_empty(&ssp->srcu_work.entry)) -+ list_add(&ssp->srcu_work.entry, &srcu_boot_list); -+ } -+} -+ - /* - * Enqueue an SRCU callback on the specified srcu_struct structure, - * initiating grace-period processing if it is not already running. -@@ -166,12 +186,7 @@ void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, - *ssp->srcu_cb_tail = rhp; - ssp->srcu_cb_tail = &rhp->next; - local_irq_restore(flags); -- if (!READ_ONCE(ssp->srcu_gp_running)) { -- if (likely(srcu_init_done)) -- schedule_work(&ssp->srcu_work); -- else if (list_empty(&ssp->srcu_work.entry)) -- list_add(&ssp->srcu_work.entry, &srcu_boot_list); -- } -+ srcu_gp_start_if_needed(ssp); - } - EXPORT_SYMBOL_GPL(call_srcu); - -@@ -190,6 +205,48 @@ void synchronize_srcu(struct srcu_struct *ssp) - } - EXPORT_SYMBOL_GPL(synchronize_srcu); - -+/* -+ * get_state_synchronize_srcu - Provide an end-of-grace-period cookie -+ */ -+unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp) -+{ -+ unsigned long ret; -+ -+ barrier(); -+ ret = (READ_ONCE(ssp->srcu_idx) + 3) & ~0x1; -+ barrier(); -+ return ret & USHRT_MAX; -+} -+EXPORT_SYMBOL_GPL(get_state_synchronize_srcu); -+ -+/* -+ * start_poll_synchronize_srcu - Provide cookie and start grace period -+ * -+ * The difference between this and get_state_synchronize_srcu() is that -+ * this function ensures that the poll_state_synchronize_srcu() will -+ * eventually return the value true. -+ */ -+unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp) -+{ -+ unsigned long ret = get_state_synchronize_srcu(ssp); -+ -+ srcu_gp_start_if_needed(ssp); -+ return ret; -+} -+EXPORT_SYMBOL_GPL(start_poll_synchronize_srcu); -+ -+/* -+ * poll_state_synchronize_srcu - Has cookie's grace period ended? -+ */ -+bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie) -+{ -+ bool ret = USHORT_CMP_GE(READ_ONCE(ssp->srcu_idx), cookie); -+ -+ barrier(); -+ return ret; -+} -+EXPORT_SYMBOL_GPL(poll_state_synchronize_srcu); -+ - /* Lockdep diagnostics. */ - void __init rcu_scheduler_starting(void) - { -diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c -index c13348ee80a5..993cdcebe334 100644 ---- a/kernel/rcu/srcutree.c -+++ b/kernel/rcu/srcutree.c -@@ -805,6 +805,46 @@ static void srcu_leak_callback(struct rcu_head *rhp) - { - } - -+/* -+ * Start an SRCU grace period, and also queue the callback if non-NULL. -+ */ -+static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp, -+ struct rcu_head *rhp, bool do_norm) -+{ -+ unsigned long flags; -+ int idx; -+ bool needexp = false; -+ bool needgp = false; -+ unsigned long s; -+ struct srcu_data *sdp; -+ -+ check_init_srcu_struct(ssp); -+ idx = srcu_read_lock(ssp); -+ sdp = raw_cpu_ptr(ssp->sda); -+ spin_lock_irqsave_rcu_node(sdp, flags); -+ if (rhp) -+ rcu_segcblist_enqueue(&sdp->srcu_cblist, rhp); -+ rcu_segcblist_advance(&sdp->srcu_cblist, -+ rcu_seq_current(&ssp->srcu_gp_seq)); -+ s = rcu_seq_snap(&ssp->srcu_gp_seq); -+ (void)rcu_segcblist_accelerate(&sdp->srcu_cblist, s); -+ if (ULONG_CMP_LT(sdp->srcu_gp_seq_needed, s)) { -+ sdp->srcu_gp_seq_needed = s; -+ needgp = true; -+ } -+ if (!do_norm && ULONG_CMP_LT(sdp->srcu_gp_seq_needed_exp, s)) { -+ sdp->srcu_gp_seq_needed_exp = s; -+ needexp = true; -+ } -+ spin_unlock_irqrestore_rcu_node(sdp, flags); -+ if (needgp) -+ srcu_funnel_gp_start(ssp, sdp, s, do_norm); -+ else if (needexp) -+ srcu_funnel_exp_start(ssp, sdp->mynode, s); -+ srcu_read_unlock(ssp, idx); -+ return s; -+} -+ - /* - * Enqueue an SRCU callback on the srcu_data structure associated with - * the current CPU and the specified srcu_struct structure, initiating -@@ -836,14 +876,6 @@ static void srcu_leak_callback(struct rcu_head *rhp) - static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, - rcu_callback_t func, bool do_norm) - { -- unsigned long flags; -- int idx; -- bool needexp = false; -- bool needgp = false; -- unsigned long s; -- struct srcu_data *sdp; -- -- check_init_srcu_struct(ssp); - if (debug_rcu_head_queue(rhp)) { - /* Probable double call_srcu(), so leak the callback. */ - WRITE_ONCE(rhp->func, srcu_leak_callback); -@@ -851,28 +883,7 @@ static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, - return; - } - rhp->func = func; -- idx = srcu_read_lock(ssp); -- sdp = raw_cpu_ptr(ssp->sda); -- spin_lock_irqsave_rcu_node(sdp, flags); -- rcu_segcblist_enqueue(&sdp->srcu_cblist, rhp); -- rcu_segcblist_advance(&sdp->srcu_cblist, -- rcu_seq_current(&ssp->srcu_gp_seq)); -- s = rcu_seq_snap(&ssp->srcu_gp_seq); -- (void)rcu_segcblist_accelerate(&sdp->srcu_cblist, s); -- if (ULONG_CMP_LT(sdp->srcu_gp_seq_needed, s)) { -- sdp->srcu_gp_seq_needed = s; -- needgp = true; -- } -- if (!do_norm && ULONG_CMP_LT(sdp->srcu_gp_seq_needed_exp, s)) { -- sdp->srcu_gp_seq_needed_exp = s; -- needexp = true; -- } -- spin_unlock_irqrestore_rcu_node(sdp, flags); -- if (needgp) -- srcu_funnel_gp_start(ssp, sdp, s, do_norm); -- else if (needexp) -- srcu_funnel_exp_start(ssp, sdp->mynode, s); -- srcu_read_unlock(ssp, idx); -+ (void)srcu_gp_start_if_needed(ssp, rhp, do_norm); - } - - /** -@@ -1001,6 +1012,62 @@ void synchronize_srcu(struct srcu_struct *ssp) - } - EXPORT_SYMBOL_GPL(synchronize_srcu); - -+/** -+ * get_state_synchronize_srcu - Provide an end-of-grace-period cookie -+ * @ssp: srcu_struct to provide cookie for. -+ * -+ * This function returns a cookie that can be passed to -+ * poll_state_synchronize_srcu(), which will return true if a full grace -+ * period has elapsed in the meantime. It is the caller's responsibility -+ * to make sure that grace period happens, for example, by invoking -+ * call_srcu() after return from get_state_synchronize_srcu(). -+ */ -+unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp) -+{ -+ // Any prior manipulation of SRCU-protected data must happen -+ // before the load from ->srcu_gp_seq. -+ smp_mb(); -+ return rcu_seq_snap(&ssp->srcu_gp_seq); -+} -+EXPORT_SYMBOL_GPL(get_state_synchronize_srcu); -+ -+/** -+ * start_poll_synchronize_srcu - Provide cookie and start grace period -+ * @ssp: srcu_struct to provide cookie for. -+ * -+ * This function returns a cookie that can be passed to -+ * poll_state_synchronize_srcu(), which will return true if a full grace -+ * period has elapsed in the meantime. Unlike get_state_synchronize_srcu(), -+ * this function also ensures that any needed SRCU grace period will be -+ * started. This convenience does come at a cost in terms of CPU overhead. -+ */ -+unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp) -+{ -+ return srcu_gp_start_if_needed(ssp, NULL, true); -+} -+EXPORT_SYMBOL_GPL(start_poll_synchronize_srcu); -+ -+/** -+ * poll_state_synchronize_srcu - Has cookie's grace period ended? -+ * @ssp: srcu_struct to provide cookie for. -+ * @cookie: Return value from get_state_synchronize_srcu() or start_poll_synchronize_srcu(). -+ * -+ * This function takes the cookie that was returned from either -+ * get_state_synchronize_srcu() or start_poll_synchronize_srcu(), and -+ * returns @true if an SRCU grace period elapsed since the time that the -+ * cookie was created. -+ */ -+bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie) -+{ -+ if (!rcu_seq_done(&ssp->srcu_gp_seq, cookie)) -+ return false; -+ // Ensure that the end of the SRCU grace period happens before -+ // any subsequent code that the caller might execute. -+ smp_mb(); // ^^^ -+ return true; -+} -+EXPORT_SYMBOL_GPL(poll_state_synchronize_srcu); -+ - /* - * Callback function for srcu_barrier() use. - */ diff --git a/lib/Kconfig b/lib/Kconfig index b46a9fd122c8..00646b0b848a 100644 --- a/lib/Kconfig