From: Waiman Long <longman@redhat.com>
To: Tejun Heo <tj@kernel.org>, Zefan Li <lizefan.x@bytedance.com>,
Johannes Weiner <hannes@cmpxchg.org>,
Frederic Weisbecker <frederic@kernel.org>,
Jonathan Corbet <corbet@lwn.net>,
"Paul E. McKenney" <paulmck@kernel.org>,
Neeraj Upadhyay <quic_neeraju@quicinc.com>,
Joel Fernandes <joel@joelfernandes.org>,
Josh Triplett <josh@joshtriplett.org>,
Boqun Feng <boqun.feng@gmail.com>,
Steven Rostedt <rostedt@goodmis.org>,
Mathieu Desnoyers <mathieu.desnoyers@efficios.com>,
Lai Jiangshan <jiangshanlai@gmail.com>,
Zqiang <qiang.zhang1211@gmail.com>,
Davidlohr Bueso <dave@stgolabs.net>,
Shuah Khan <shuah@kernel.org>
Cc: cgroups@vger.kernel.org, linux-doc@vger.kernel.org,
linux-kernel@vger.kernel.org, rcu@vger.kernel.org,
linux-kselftest@vger.kernel.org, Mrunal Patel <mpatel@redhat.com>,
Ryan Phillips <rphillips@redhat.com>,
Brent Rowsell <browsell@redhat.com>,
Peter Hunt <pehunt@redhat.com>,
Cestmir Kalina <ckalina@redhat.com>,
Nicolas Saenz Julienne <nsaenz@kernel.org>,
Alex Gladkov <agladkov@redhat.com>,
Marcelo Tosatti <mtosatti@redhat.com>,
Phil Auld <pauld@redhat.com>,
Paul Gortmaker <paul.gortmaker@windriver.com>,
Daniel Bristot de Oliveira <bristot@kernel.org>,
Juri Lelli <juri.lelli@redhat.com>,
Peter Zijlstra <peterz@infradead.org>,
Costa Shulyupin <cshulyup@redhat.com>,
Waiman Long <longman@redhat.com>
Subject: [RFC PATCH 1/8] rcu/nocb: Pass a cpumask instead of a single CPU to offload/deoffload
Date: Wed, 17 Jan 2024 11:35:04 -0500 [thread overview]
Message-ID: <20240117163511.88173-2-longman@redhat.com> (raw)
In-Reply-To: <20240117163511.88173-1-longman@redhat.com>
From: Frederic Weisbecker <frederic@kernel.org>
Currently the interface to toggle callbacks offloading state only takes
a single CPU per call. Now driving RCU NOCB through cpusets requires
to be able to change the offloading state of a whole set of CPUs.
To make it easier, extend the (de-)offloading interface to support a
cpumask.
Signed-off-by: Frederic Weisbecker <frederic@kernel.org>
Cc: Zefan Li <lizefan.x@bytedance.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Paul E. McKenney <paulmck@kernel.org>
Cc: Phil Auld <pauld@redhat.com>
Cc: Nicolas Saenz Julienne <nsaenz@kernel.org>
Cc: Marcelo Tosatti <mtosatti@redhat.com>
Cc: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: Waiman Long <longman@redhat.com>
Cc: Daniel Bristot de Oliveira <bristot@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
---
include/linux/rcupdate.h | 9 ++--
kernel/rcu/rcutorture.c | 4 +-
kernel/rcu/tree_nocb.h | 102 ++++++++++++++++++++++++++-------------
3 files changed, 76 insertions(+), 39 deletions(-)
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index 0746b1b0b663..b649344075d2 100644
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -142,13 +142,14 @@ static inline void rcu_irq_work_resched(void) { }
#ifdef CONFIG_RCU_NOCB_CPU
void rcu_init_nohz(void);
-int rcu_nocb_cpu_offload(int cpu);
-int rcu_nocb_cpu_deoffload(int cpu);
+int rcu_nocb_cpumask_update(const struct cpumask *cpumask, bool offload);
void rcu_nocb_flush_deferred_wakeup(void);
#else /* #ifdef CONFIG_RCU_NOCB_CPU */
static inline void rcu_init_nohz(void) { }
-static inline int rcu_nocb_cpu_offload(int cpu) { return -EINVAL; }
-static inline int rcu_nocb_cpu_deoffload(int cpu) { return 0; }
+static inline int rcu_nocb_cpumask_update(const struct cpumask *cpumask, bool offload)
+{
+ return -EINVAL;
+}
static inline void rcu_nocb_flush_deferred_wakeup(void) { }
#endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */
diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c
index 7567ca8e743c..228a5488eb5e 100644
--- a/kernel/rcu/rcutorture.c
+++ b/kernel/rcu/rcutorture.c
@@ -2140,10 +2140,10 @@ static int rcu_nocb_toggle(void *arg)
r = torture_random(&rand);
cpu = (r >> 1) % (maxcpu + 1);
if (r & 0x1) {
- rcu_nocb_cpu_offload(cpu);
+ rcu_nocb_cpumask_update(cpumask_of(cpu), true);
atomic_long_inc(&n_nocb_offload);
} else {
- rcu_nocb_cpu_deoffload(cpu);
+ rcu_nocb_cpumask_update(cpumask_of(cpu), false);
atomic_long_inc(&n_nocb_deoffload);
}
toggle_delay = torture_random(&rand) % toggle_fuzz + toggle_interval;
diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h
index 4efbf7333d4e..60b0a15ed6e2 100644
--- a/kernel/rcu/tree_nocb.h
+++ b/kernel/rcu/tree_nocb.h
@@ -1203,29 +1203,23 @@ static long rcu_nocb_rdp_deoffload(void *arg)
return 0;
}
-int rcu_nocb_cpu_deoffload(int cpu)
+static int rcu_nocb_cpu_deoffload(int cpu)
{
struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
int ret = 0;
- cpus_read_lock();
- mutex_lock(&rcu_state.barrier_mutex);
- if (rcu_rdp_is_offloaded(rdp)) {
- if (cpu_online(cpu)) {
- ret = work_on_cpu(cpu, rcu_nocb_rdp_deoffload, rdp);
- if (!ret)
- cpumask_clear_cpu(cpu, rcu_nocb_mask);
- } else {
- pr_info("NOCB: Cannot CB-deoffload offline CPU %d\n", rdp->cpu);
- ret = -EINVAL;
- }
- }
- mutex_unlock(&rcu_state.barrier_mutex);
- cpus_read_unlock();
+ if (cpu_is_offline(cpu))
+ return -EINVAL;
+
+ if (!rcu_rdp_is_offloaded(rdp))
+ return 0;
+
+ ret = work_on_cpu(cpu, rcu_nocb_rdp_deoffload, rdp);
+ if (!ret)
+ cpumask_clear_cpu(cpu, rcu_nocb_mask);
return ret;
}
-EXPORT_SYMBOL_GPL(rcu_nocb_cpu_deoffload);
static long rcu_nocb_rdp_offload(void *arg)
{
@@ -1236,12 +1230,6 @@ static long rcu_nocb_rdp_offload(void *arg)
struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;
WARN_ON_ONCE(rdp->cpu != raw_smp_processor_id());
- /*
- * For now we only support re-offload, ie: the rdp must have been
- * offloaded on boot first.
- */
- if (!rdp->nocb_gp_rdp)
- return -EINVAL;
if (WARN_ON_ONCE(!rdp_gp->nocb_gp_kthread))
return -EINVAL;
@@ -1288,29 +1276,77 @@ static long rcu_nocb_rdp_offload(void *arg)
return 0;
}
-int rcu_nocb_cpu_offload(int cpu)
+static int rcu_nocb_cpu_offload(int cpu)
{
struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
- int ret = 0;
+ int ret;
+
+ if (cpu_is_offline(cpu))
+ return -EINVAL;
+
+ if (rcu_rdp_is_offloaded(rdp))
+ return 0;
+
+ ret = work_on_cpu(cpu, rcu_nocb_rdp_offload, rdp);
+ if (!ret)
+ cpumask_set_cpu(cpu, rcu_nocb_mask);
+
+ return ret;
+}
+
+int rcu_nocb_cpumask_update(const struct cpumask *cpumask, bool offload)
+{
+ int cpu;
+ int err = 0;
+ int err_cpu;
+ cpumask_var_t saved_nocb_mask;
+
+ if (!alloc_cpumask_var(&saved_nocb_mask, GFP_KERNEL))
+ return -ENOMEM;
+
+ cpumask_copy(saved_nocb_mask, rcu_nocb_mask);
cpus_read_lock();
mutex_lock(&rcu_state.barrier_mutex);
- if (!rcu_rdp_is_offloaded(rdp)) {
- if (cpu_online(cpu)) {
- ret = work_on_cpu(cpu, rcu_nocb_rdp_offload, rdp);
- if (!ret)
- cpumask_set_cpu(cpu, rcu_nocb_mask);
+ for_each_cpu(cpu, cpumask) {
+ if (offload) {
+ err = rcu_nocb_cpu_offload(cpu);
+ if (err < 0) {
+ err_cpu = cpu;
+ pr_err("NOCB: offload cpu %d failed (%d)\n", cpu, err);
+ break;
+ }
} else {
- pr_info("NOCB: Cannot CB-offload offline CPU %d\n", rdp->cpu);
- ret = -EINVAL;
+ err = rcu_nocb_cpu_deoffload(cpu);
+ if (err < 0) {
+ err_cpu = cpu;
+ pr_err("NOCB: deoffload cpu %d failed (%d)\n", cpu, err);
+ break;
+ }
+ }
+ }
+
+ /* Rollback in case of error */
+ if (err < 0) {
+ err_cpu = cpu;
+ for_each_cpu(cpu, cpumask) {
+ if (err_cpu == cpu)
+ break;
+ if (cpumask_test_cpu(cpu, saved_nocb_mask))
+ WARN_ON_ONCE(rcu_nocb_cpu_offload(cpu));
+ else
+ WARN_ON_ONCE(rcu_nocb_cpu_deoffload(cpu));
}
}
+
mutex_unlock(&rcu_state.barrier_mutex);
cpus_read_unlock();
- return ret;
+ free_cpumask_var(saved_nocb_mask);
+
+ return err;
}
-EXPORT_SYMBOL_GPL(rcu_nocb_cpu_offload);
+EXPORT_SYMBOL_GPL(rcu_nocb_cpumask_update);
#ifdef CONFIG_RCU_LAZY
static unsigned long
--
2.39.3
next prev parent reply other threads:[~2024-01-17 16:36 UTC|newest]
Thread overview: 20+ messages / expand[flat|nested] mbox.gz Atom feed top
2024-01-17 16:35 [RFC PATCH 0/8] cgroup/cpuset: Support RCU_NOCB on isolated partitions Waiman Long
2024-01-17 16:35 ` Waiman Long [this message]
2024-01-17 16:35 ` [RFC PATCH 2/8] rcu/nocb: Prepare to change nocb cpumask from CPU-hotplug protected cpuset caller Waiman Long
2024-01-17 16:35 ` [RFC PATCH 3/8] rcu/no_cb: Add rcu_nocb_enabled() to expose the rcu_nocb state Waiman Long
2024-01-17 16:35 ` [RFC PATCH 4/8] cgroup/cpuset: Better tracking of addition/deletion of isolated CPUs Waiman Long
2024-01-17 16:35 ` [RFC PATCH 5/8] cgroup/cpuset: Add cpuset.cpus.isolation_full Waiman Long
2024-01-17 16:35 ` [RFC PATCH 6/8] cgroup/cpuset: Enable dynamic rcu_nocb mode on isolated CPUs Waiman Long
2024-01-17 16:35 ` [RFC PATCH 7/8] cgroup/cpuset: Document the new cpuset.cpus.isolation_full control file Waiman Long
2024-01-17 16:35 ` [RFC PATCH 8/8] cgroup/cpuset: Update test_cpuset_prs.sh to handle cpuset.cpus.isolation_full Waiman Long
2024-01-17 17:07 ` [RFC PATCH 0/8] cgroup/cpuset: Support RCU_NOCB on isolated partitions Tejun Heo
2024-01-17 17:15 ` Waiman Long
2024-02-06 12:56 ` Frederic Weisbecker
2024-02-06 19:15 ` Marcelo Tosatti
2024-02-07 14:47 ` Frederic Weisbecker
2024-02-07 14:59 ` Marcelo Tosatti
2024-02-10 4:19 ` Waiman Long
2024-01-19 10:24 ` Paul E. McKenney
2024-02-11 1:46 ` Waiman Long
2024-01-22 15:07 ` Michal Koutný
2024-01-23 5:50 ` Waiman Long
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20240117163511.88173-2-longman@redhat.com \
--to=longman@redhat.com \
--cc=agladkov@redhat.com \
--cc=boqun.feng@gmail.com \
--cc=bristot@kernel.org \
--cc=browsell@redhat.com \
--cc=cgroups@vger.kernel.org \
--cc=ckalina@redhat.com \
--cc=corbet@lwn.net \
--cc=cshulyup@redhat.com \
--cc=dave@stgolabs.net \
--cc=frederic@kernel.org \
--cc=hannes@cmpxchg.org \
--cc=jiangshanlai@gmail.com \
--cc=joel@joelfernandes.org \
--cc=josh@joshtriplett.org \
--cc=juri.lelli@redhat.com \
--cc=linux-doc@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-kselftest@vger.kernel.org \
--cc=lizefan.x@bytedance.com \
--cc=mathieu.desnoyers@efficios.com \
--cc=mpatel@redhat.com \
--cc=mtosatti@redhat.com \
--cc=nsaenz@kernel.org \
--cc=paul.gortmaker@windriver.com \
--cc=pauld@redhat.com \
--cc=paulmck@kernel.org \
--cc=pehunt@redhat.com \
--cc=peterz@infradead.org \
--cc=qiang.zhang1211@gmail.com \
--cc=quic_neeraju@quicinc.com \
--cc=rcu@vger.kernel.org \
--cc=rostedt@goodmis.org \
--cc=rphillips@redhat.com \
--cc=shuah@kernel.org \
--cc=tj@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).