From: Costa Shulyupin <costa.shul@redhat.com>
To: longman@redhat.com, pauld@redhat.com, juri.lelli@redhat.com,
prarit@redhat.com, vschneid@redhat.com,
Anna-Maria Behnsen <anna-maria@linutronix.de>,
Frederic Weisbecker <frederic@kernel.org>,
Thomas Gleixner <tglx@linutronix.de>,
Zefan Li <lizefan.x@bytedance.com>, Tejun Heo <tj@kernel.org>,
Johannes Weiner <hannes@cmpxchg.org>,
Ingo Molnar <mingo@redhat.com>,
Peter Zijlstra <peterz@infradead.org>,
Vincent Guittot <vincent.guittot@linaro.org>,
Dietmar Eggemann <dietmar.eggemann@arm.com>,
Steven Rostedt <rostedt@goodmis.org>,
Ben Segall <bsegall@google.com>, Mel Gorman <mgorman@suse.de>,
Daniel Bristot de Oliveira <bristot@redhat.com>,
Petr Mladek <pmladek@suse.com>,
Andrew Morton <akpm@linux-foundation.org>,
Masahiro Yamada <masahiroy@kernel.org>,
Randy Dunlap <rdunlap@infradead.org>,
Yoann Congal <yoann.congal@smile.fr>,
"Gustavo A. R. Silva" <gustavoars@kernel.org>,
Nhat Pham <nphamcs@gmail.com>,
Costa Shulyupin <costa.shul@redhat.com>,
linux-kernel@vger.kernel.org, cgroups@vger.kernel.org
Subject: [PATCH v1 2/7] sched/isolation: Adjust affinity of timers according to change of housekeeping cpumask
Date: Thu, 16 May 2024 22:04:32 +0300 [thread overview]
Message-ID: <20240516190437.3545310-3-costa.shul@redhat.com> (raw)
In-Reply-To: <20240516190437.3545310-1-costa.shul@redhat.com>
Adjust affinity timers and watchdog_cpumask according to
change of housekeeping.cpumasks[HK_TYPE_TIMER] during runtime.
watchdog_cpumask is initialized during boot in lockup_detector_init()
from housekeeping_cpumask(HK_TYPE_TIMER).
lockup_detector_reconfigure() utilizes updated watchdog_cpumask
via __lockup_detector_reconfigure().
timers_resettle_from_cpu() is blindly prototyped from timers_dead_cpu().
local_irq_disable is used because cpuhp_thread_fun uses it before
cpuhp_invoke_callback() call.
Core test snippets without infrastructure:
1. Create timer on specific cpu with:
timer_setup(&test_timer, test_timer_cb, TIMER_PINNED);
test_timer.expires = KTIME_MAX;
add_timer_on(&test_timer, test_cpu);
2. Call housekeeping_update()
3. Assure that there is no timers on specified cpu at the end
of timers_resettle_from_cpu() with:
static int count_timers(int cpu)
{
struct timer_base *base;
int b, v, count = 0;
for (b = 0; b < NR_BASES; b++) {
base = per_cpu_ptr(&timer_bases[b], cpu);
raw_spin_lock_irq(&base->lock);
for (v = 0; v < WHEEL_SIZE; v++) {
struct hlist_node *c;
hlist_for_each(c, base->vectors + v)
count++;
}
raw_spin_unlock_irq(&base->lock);
}
return count;
}
Signed-off-by: Costa Shulyupin <costa.shul@redhat.com>
---
include/linux/timer.h | 2 ++
init/Kconfig | 1 +
kernel/sched/isolation.c | 27 ++++++++++++++++++++++++++
kernel/time/timer.c | 42 ++++++++++++++++++++++++++++++++++++++++
4 files changed, 72 insertions(+)
diff --git a/include/linux/timer.h b/include/linux/timer.h
index e67ecd1cbc97d..a09266abdb18a 100644
--- a/include/linux/timer.h
+++ b/include/linux/timer.h
@@ -219,9 +219,11 @@ unsigned long round_jiffies_up_relative(unsigned long j);
#ifdef CONFIG_HOTPLUG_CPU
int timers_prepare_cpu(unsigned int cpu);
int timers_dead_cpu(unsigned int cpu);
+void timers_resettle_from_cpu(unsigned int cpu);
#else
#define timers_prepare_cpu NULL
#define timers_dead_cpu NULL
+static inline void timers_resettle_from_cpu(unsigned int cpu) { }
#endif
#endif
diff --git a/init/Kconfig b/init/Kconfig
index 72404c1f21577..fac49c6bb965a 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -682,6 +682,7 @@ config CPU_ISOLATION
bool "CPU isolation"
depends on SMP || COMPILE_TEST
default y
+ select HOTPLUG_CPU
help
Make sure that CPUs running critical tasks are not disturbed by
any source of "noise" such as unbound workqueues, timers, kthreads...
diff --git a/kernel/sched/isolation.c b/kernel/sched/isolation.c
index 036e48f0e7d1b..3b63f0212887e 100644
--- a/kernel/sched/isolation.c
+++ b/kernel/sched/isolation.c
@@ -8,6 +8,10 @@
*
*/
+#ifdef CONFIG_LOCKUP_DETECTOR
+#include <linux/nmi.h>
+#endif
+
enum hk_flags {
HK_FLAG_TIMER = BIT(HK_TYPE_TIMER),
HK_FLAG_RCU = BIT(HK_TYPE_RCU),
@@ -116,6 +120,19 @@ static void __init housekeeping_setup_type(enum hk_type type,
housekeeping_staging);
}
+static void resettle_all_timers(cpumask_var_t enable_mask, cpumask_var_t disable_mask)
+{
+ unsigned int cpu;
+
+ for_each_cpu(cpu, enable_mask) {
+ timers_prepare_cpu(cpu);
+ }
+
+ for_each_cpu(cpu, disable_mask) {
+ timers_resettle_from_cpu(cpu);
+ }
+}
+
/*
* housekeeping_update - change housekeeping.cpumasks[type] and propagate the
* change.
@@ -144,6 +161,16 @@ static int housekeeping_update(enum hk_type type, cpumask_var_t update)
if (!static_branch_unlikely(&housekeeping_overridden))
static_key_enable_cpuslocked(&housekeeping_overridden.key);
+ switch (type) {
+ case HK_TYPE_TIMER:
+ resettle_all_timers(&masks->enable, &masks->disable);
+#ifdef CONFIG_LOCKUP_DETECTOR
+ cpumask_copy(&watchdog_cpumask, housekeeping_cpumask(HK_TYPE_TIMER));
+ lockup_detector_reconfigure();
+#endif
+ break;
+ default:
+ }
kfree(masks);
return 0;
diff --git a/kernel/time/timer.c b/kernel/time/timer.c
index 48288dd4a102f..2d15c0e7b0550 100644
--- a/kernel/time/timer.c
+++ b/kernel/time/timer.c
@@ -51,6 +51,7 @@
#include <asm/div64.h>
#include <asm/timex.h>
#include <asm/io.h>
+#include <linux/sched/isolation.h>
#include "tick-internal.h"
#include "timer_migration.h"
@@ -2657,6 +2658,47 @@ int timers_prepare_cpu(unsigned int cpu)
return 0;
}
+/**
+ * timers_resettle_from_cpu - resettles timers from
+ * specified cpu to housekeeping cpus.
+ */
+void timers_resettle_from_cpu(unsigned int cpu)
+{
+ struct timer_base *old_base;
+ struct timer_base *new_base;
+ int b, i;
+
+ local_irq_disable();
+ for (b = 0; b < NR_BASES; b++) {
+ old_base = per_cpu_ptr(&timer_bases[b], cpu);
+ new_base = per_cpu_ptr(&timer_bases[b],
+ cpumask_any_and(cpu_active_mask,
+ housekeeping_cpumask(HK_TYPE_TIMER)));
+ /*
+ * The caller is globally serialized and nobody else
+ * takes two locks at once, deadlock is not possible.
+ */
+ raw_spin_lock_irq(&new_base->lock);
+ raw_spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING);
+
+ /*
+ * The current CPUs base clock might be stale. Update it
+ * before moving the timers over.
+ */
+ forward_timer_base(new_base);
+
+ WARN_ON_ONCE(old_base->running_timer);
+ old_base->running_timer = NULL;
+
+ for (i = 0; i < WHEEL_SIZE; i++)
+ migrate_timer_list(new_base, old_base->vectors + i);
+
+ raw_spin_unlock(&old_base->lock);
+ raw_spin_unlock_irq(&new_base->lock);
+ }
+ local_irq_enable();
+}
+
int timers_dead_cpu(unsigned int cpu)
{
struct timer_base *old_base;
--
2.45.0
next prev parent reply other threads:[~2024-05-16 19:07 UTC|newest]
Thread overview: 14+ messages / expand[flat|nested] mbox.gz Atom feed top
2024-05-16 19:04 [PATCH v1 0/7] sched: Adjust affinity according to change of housekeeping cpumask Costa Shulyupin
2024-05-16 19:04 ` [PATCH v1 1/7] sched/isolation: Add infrastructure to adjust affinity for dynamic CPU isolation Costa Shulyupin
2024-05-17 21:37 ` Thomas Gleixner
2024-05-16 19:04 ` Costa Shulyupin [this message]
2024-05-17 22:39 ` [PATCH v1 2/7] sched/isolation: Adjust affinity of timers according to change of housekeeping cpumask Thomas Gleixner
2024-05-17 22:52 ` Thomas Gleixner
2024-05-16 19:04 ` [PATCH v1 3/7] sched/isolation: Adjust affinity of hrtimers " Costa Shulyupin
2024-05-17 23:42 ` Thomas Gleixner
2024-05-16 19:04 ` [PATCH v1 4/7] sched/isolation: Adjust affinity of managed irqs " Costa Shulyupin
2024-05-18 1:17 ` Thomas Gleixner
2024-05-18 1:25 ` Thomas Gleixner
2024-05-16 19:04 ` [PATCH v1 5/7] [NOT-FOR-MERGE] test timers affinity adjustment Costa Shulyupin
2024-05-16 19:04 ` [PATCH v1 6/7] [NOT-FOR-MERGE] test timers and hrtimers " Costa Shulyupin
2024-05-16 19:04 ` [PATCH v1 7/7] [NOT-FOR-MERGE] test managed irqs " Costa Shulyupin
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20240516190437.3545310-3-costa.shul@redhat.com \
--to=costa.shul@redhat.com \
--cc=akpm@linux-foundation.org \
--cc=anna-maria@linutronix.de \
--cc=bristot@redhat.com \
--cc=bsegall@google.com \
--cc=cgroups@vger.kernel.org \
--cc=dietmar.eggemann@arm.com \
--cc=frederic@kernel.org \
--cc=gustavoars@kernel.org \
--cc=hannes@cmpxchg.org \
--cc=juri.lelli@redhat.com \
--cc=linux-kernel@vger.kernel.org \
--cc=lizefan.x@bytedance.com \
--cc=longman@redhat.com \
--cc=masahiroy@kernel.org \
--cc=mgorman@suse.de \
--cc=mingo@redhat.com \
--cc=nphamcs@gmail.com \
--cc=pauld@redhat.com \
--cc=peterz@infradead.org \
--cc=pmladek@suse.com \
--cc=prarit@redhat.com \
--cc=rdunlap@infradead.org \
--cc=rostedt@goodmis.org \
--cc=tglx@linutronix.de \
--cc=tj@kernel.org \
--cc=vincent.guittot@linaro.org \
--cc=vschneid@redhat.com \
--cc=yoann.congal@smile.fr \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).