From: tip-bot for Arjan van de Ven <arjan@infradead.org>
To: linux-tip-commits@vger.kernel.org
Cc: linux-kernel@vger.kernel.org, hpa@zytor.com, mingo@redhat.com,
eric.dumazet@gmail.com, a.p.zijlstra@chello.nl,
johnstul@us.ibm.com, torvalds@linux-foundation.org,
arjan@linux.intel.com, schwidefsky@de.ibm.com,
arjan@infradead.org, tglx@linutronix.de, mingo@elte.hu
Subject: [tip:sched/urgent] x86: Provide an alternative() based cmpxchg64()
Date: Wed, 30 Sep 2009 19:37:13 GMT [thread overview]
Message-ID: <tip-8da752098d4e584fbcc149046e5c716f5d356db4@git.kernel.org> (raw)
In-Reply-To: <20090930170754.0886ff2e@infradead.org>
Commit-ID: 8da752098d4e584fbcc149046e5c716f5d356db4
Gitweb: http://git.kernel.org/tip/8da752098d4e584fbcc149046e5c716f5d356db4
Author: Arjan van de Ven <arjan@infradead.org>
AuthorDate: Wed, 30 Sep 2009 17:07:54 +0200
Committer: Ingo Molnar <mingo@elte.hu>
CommitDate: Wed, 30 Sep 2009 20:58:26 +0200
x86: Provide an alternative() based cmpxchg64()
cmpxchg64() today generates, to quote Linus, "barf bag" code.
cmpxchg64() is about to get used in the scheduler to a bug there,
but it's a prerequisite that cmpxchg64() first be made non-sucking.
This patch turns cmpxchg64() into an efficient implementation that
uses the alternative() mechanism to just use the raw instruction on
all modern systens
Note: the fallback is NOT smp safe, just like the current fallback
is not SMP safe. (Interested parties with i486 based SMP systems
are welcome to submit fix patches for that.)
Signed-off-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: John Stultz <johnstul@us.ibm.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <20090930170754.0886ff2e@infradead.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
---
arch/x86/include/asm/cmpxchg_32.h | 29 ++++++++++--------
arch/x86/kernel/i386_ksyms_32.c | 8 +++++
arch/x86/lib/Makefile | 2 +-
arch/x86/lib/cmpxchg8b_emu.S | 61 +++++++++++++++++++++++++++++++++++++
4 files changed, 86 insertions(+), 14 deletions(-)
diff --git a/arch/x86/include/asm/cmpxchg_32.h b/arch/x86/include/asm/cmpxchg_32.h
index 82ceb78..3b21afa 100644
--- a/arch/x86/include/asm/cmpxchg_32.h
+++ b/arch/x86/include/asm/cmpxchg_32.h
@@ -312,19 +312,22 @@ static inline unsigned long cmpxchg_386(volatile void *ptr, unsigned long old,
extern unsigned long long cmpxchg_486_u64(volatile void *, u64, u64);
-#define cmpxchg64(ptr, o, n) \
-({ \
- __typeof__(*(ptr)) __ret; \
- if (likely(boot_cpu_data.x86 > 4)) \
- __ret = (__typeof__(*(ptr)))__cmpxchg64((ptr), \
- (unsigned long long)(o), \
- (unsigned long long)(n)); \
- else \
- __ret = (__typeof__(*(ptr)))cmpxchg_486_u64((ptr), \
- (unsigned long long)(o), \
- (unsigned long long)(n)); \
- __ret; \
-})
+#define cmpxchg64(ptr, o, n) \
+({ \
+ __typeof__(*(ptr)) __ret; \
+ __typeof__(*(ptr)) __old = (o); \
+ __typeof__(*(ptr)) __new = (n); \
+ alternative_io("call cmpxchg8b_emu", \
+ "lock; cmpxchg8b (%%esi)" , \
+ X86_FEATURE_CX8, \
+ "=A" (__ret), \
+ "S" ((ptr)), "0" (__old), \
+ "b" ((unsigned int)__new), \
+ "c" ((unsigned int)(__new>>32))); \
+ __ret; })
+
+
+
#define cmpxchg64_local(ptr, o, n) \
({ \
__typeof__(*(ptr)) __ret; \
diff --git a/arch/x86/kernel/i386_ksyms_32.c b/arch/x86/kernel/i386_ksyms_32.c
index 43cec6b..1736c5a 100644
--- a/arch/x86/kernel/i386_ksyms_32.c
+++ b/arch/x86/kernel/i386_ksyms_32.c
@@ -10,6 +10,14 @@
EXPORT_SYMBOL(mcount);
#endif
+/*
+ * Note, this is a prototype to get at the symbol for
+ * the export, but dont use it from C code, it is used
+ * by assembly code and is not using C calling convention!
+ */
+extern void cmpxchg8b_emu(void);
+EXPORT_SYMBOL(cmpxchg8b_emu);
+
/* Networking helper routines. */
EXPORT_SYMBOL(csum_partial_copy_generic);
diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile
index 9e60920..3e549b8 100644
--- a/arch/x86/lib/Makefile
+++ b/arch/x86/lib/Makefile
@@ -15,7 +15,7 @@ ifeq ($(CONFIG_X86_32),y)
obj-y += atomic64_32.o
lib-y += checksum_32.o
lib-y += strstr_32.o
- lib-y += semaphore_32.o string_32.o
+ lib-y += semaphore_32.o string_32.o cmpxchg8b_emu.o
lib-$(CONFIG_X86_USE_3DNOW) += mmx_32.o
else
diff --git a/arch/x86/lib/cmpxchg8b_emu.S b/arch/x86/lib/cmpxchg8b_emu.S
new file mode 100644
index 0000000..b8af4c7
--- /dev/null
+++ b/arch/x86/lib/cmpxchg8b_emu.S
@@ -0,0 +1,61 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 2
+ * of the License.
+ *
+ */
+
+#include <linux/linkage.h>
+#include <asm/alternative-asm.h>
+#include <asm/frame.h>
+#include <asm/dwarf2.h>
+
+
+.text
+
+/*
+ * Inputs:
+ * %esi : memory location to compare
+ * %eax : low 32 bits of old value
+ * %edx : high 32 bits of old value
+ * %ebx : low 32 bits of new value
+ * %ecx : high 32 bits of new value
+ */
+ENTRY(cmpxchg8b_emu)
+ CFI_STARTPROC
+
+ push %edi
+ push %ebx
+ push %ecx
+ /* disable interrupts */
+ pushf
+ pop %edi
+ cli
+
+ cmpl %edx, 4(%esi)
+ jne 1f
+ cmpl %eax, (%esi)
+ jne 1f
+
+ xchg (%esi), %ebx
+ xchg 4(%esi), %ecx
+ mov %ebx, %eax
+ mov %ecx, %edx
+
+2:
+ /* restore interrupts */
+ push %edi
+ popf
+
+ pop %ecx
+ pop %ebx
+ pop %edi
+ ret
+1:
+ mov (%esi), %eax
+ mov 4(%esi), %edx
+ jmp 2b
+ CFI_ENDPROC
+ENDPROC(cmpxchg8b_emu)
+
next prev parent reply other threads:[~2009-09-30 19:38 UTC|newest]
Thread overview: 79+ messages / expand[flat|nested] mbox.gz Atom feed top
2009-09-27 22:34 Linux 2.6.32-rc1 Linus Torvalds
2009-09-27 23:44 ` Stephen Rothwell
2009-09-27 23:48 ` Diego Calleja
2009-09-27 23:52 ` Linus Torvalds
2009-09-28 0:17 ` Stephen Rothwell
2009-09-28 7:07 ` Eric Dumazet
2009-09-28 15:39 ` Linus Torvalds
2009-09-28 17:15 ` Martin Schwidefsky
2009-09-28 18:41 ` Eric Dumazet
2009-09-28 20:56 ` Martin Schwidefsky
2009-09-29 20:42 ` Eric Dumazet
2009-09-29 21:17 ` Linus Torvalds
2009-09-29 21:22 ` Arjan van de Ven
2009-09-29 21:56 ` Linus Torvalds
2009-09-30 15:07 ` Arjan van de Ven
2009-09-30 15:27 ` Eric Dumazet
2009-09-30 15:31 ` Arjan van de Ven
2009-10-01 0:42 ` Yuhong Bao
2009-09-30 15:57 ` Eric Dumazet
2009-09-30 16:13 ` Arjan van de Ven
2009-09-30 16:14 ` Linus Torvalds
2009-09-30 18:53 ` Ingo Molnar
2009-09-30 22:03 ` [GIT PULL] scheduler fixes Ingo Molnar
2009-10-01 0:42 ` Linus Torvalds
2009-10-01 0:57 ` Linus Torvalds
2009-10-01 5:30 ` Eric Dumazet
2009-10-01 6:11 ` Ingo Molnar
2009-10-01 6:18 ` Eric Dumazet
2009-10-01 6:42 ` Ingo Molnar
2009-10-01 6:59 ` Eric Dumazet
2009-10-01 7:28 ` Sam Ravnborg
2009-10-01 6:49 ` [tip:x86/urgent] x86: Don't generate cmpxchg8b_emu if CONFIG_X86_CMPXCHG64=y tip-bot for Eric Dumazet
2009-10-01 6:40 ` [tip:x86/urgent] x86: Optimize cmpxchg64() at build-time some more tip-bot for Linus Torvalds
2009-10-02 16:40 ` [GIT PULL] scheduler fixes Yuhong Bao
2009-10-01 6:05 ` Ingo Molnar
2009-09-30 16:14 ` Linux 2.6.32-rc1 Cyrill Gorcunov
2009-09-30 16:55 ` Stefan Richter
2009-09-30 17:08 ` Linus Torvalds
2009-09-30 17:40 ` Stefan Richter
2009-09-30 19:32 ` Ingo Molnar
2009-09-30 19:35 ` Ingo Molnar
2009-09-30 20:16 ` Eric Dumazet
2009-09-30 20:20 ` Linus Torvalds
2009-09-30 20:22 ` Ingo Molnar
2009-09-30 20:38 ` Ingo Molnar
2009-10-01 7:18 ` Arjan van de Ven
2009-09-30 19:37 ` tip-bot for Arjan van de Ven [this message]
2009-09-30 19:37 ` [tip:sched/urgent] sched_clock: Fix atomicity/continuity bug by using cmpxchg64() tip-bot for Arjan van de Ven
2009-09-30 19:39 ` Ingo Molnar
2009-09-30 19:39 ` tip-bot for Eric Dumazet
2009-09-30 20:19 ` Linux 2.6.32-rc1 Linus Torvalds
2009-09-30 20:24 ` Eric Dumazet
2009-09-30 20:41 ` Linus Torvalds
2009-09-30 20:49 ` Ingo Molnar
2009-09-30 20:53 ` Eric Dumazet
2009-09-30 21:00 ` Ingo Molnar
2009-09-30 20:54 ` Linus Torvalds
2009-09-30 21:53 ` David Miller
2009-10-01 12:48 ` Christoph Hellwig
2009-10-01 16:08 ` Valdis.Kletnieks
2009-10-05 14:39 ` Peter Zijlstra
2009-09-30 20:40 ` [tip:sched/urgent] x86: Provide an alternative() based cmpxchg64() tip-bot for Arjan van de Ven
2009-09-30 20:58 ` Ingo Molnar
2009-09-30 20:40 ` [tip:sched/urgent] sched_clock: Fix atomicity/continuity bug by using cmpxchg64() tip-bot for Eric Dumazet
2009-09-30 20:55 ` [tip:sched/urgent] x86: Provide an alternative() based cmpxchg64() tip-bot for Arjan van de Ven
2009-09-30 20:55 ` [tip:sched/urgent] sched_clock: Fix atomicity/continuity bug by using cmpxchg64() tip-bot for Eric Dumazet
2009-09-30 21:00 ` [tip:sched/urgent] x86: Provide an alternative() based cmpxchg64() tip-bot for Arjan van de Ven
2009-09-30 21:01 ` [tip:sched/urgent] sched_clock: Fix atomicity/continuity bug by using cmpxchg64() tip-bot for Eric Dumazet
2009-10-05 16:00 ` [PATCH] x86: Generate cmpxchg build failures Peter Zijlstra
2009-10-05 18:51 ` Maciej Żenczykowski
2009-10-05 19:16 ` Linus Torvalds
2009-10-05 19:33 ` Peter Zijlstra
2009-10-05 20:54 ` Linus Torvalds
2009-10-09 14:23 ` [tip:x86/asm] " tip-bot for Peter Zijlstra
2009-09-28 14:34 ` Linux 2.6.32-rc1 compile error Wolfgang Erig
2009-09-28 15:10 ` Jaswinder Singh Rajput
2009-09-28 15:32 ` Wolfgang Erig
2009-09-28 16:25 ` [PATCH] isdn: fix netjet/isdnhdlc build errors Randy Dunlap
2009-09-28 19:47 ` David Miller
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=tip-8da752098d4e584fbcc149046e5c716f5d356db4@git.kernel.org \
--to=arjan@infradead.org \
--cc=a.p.zijlstra@chello.nl \
--cc=arjan@linux.intel.com \
--cc=eric.dumazet@gmail.com \
--cc=hpa@zytor.com \
--cc=johnstul@us.ibm.com \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-tip-commits@vger.kernel.org \
--cc=mingo@elte.hu \
--cc=mingo@redhat.com \
--cc=schwidefsky@de.ibm.com \
--cc=tglx@linutronix.de \
--cc=torvalds@linux-foundation.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.