From: Jeff Layton <jlayton@kernel.org>
To: Steven Rostedt <rostedt@goodmis.org>,
Masami Hiramatsu <mhiramat@kernel.org>,
Mathieu Desnoyers <mathieu.desnoyers@efficios.com>,
Chuck Lever <chuck.lever@oracle.com>,
Alexander Viro <viro@zeniv.linux.org.uk>,
Christian Brauner <brauner@kernel.org>, Jan Kara <jack@suse.cz>,
Eric Van Hensbergen <ericvh@kernel.org>,
Latchesar Ionkov <lucho@ionkov.net>,
Dominique Martinet <asmadeus@codewreck.org>,
Christian Schoenebeck <linux_oss@crudebyte.com>,
David Howells <dhowells@redhat.com>,
Marc Dionne <marc.dionne@auristor.com>,
Xiubo Li <xiubli@redhat.com>, Ilya Dryomov <idryomov@gmail.com>,
Alexander Aring <aahringo@redhat.com>,
David Teigland <teigland@redhat.com>,
Andreas Gruenbacher <agruenba@redhat.com>,
Neil Brown <neilb@suse.de>, Olga Kornievskaia <kolga@netapp.com>,
Dai Ngo <Dai.Ngo@oracle.com>, Tom Talpey <tom@talpey.com>,
Trond Myklebust <trond.myklebust@hammerspace.com>,
Anna Schumaker <anna@kernel.org>, Mark Fasheh <mark@fasheh.com>,
Joel Becker <jlbec@evilplan.org>,
Joseph Qi <joseph.qi@linux.alibaba.com>,
Steve French <sfrench@samba.org>,
Paulo Alcantara <pc@manguebit.com>,
Ronnie Sahlberg <ronniesahlberg@gmail.com>,
Shyam Prasad N <sprasad@microsoft.com>,
Namjae Jeon <linkinjeon@kernel.org>,
Sergey Senozhatsky <senozhatsky@chromium.org>,
Miklos Szeredi <miklos@szeredi.hu>
Cc: linux-kernel@vger.kernel.org, linux-trace-kernel@vger.kernel.org,
linux-fsdevel@vger.kernel.org, v9fs@lists.linux.dev,
linux-afs@lists.infradead.org, ceph-devel@vger.kernel.org,
gfs2@lists.linux.dev, linux-nfs@vger.kernel.org,
ocfs2-devel@lists.linux.dev, linux-cifs@vger.kernel.org
Subject: Re: [PATCH v3 25/47] filelock: convert __locks_insert_block, conflict and deadlock checks to use file_lock_core
Date: Sun, 18 Feb 2024 07:16:34 -0500 [thread overview]
Message-ID: <fb748c596936651a6b42781653f0ca29dc45fb5b.camel@kernel.org> (raw)
In-Reply-To: <20240131-flsplit-v3-25-c6129007ee8d@kernel.org>
On Wed, 2024-01-31 at 18:02 -0500, Jeff Layton wrote:
> Have both __locks_insert_block and the deadlock and conflict checking
> functions take a struct file_lock_core pointer instead of a struct
> file_lock one. Also, change posix_locks_deadlock to return bool.
>
> Signed-off-by: Jeff Layton <jlayton@kernel.org>
> ---
> fs/locks.c | 132 +++++++++++++++++++++++++++++++++----------------------------
> 1 file changed, 72 insertions(+), 60 deletions(-)
>
> diff --git a/fs/locks.c b/fs/locks.c
> index 1e8b943bd7f9..0dc1c9da858c 100644
> --- a/fs/locks.c
> +++ b/fs/locks.c
> @@ -757,39 +757,41 @@ EXPORT_SYMBOL(locks_delete_block);
> * waiters, and add beneath any waiter that blocks the new waiter.
> * Thus wakeups don't happen until needed.
> */
> -static void __locks_insert_block(struct file_lock *blocker,
> - struct file_lock *waiter,
> - bool conflict(struct file_lock *,
> - struct file_lock *))
> +static void __locks_insert_block(struct file_lock *blocker_fl,
> + struct file_lock *waiter_fl,
> + bool conflict(struct file_lock_core *,
> + struct file_lock_core *))
> {
> - struct file_lock *fl;
> - BUG_ON(!list_empty(&waiter->c.flc_blocked_member));
> + struct file_lock_core *blocker = &blocker_fl->c;
> + struct file_lock_core *waiter = &waiter_fl->c;
> + struct file_lock_core *flc;
>
> + BUG_ON(!list_empty(&waiter->flc_blocked_member));
> new_blocker:
> - list_for_each_entry(fl, &blocker->c.flc_blocked_requests,
> - c.flc_blocked_member)
> - if (conflict(fl, waiter)) {
> - blocker = fl;
> + list_for_each_entry(flc, &blocker->flc_blocked_requests, flc_blocked_member)
> + if (conflict(flc, waiter)) {
> + blocker = flc;
> goto new_blocker;
> }
> - waiter->c.flc_blocker = blocker;
> - list_add_tail(&waiter->c.flc_blocked_member,
> - &blocker->c.flc_blocked_requests);
> - if ((blocker->c.flc_flags & (FL_POSIX|FL_OFDLCK)) == FL_POSIX)
> - locks_insert_global_blocked(&waiter->c);
> + waiter->flc_blocker = file_lock(blocker);
> + list_add_tail(&waiter->flc_blocked_member,
> + &blocker->flc_blocked_requests);
>
> - /* The requests in waiter->fl_blocked are known to conflict with
> + if ((blocker->flc_flags & (FL_POSIX|FL_OFDLCK)) == (FL_POSIX|FL_OFDLCK))
Christian,
There is a bug in the above delta. That should read:
if ((blocker->flc_flags & (FL_POSIX|FL_OFDLCK)) == FL_POSIX)
I suspect that is the cause of the performance regression noted by the
KTR.
I believe the bug is fairly harmless -- it's just putting OFD locks into
the global hash when it doesn't need to, which probably slows down
deadlock checking. I'm going to spin up a patch and test it today, but I
wanted to give you a heads up.
I'll send the patch later today or tomorrow.
> + locks_insert_global_blocked(waiter);
> +
> + /* The requests in waiter->flc_blocked are known to conflict with
> * waiter, but might not conflict with blocker, or the requests
> * and lock which block it. So they all need to be woken.
> */
> - __locks_wake_up_blocks(&waiter->c);
> + __locks_wake_up_blocks(waiter);
> }
>
> /* Must be called with flc_lock held. */
> static void locks_insert_block(struct file_lock *blocker,
> struct file_lock *waiter,
> - bool conflict(struct file_lock *,
> - struct file_lock *))
> + bool conflict(struct file_lock_core *,
> + struct file_lock_core *))
> {
> spin_lock(&blocked_lock_lock);
> __locks_insert_block(blocker, waiter, conflict);
> @@ -846,12 +848,12 @@ locks_delete_lock_ctx(struct file_lock *fl, struct list_head *dispose)
> /* Determine if lock sys_fl blocks lock caller_fl. Common functionality
> * checks for shared/exclusive status of overlapping locks.
> */
> -static bool locks_conflict(struct file_lock *caller_fl,
> - struct file_lock *sys_fl)
> +static bool locks_conflict(struct file_lock_core *caller_flc,
> + struct file_lock_core *sys_flc)
> {
> - if (lock_is_write(sys_fl))
> + if (sys_flc->flc_type == F_WRLCK)
> return true;
> - if (lock_is_write(caller_fl))
> + if (caller_flc->flc_type == F_WRLCK)
> return true;
> return false;
> }
> @@ -859,20 +861,23 @@ static bool locks_conflict(struct file_lock *caller_fl,
> /* Determine if lock sys_fl blocks lock caller_fl. POSIX specific
> * checking before calling the locks_conflict().
> */
> -static bool posix_locks_conflict(struct file_lock *caller_fl,
> - struct file_lock *sys_fl)
> +static bool posix_locks_conflict(struct file_lock_core *caller_flc,
> + struct file_lock_core *sys_flc)
> {
> + struct file_lock *caller_fl = file_lock(caller_flc);
> + struct file_lock *sys_fl = file_lock(sys_flc);
> +
> /* POSIX locks owned by the same process do not conflict with
> * each other.
> */
> - if (posix_same_owner(&caller_fl->c, &sys_fl->c))
> + if (posix_same_owner(caller_flc, sys_flc))
> return false;
>
> /* Check whether they overlap */
> if (!locks_overlap(caller_fl, sys_fl))
> return false;
>
> - return locks_conflict(caller_fl, sys_fl);
> + return locks_conflict(caller_flc, sys_flc);
> }
>
> /* Determine if lock sys_fl blocks lock caller_fl. Used on xx_GETLK
> @@ -881,28 +886,31 @@ static bool posix_locks_conflict(struct file_lock *caller_fl,
> static bool posix_test_locks_conflict(struct file_lock *caller_fl,
> struct file_lock *sys_fl)
> {
> + struct file_lock_core *caller = &caller_fl->c;
> + struct file_lock_core *sys = &sys_fl->c;
> +
> /* F_UNLCK checks any locks on the same fd. */
> if (lock_is_unlock(caller_fl)) {
> - if (!posix_same_owner(&caller_fl->c, &sys_fl->c))
> + if (!posix_same_owner(caller, sys))
> return false;
> return locks_overlap(caller_fl, sys_fl);
> }
> - return posix_locks_conflict(caller_fl, sys_fl);
> + return posix_locks_conflict(caller, sys);
> }
>
> /* Determine if lock sys_fl blocks lock caller_fl. FLOCK specific
> * checking before calling the locks_conflict().
> */
> -static bool flock_locks_conflict(struct file_lock *caller_fl,
> - struct file_lock *sys_fl)
> +static bool flock_locks_conflict(struct file_lock_core *caller_flc,
> + struct file_lock_core *sys_flc)
> {
> /* FLOCK locks referring to the same filp do not conflict with
> * each other.
> */
> - if (caller_fl->c.flc_file == sys_fl->c.flc_file)
> + if (caller_flc->flc_file == sys_flc->flc_file)
> return false;
>
> - return locks_conflict(caller_fl, sys_fl);
> + return locks_conflict(caller_flc, sys_flc);
> }
>
> void
> @@ -980,25 +988,27 @@ EXPORT_SYMBOL(posix_test_lock);
>
> #define MAX_DEADLK_ITERATIONS 10
>
> -/* Find a lock that the owner of the given block_fl is blocking on. */
> -static struct file_lock *what_owner_is_waiting_for(struct file_lock *block_fl)
> +/* Find a lock that the owner of the given @blocker is blocking on. */
> +static struct file_lock_core *what_owner_is_waiting_for(struct file_lock_core *blocker)
> {
> - struct file_lock *fl;
> + struct file_lock_core *flc;
>
> - hash_for_each_possible(blocked_hash, fl, c.flc_link, posix_owner_key(&block_fl->c)) {
> - if (posix_same_owner(&fl->c, &block_fl->c)) {
> - while (fl->c.flc_blocker)
> - fl = fl->c.flc_blocker;
> - return fl;
> + hash_for_each_possible(blocked_hash, flc, flc_link, posix_owner_key(blocker)) {
> + if (posix_same_owner(flc, blocker)) {
> + while (flc->flc_blocker)
> + flc = &flc->flc_blocker->c;
> + return flc;
> }
> }
> return NULL;
> }
>
> /* Must be called with the blocked_lock_lock held! */
> -static int posix_locks_deadlock(struct file_lock *caller_fl,
> - struct file_lock *block_fl)
> +static bool posix_locks_deadlock(struct file_lock *caller_fl,
> + struct file_lock *block_fl)
> {
> + struct file_lock_core *caller = &caller_fl->c;
> + struct file_lock_core *blocker = &block_fl->c;
> int i = 0;
>
> lockdep_assert_held(&blocked_lock_lock);
> @@ -1007,16 +1017,16 @@ static int posix_locks_deadlock(struct file_lock *caller_fl,
> * This deadlock detector can't reasonably detect deadlocks with
> * FL_OFDLCK locks, since they aren't owned by a process, per-se.
> */
> - if (caller_fl->c.flc_flags & FL_OFDLCK)
> - return 0;
> + if (caller->flc_flags & FL_OFDLCK)
> + return false;
>
> - while ((block_fl = what_owner_is_waiting_for(block_fl))) {
> + while ((blocker = what_owner_is_waiting_for(blocker))) {
> if (i++ > MAX_DEADLK_ITERATIONS)
> - return 0;
> - if (posix_same_owner(&caller_fl->c, &block_fl->c))
> - return 1;
> + return false;
> + if (posix_same_owner(caller, blocker))
> + return true;
> }
> - return 0;
> + return false;
> }
>
> /* Try to create a FLOCK lock on filp. We always insert new FLOCK locks
> @@ -1071,7 +1081,7 @@ static int flock_lock_inode(struct inode *inode, struct file_lock *request)
>
> find_conflict:
> list_for_each_entry(fl, &ctx->flc_flock, c.flc_list) {
> - if (!flock_locks_conflict(request, fl))
> + if (!flock_locks_conflict(&request->c, &fl->c))
> continue;
> error = -EAGAIN;
> if (!(request->c.flc_flags & FL_SLEEP))
> @@ -1140,7 +1150,7 @@ static int posix_lock_inode(struct inode *inode, struct file_lock *request,
> */
> if (request->c.flc_type != F_UNLCK) {
> list_for_each_entry(fl, &ctx->flc_posix, c.flc_list) {
> - if (!posix_locks_conflict(request, fl))
> + if (!posix_locks_conflict(&request->c, &fl->c))
> continue;
> if (fl->fl_lmops && fl->fl_lmops->lm_lock_expirable
> && (*fl->fl_lmops->lm_lock_expirable)(fl)) {
> @@ -1442,23 +1452,25 @@ static void time_out_leases(struct inode *inode, struct list_head *dispose)
> }
> }
>
> -static bool leases_conflict(struct file_lock *lease, struct file_lock *breaker)
> +static bool leases_conflict(struct file_lock_core *lc, struct file_lock_core *bc)
> {
> bool rc;
> + struct file_lock *lease = file_lock(lc);
> + struct file_lock *breaker = file_lock(bc);
>
> if (lease->fl_lmops->lm_breaker_owns_lease
> && lease->fl_lmops->lm_breaker_owns_lease(lease))
> return false;
> - if ((breaker->c.flc_flags & FL_LAYOUT) != (lease->c.flc_flags & FL_LAYOUT)) {
> + if ((bc->flc_flags & FL_LAYOUT) != (lc->flc_flags & FL_LAYOUT)) {
> rc = false;
> goto trace;
> }
> - if ((breaker->c.flc_flags & FL_DELEG) && (lease->c.flc_flags & FL_LEASE)) {
> + if ((bc->flc_flags & FL_DELEG) && (lc->flc_flags & FL_LEASE)) {
> rc = false;
> goto trace;
> }
>
> - rc = locks_conflict(breaker, lease);
> + rc = locks_conflict(bc, lc);
> trace:
> trace_leases_conflict(rc, lease, breaker);
> return rc;
> @@ -1468,12 +1480,12 @@ static bool
> any_leases_conflict(struct inode *inode, struct file_lock *breaker)
> {
> struct file_lock_context *ctx = inode->i_flctx;
> - struct file_lock *fl;
> + struct file_lock_core *flc;
>
> lockdep_assert_held(&ctx->flc_lock);
>
> - list_for_each_entry(fl, &ctx->flc_lease, c.flc_list) {
> - if (leases_conflict(fl, breaker))
> + list_for_each_entry(flc, &ctx->flc_lease, flc_list) {
> + if (leases_conflict(flc, &breaker->c))
> return true;
> }
> return false;
> @@ -1529,7 +1541,7 @@ int __break_lease(struct inode *inode, unsigned int mode, unsigned int type)
> }
>
> list_for_each_entry_safe(fl, tmp, &ctx->flc_lease, c.flc_list) {
> - if (!leases_conflict(fl, new_fl))
> + if (!leases_conflict(&fl->c, &new_fl->c))
> continue;
> if (want_write) {
> if (fl->c.flc_flags & FL_UNLOCK_PENDING)
>
--
Jeff Layton <jlayton@kernel.org>
next prev parent reply other threads:[~2024-02-18 12:16 UTC|newest]
Thread overview: 57+ messages / expand[flat|nested] mbox.gz Atom feed top
2024-01-31 23:01 [PATCH v3 00/47] filelock: split file leases out of struct file_lock Jeff Layton
2024-01-31 23:01 ` [PATCH v3 01/47] filelock: fl_pid field should be signed int Jeff Layton
2024-01-31 23:01 ` [PATCH v3 02/47] filelock: rename some fields in tracepoints Jeff Layton
2024-01-31 23:01 ` [PATCH v3 03/47] filelock: rename fl_pid variable in lock_get_status Jeff Layton
2024-01-31 23:01 ` [PATCH v3 04/47] filelock: add some new helper functions Jeff Layton
2024-02-05 11:36 ` Christian Brauner
2024-02-05 11:55 ` Jeff Layton
2024-02-05 11:57 ` Christian Brauner
2024-02-05 12:06 ` Jeff Layton
2024-02-05 12:19 ` Christian Brauner
2024-01-31 23:01 ` [PATCH v3 05/47] 9p: rename fl_type variable in v9fs_file_do_lock Jeff Layton
2024-01-31 23:01 ` [PATCH v3 06/47] afs: convert to using new filelock helpers Jeff Layton
2024-01-31 23:01 ` [PATCH v3 07/47] ceph: " Jeff Layton
2024-01-31 23:01 ` [PATCH v3 08/47] dlm: " Jeff Layton
2024-01-31 23:01 ` [PATCH v3 09/47] gfs2: " Jeff Layton
2024-01-31 23:01 ` [PATCH v3 10/47] lockd: " Jeff Layton
2024-01-31 23:01 ` [PATCH v3 11/47] nfs: " Jeff Layton
2024-01-31 23:01 ` [PATCH v3 12/47] nfsd: " Jeff Layton
2024-01-31 23:01 ` [PATCH v3 13/47] ocfs2: " Jeff Layton
2024-01-31 23:01 ` [PATCH v3 14/47] smb/client: " Jeff Layton
2024-01-31 23:01 ` [PATCH v3 15/47] smb/server: " Jeff Layton
2024-01-31 23:01 ` [PATCH v3 16/47] filelock: drop the IS_* macros Jeff Layton
2024-01-31 23:01 ` [PATCH v3 17/47] filelock: split common fields into struct file_lock_core Jeff Layton
2024-01-31 23:01 ` [PATCH v3 18/47] filelock: have fs/locks.c deal with file_lock_core directly Jeff Layton
2024-01-31 23:02 ` [PATCH v3 19/47] filelock: convert more internal functions to use file_lock_core Jeff Layton
2024-01-31 23:02 ` [PATCH v3 20/47] filelock: make posix_same_owner take file_lock_core pointers Jeff Layton
2024-01-31 23:02 ` [PATCH v3 21/47] filelock: convert posix_owner_key to take file_lock_core arg Jeff Layton
2024-01-31 23:02 ` [PATCH v3 22/47] filelock: make locks_{insert,delete}_global_locks " Jeff Layton
2024-01-31 23:02 ` [PATCH v3 23/47] filelock: convert locks_{insert,delete}_global_blocked Jeff Layton
2024-01-31 23:02 ` [PATCH v3 24/47] filelock: make __locks_delete_block and __locks_wake_up_blocks take file_lock_core Jeff Layton
2024-01-31 23:02 ` [PATCH v3 25/47] filelock: convert __locks_insert_block, conflict and deadlock checks to use file_lock_core Jeff Layton
2024-02-18 12:16 ` Jeff Layton [this message]
2024-01-31 23:02 ` [PATCH v3 26/47] filelock: convert fl_blocker to file_lock_core Jeff Layton
2024-01-31 23:02 ` [PATCH v3 27/47] filelock: clean up locks_delete_block internals Jeff Layton
2024-01-31 23:02 ` [PATCH v3 28/47] filelock: reorganize locks_delete_block and __locks_insert_block Jeff Layton
2024-01-31 23:02 ` [PATCH v3 29/47] filelock: make assign_type helper take a file_lock_core pointer Jeff Layton
2024-01-31 23:02 ` [PATCH v3 30/47] filelock: convert locks_wake_up_blocks to " Jeff Layton
2024-01-31 23:02 ` [PATCH v3 31/47] filelock: convert locks_insert_lock_ctx and locks_delete_lock_ctx Jeff Layton
2024-01-31 23:02 ` [PATCH v3 32/47] filelock: convert locks_translate_pid to take file_lock_core Jeff Layton
2024-01-31 23:02 ` [PATCH v3 33/47] filelock: convert seqfile handling to use file_lock_core Jeff Layton
2024-01-31 23:02 ` [PATCH v3 34/47] 9p: adapt to breakup of struct file_lock Jeff Layton
2024-02-01 22:40 ` NeilBrown
2024-01-31 23:02 ` [PATCH v3 35/47] afs: " Jeff Layton
2024-01-31 23:02 ` [PATCH v3 36/47] ceph: " Jeff Layton
2024-01-31 23:02 ` [PATCH v3 37/47] dlm: " Jeff Layton
2024-01-31 23:02 ` [PATCH v3 38/47] gfs2: " Jeff Layton
2024-01-31 23:02 ` [PATCH v3 39/47] fuse: " Jeff Layton
2024-01-31 23:02 ` [PATCH v3 40/47] lockd: " Jeff Layton
2024-01-31 23:02 ` [PATCH v3 41/47] nfs: " Jeff Layton
2024-01-31 23:02 ` [PATCH v3 42/47] nfsd: " Jeff Layton
2024-01-31 23:02 ` [PATCH v3 43/47] ocfs2: " Jeff Layton
2024-01-31 23:02 ` [PATCH v3 44/47] smb/client: " Jeff Layton
2024-01-31 23:02 ` [PATCH v3 45/47] smb/server: " Jeff Layton
2024-01-31 23:02 ` [PATCH v3 46/47] filelock: remove temporary compatibility macros Jeff Layton
2024-01-31 23:02 ` [PATCH v3 47/47] filelock: split leases out of struct file_lock Jeff Layton
2024-02-01 22:49 ` [PATCH v3 00/47] filelock: split file " NeilBrown
2024-02-02 11:16 ` Christian Brauner
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=fb748c596936651a6b42781653f0ca29dc45fb5b.camel@kernel.org \
--to=jlayton@kernel.org \
--cc=Dai.Ngo@oracle.com \
--cc=aahringo@redhat.com \
--cc=agruenba@redhat.com \
--cc=anna@kernel.org \
--cc=asmadeus@codewreck.org \
--cc=brauner@kernel.org \
--cc=ceph-devel@vger.kernel.org \
--cc=chuck.lever@oracle.com \
--cc=dhowells@redhat.com \
--cc=ericvh@kernel.org \
--cc=gfs2@lists.linux.dev \
--cc=idryomov@gmail.com \
--cc=jack@suse.cz \
--cc=jlbec@evilplan.org \
--cc=joseph.qi@linux.alibaba.com \
--cc=kolga@netapp.com \
--cc=linkinjeon@kernel.org \
--cc=linux-afs@lists.infradead.org \
--cc=linux-cifs@vger.kernel.org \
--cc=linux-fsdevel@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-nfs@vger.kernel.org \
--cc=linux-trace-kernel@vger.kernel.org \
--cc=linux_oss@crudebyte.com \
--cc=lucho@ionkov.net \
--cc=marc.dionne@auristor.com \
--cc=mark@fasheh.com \
--cc=mathieu.desnoyers@efficios.com \
--cc=mhiramat@kernel.org \
--cc=miklos@szeredi.hu \
--cc=neilb@suse.de \
--cc=ocfs2-devel@lists.linux.dev \
--cc=pc@manguebit.com \
--cc=ronniesahlberg@gmail.com \
--cc=rostedt@goodmis.org \
--cc=senozhatsky@chromium.org \
--cc=sfrench@samba.org \
--cc=sprasad@microsoft.com \
--cc=teigland@redhat.com \
--cc=tom@talpey.com \
--cc=trond.myklebust@hammerspace.com \
--cc=v9fs@lists.linux.dev \
--cc=viro@zeniv.linux.org.uk \
--cc=xiubli@redhat.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).