From: Eric Wong <e@80x24.org>
To: spew@80x24.org
Subject: [PATCH 1/2] ipc: move nproc_shards from v2writable
Date: Fri, 17 Feb 2023 04:01:12 +0000 [thread overview]
Message-ID: <20230217040113.111644-1-e@80x24.org> (raw)
We'll be using nproc_shards for indexing non-Inbox stuff.
---
lib/PublicInbox/IPC.pm | 26 +++++++++++++++++++++++++-
lib/PublicInbox/V2Writable.pm | 26 +-------------------------
2 files changed, 26 insertions(+), 26 deletions(-)
diff --git a/lib/PublicInbox/IPC.pm b/lib/PublicInbox/IPC.pm
index 548a72eb..730f2cf6 100644
--- a/lib/PublicInbox/IPC.pm
+++ b/lib/PublicInbox/IPC.pm
@@ -19,7 +19,7 @@ use PublicInbox::WQWorker;
use Socket qw(AF_UNIX MSG_EOR SOCK_STREAM);
my $MY_MAX_ARG_STRLEN = 4096 * 33; # extra 4K for serialization
my $SEQPACKET = eval { Socket::SOCK_SEQPACKET() }; # portable enough?
-our @EXPORT_OK = qw(ipc_freeze ipc_thaw);
+our @EXPORT_OK = qw(ipc_freeze ipc_thaw nproc_shards);
my ($enc, $dec);
# ->imports at BEGIN turns sereal_*_with_object into custom ops on 5.14+
# and eliminate method call overhead
@@ -454,4 +454,28 @@ sub detect_nproc () {
undef
}
+# SATA storage lags behind what CPUs are capable of, so relying on
+# nproc(1) can be misleading and having extra Xapian shards is a
+# waste of FDs and space. It can also lead to excessive IO latency
+# and slow things down. Users on NVME or other fast storage can
+# use the NPROC env or switches in our script/public-inbox-* programs
+# to increase Xapian shards
+our $NPROC_MAX_DEFAULT = 4;
+
+sub nproc_shards ($) {
+ my ($creat_opt) = @_;
+ my $n = $creat_opt->{nproc} if ref($creat_opt) eq 'HASH';
+ $n //= $ENV{NPROC};
+ if (!$n) {
+ # assume 2 cores if not detectable or zero
+ state $NPROC_DETECTED = PublicInbox::IPC::detect_nproc() || 2;
+ $n = $NPROC_DETECTED;
+ $n = $NPROC_MAX_DEFAULT if $n > $NPROC_MAX_DEFAULT;
+ }
+
+ # subtract for the main process and git-fast-import
+ $n -= 1;
+ $n < 1 ? 1 : $n;
+}
+
1;
diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index ed5182ae..d3d13941 100644
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -8,7 +8,7 @@ use strict;
use v5.10.1;
use parent qw(PublicInbox::Lock PublicInbox::IPC);
use PublicInbox::SearchIdxShard;
-use PublicInbox::IPC;
+use PublicInbox::IPC qw(nproc_shards);
use PublicInbox::Eml;
use PublicInbox::Git;
use PublicInbox::Import;
@@ -29,30 +29,6 @@ my $OID = qr/[a-f0-9]{40,}/;
# an estimate of the post-packed size to the raw uncompressed size
our $PACKING_FACTOR = 0.4;
-# SATA storage lags behind what CPUs are capable of, so relying on
-# nproc(1) can be misleading and having extra Xapian shards is a
-# waste of FDs and space. It can also lead to excessive IO latency
-# and slow things down. Users on NVME or other fast storage can
-# use the NPROC env or switches in our script/public-inbox-* programs
-# to increase Xapian shards
-our $NPROC_MAX_DEFAULT = 4;
-
-sub nproc_shards ($) {
- my ($creat_opt) = @_;
- my $n = $creat_opt->{nproc} if ref($creat_opt) eq 'HASH';
- $n //= $ENV{NPROC};
- if (!$n) {
- # assume 2 cores if not detectable or zero
- state $NPROC_DETECTED = PublicInbox::IPC::detect_nproc() || 2;
- $n = $NPROC_DETECTED;
- $n = $NPROC_MAX_DEFAULT if $n > $NPROC_MAX_DEFAULT;
- }
-
- # subtract for the main process and git-fast-import
- $n -= 1;
- $n < 1 ? 1 : $n;
-}
-
sub count_shards ($) {
my ($self) = @_;
# always load existing shards in case core count changes:
next reply other threads:[~2023-02-17 4:01 UTC|newest]
Thread overview: 2+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-02-17 4:01 Eric Wong [this message]
2023-02-17 4:01 ` [PATCH 2/2] WIP-reposearchidx Eric Wong
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20230217040113.111644-1-e@80x24.org \
--to=e@80x24.org \
--cc=spew@80x24.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).