From: Eric Wong <e@80x24.org>
To: spew@80x24.org
Subject: [PATCH 4/4] lei/store: stop shard workers + cat-file on idle
Date: Tue, 16 Apr 2024 20:47:28 +0000 [thread overview]
Message-ID: <20240416204728.3609009-4-e@80x24.org> (raw)
In-Reply-To: <20240416204728.3609009-1-e@80x24.org>
Schedule a timer to stop shard workers and the git-cat-file
process after a `barrier' command. This allows us to save
some RAM again when the lei process is idle but preserves
the fork overhead reduction from issuing many commands
in quick succession.
---
lib/PublicInbox/LeiStore.pm | 46 ++++++++++++++++++-------------------
1 file changed, 23 insertions(+), 23 deletions(-)
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 162c915f..a054f649 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -571,21 +571,11 @@ sub set_xvmd {
sto_export_kw($self, $smsg->{num}, $vmd);
}
-sub barrier {
+sub check_done {
my ($self) = @_;
- my ($errfh, $lei_sock) = @$self{0, 1}; # via sto_barrier_request
- my @err;
- if ($self->{im}) {
- eval { $self->{im}->barrier };
- push(@err, "E: import barrier: $@\n") if $@;
- }
- delete $self->{lms};
- eval { $self->{priv_eidx}->barrier };
- push(@err, "E: priv_eidx barrier: $@\n") if $@;
- print { $errfh // \*STDERR } @err;
- send($lei_sock, 'child_error 256', 0) if @err && $lei_sock;
- xchg_stderr($self);
- die @err if @err;
+ $self->git->_active ?
+ add_uniq_timer("$self-check_done", \&check_done, $self) :
+ done($self);
}
sub xchg_stderr {
@@ -602,23 +592,33 @@ sub xchg_stderr {
undef;
}
-sub done {
- my ($self) = @_;
- my ($errfh, $lei_sock) = @$self{0, 1};
+sub _commit ($$) {
+ my ($self, $cmd) = @_; # cmd is 'done' or 'barrier'
+ my ($errfh, $lei_sock) = @$self{0, 1}; # via sto_barrier_request
my @err;
- if (my $im = delete($self->{im})) {
- eval { $im->done };
- push(@err, "E: import done: $@\n") if $@;
+ if ($self->{im}) {
+ eval { $self->{im}->$cmd };
+ push(@err, "E: import $cmd: $@\n") if $@;
}
delete $self->{lms};
- eval { $self->{priv_eidx}->done }; # V2Writable::done
- push(@err, "E: priv_eidx done: $@\n") if $@;
- print { $errfh // *STDERR{GLOB} } @err;
+ eval { $self->{priv_eidx}->$cmd };
+ push(@err, "E: priv_eidx $cmd: $@\n") if $@;
+ print { $errfh // \*STDERR } @err;
send($lei_sock, 'child_error 256', 0) if @err && $lei_sock;
xchg_stderr($self);
die @err if @err;
+ # $lei_sock goes out-of-scope and script/lei can terminate
+}
+
+sub barrier {
+ my ($self) = @_;
+ _commit $self, 'barrier';
+ add_uniq_timer("$self-check_done", 5, \&check_done, $self);
+ undef;
}
+sub done { _commit $_[0], 'done' }
+
sub ipc_atfork_child {
my ($self) = @_;
my $lei = $self->{lei};
prev parent reply other threads:[~2024-04-16 20:47 UTC|newest]
Thread overview: 4+ messages / expand[flat|nested] mbox.gz Atom feed top
2024-04-16 20:47 [PATCH 1/4] v2 + lei/store: always wait for fast-import checkpoint Eric Wong
2024-04-16 20:47 ` [PATCH 2/4] lei: use ->barrier to commit to lei/store Eric Wong
2024-04-16 20:47 ` [PATCH 3/4] lei: use async barrier for --import-before Eric Wong
2024-04-16 20:47 ` Eric Wong [this message]
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20240416204728.3609009-4-e@80x24.org \
--to=e@80x24.org \
--cc=spew@80x24.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).