diff options
author | Eric Wong <e@80x24.org> | 2017-02-09 21:11:00 +0000 |
---|---|---|
committer | Eric Wong <e@80x24.org> | 2017-02-09 21:11:00 +0000 |
commit | e6dfbf68639876580a63ed0502174c4e7f8fa722 (patch) | |
tree | 397676fd8cec3ca60a0887814706fa6c3c6733ad /lib | |
parent | d9563ea5516e8e786debf223e10ec11695aee9d7 (diff) | |
download | public-inbox-e6dfbf68639876580a63ed0502174c4e7f8fa722.tar.gz |
We need to flush Xapian more frequently to account for gigantic commits which introduce lots of text, so do it when accounting for each line processed, and not for each commit processed.
Diffstat (limited to 'lib')
-rw-r--r-- | lib/PublicInbox/RepoGitSearchIdx.pm | 10 |
1 files changed, 5 insertions, 5 deletions
diff --git a/lib/PublicInbox/RepoGitSearchIdx.pm b/lib/PublicInbox/RepoGitSearchIdx.pm index 333558ca..bb92b0ff 100644 --- a/lib/PublicInbox/RepoGitSearchIdx.pm +++ b/lib/PublicInbox/RepoGitSearchIdx.pm @@ -197,6 +197,11 @@ sub each_log_line ($$) { local $/ = "\n"; while (defined(my $l = <$log>)) { $batch -= bytes::length($l); + # prevent memory growth from Xapian + if ($batch <= 0) { + $db->flush; + $batch = BATCH_BYTES; + } if ($l =~ /^commit (\S+)(\s+\([^\)]+\))?/) { my ($oid, $decor) = ($1, $2); commit_doc($self, $doc_id, $doc) if $doc; @@ -204,11 +209,6 @@ sub each_log_line ($$) { $state = 0; $cc_ins = $cc_del = undef; - # prevent OOM - if ($batch <= 0) { - $db->flush; - $batch = BATCH_BYTES; - } $doc = get_doc($self, \$doc_id, 'commit', $oid); decor_update($self, $doc, $decor, $oid) if $decor; # old commit |