From 2752bc2b24d1e2c96c18b5f18372fc25e0ffc1fa Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Mon, 24 Jun 2019 23:29:37 +0000 Subject: msgmap: mid_insert: use plain "INSERT" to detect duplicates "INSERT OR IGNORE" still bumps the auto-increment counter in SQLite, which causes gaps to appear in NNTP article numbering. This bug appeared in v2 repos where V2Writable may call ->add repeatedly on the same message. This bug is apparent with public-inbox-watch and work-in-progress IMAP watchers which may rescan and (attempt to) reinsert the same message on mailbox changes. Most uses of public-inbox-mda were not affected, unless the same message is actually delivered multiple times to the mda. v1 is not affected, either, since deduplication is only based on Message-ID and msgmap never sees the duplicate. Reported-by: "Eric W. Biederman" --- lib/PublicInbox/Msgmap.pm | 4 ++-- t/msgmap.t | 3 +++ t/v2writable.t | 2 ++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/PublicInbox/Msgmap.pm b/lib/PublicInbox/Msgmap.pm index 0035c9e3..5a89b85a 100644 --- a/lib/PublicInbox/Msgmap.pm +++ b/lib/PublicInbox/Msgmap.pm @@ -126,9 +126,9 @@ sub mid_insert { my ($self, $mid) = @_; my $dbh = $self->{dbh}; my $sth = $dbh->prepare_cached(<<''); -INSERT OR IGNORE INTO msgmap (mid) VALUES (?) +INSERT INTO msgmap (mid) VALUES (?) - return if $sth->execute($mid) == 0; + return unless eval { $sth->execute($mid) }; my $num = $dbh->last_insert_id(undef, undef, 'msgmap', 'num'); $self->num_highwater($num) if defined($num); $num; diff --git a/t/msgmap.t b/t/msgmap.t index 4dddd0a8..20985ce8 100644 --- a/t/msgmap.t +++ b/t/msgmap.t @@ -30,6 +30,9 @@ $@ = undef; my $ret = $d->mid_insert('a@b'); is($ret, undef, 'duplicate mid_insert in undef result'); is($d->num_for('a@b'), $mid2num{'a@b'}, 'existing number not clobbered'); +my $next = (sort(keys %num2mid))[-1]; +is($d->mid_insert('ok@unique'), $next + 1, + 'got expected num after failing mid_insert'); foreach my $n (keys %num2mid) { is($d->mid_for($n), $num2mid{$n}, "num:$n maps correctly"); diff --git a/t/v2writable.t b/t/v2writable.t index 88df2d64..8f32fbe5 100644 --- a/t/v2writable.t +++ b/t/v2writable.t @@ -118,6 +118,8 @@ if ('ensure git configs are correct') { $mime->header_set('References', ''); ok($im->add($mime), 'message with multiple Message-ID'); $im->done; + my ($total, undef) = $ibx->over->recent; + is($ibx->mm->num_highwater, $total, 'got expected highwater value'); my $srch = $ibx->search; my $mset1 = $srch->reopen->query('m:abcde@1', { mset => 1 }); is($mset1->size, 1, 'message found by first MID'); -- cgit v1.2.3-24-ge0c7