[PATCH 0/4] mlib: misc updates

[PATCH 0/4] mlib: misc updates

[Eric Wong]

Admittedly, I haven't used mlib in a while, so there was some
bitrot.  However, Sequel 5.x is out and it can benefit from
the encoding tweaks I did the other day.

Eric Wong (4):
      mlib: compatibility with Sequel 5.x
      mlib: remove redundant tag massaging and encoding
      mlib: use flock to get around SQLite busy errors
      mlib: ignore files with nil times

 lib/dtas/mlib.rb  | 66 +++++++++++++++++++++++++++++++------------------------
 test/test_mlib.rb |  4 ++--
 2 files changed, 39 insertions(+), 31 deletions(-)

[PATCH 1/4] mlib: compatibility with Sequel 5.x

[Eric Wong]

Apparently some degree of thread-safety is being enforced;
not sure I agree, but oh well...
---
 lib/dtas/mlib.rb  | 15 ++++++---------
 test/test_mlib.rb |  4 ++--
 2 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/lib/dtas/mlib.rb b/lib/dtas/mlib.rb
index 24393d5..e217b59 100644
--- a/lib/dtas/mlib.rb
+++ b/lib/dtas/mlib.rb
@@ -37,16 +37,13 @@ class DTAS::Mlib # :nodoc:
 
   def initialize(db)
     if String === db
+      require 'sequel'
+      opts = { single_threaded: true }
       db = "sqlite://#{db}" unless db.include?('://')
-      require 'sequel/no_core_ext'
-      db = Sequel.connect(db, single_threaded: true)
-    end
-    if db.class.to_s.downcase.include?('sqlite')
-      db.transaction_mode = :immediate
-      db.synchronous = :off
-      db.case_sensitive_like = false
-    else
-      warn 'non-SQLite databases may not work in the future'
+      opts[:transaction_mode] = :immediate
+      opts[:synchronous] = :off
+      opts[:case_sensitive_like] = false # only for 'search'
+      db = Sequel.connect(db, opts)
     end
     @db = db
     @pwd = nil
diff --git a/test/test_mlib.rb b/test/test_mlib.rb
index a5c446f..0241314 100644
--- a/test/test_mlib.rb
+++ b/test/test_mlib.rb
@@ -4,7 +4,7 @@
 require_relative 'helper'
 begin
   require 'dtas/mlib'
-  require 'sequel/no_core_ext'
+  require 'sequel'
   require 'sqlite3'
 rescue LoadError => err
   warn "skipping mlib test: #{err.message}"
@@ -13,7 +13,7 @@
 
 class TestMlib < Testcase
   def setup
-    @db = Sequel.sqlite(':memory:')
+    @db = Sequel.sqlite(':memory:', case_sensitive_like: false)
   end
 
   def test_migrate
-- 
EW

[PATCH 2/4] mlib: remove redundant tag massaging and encoding

[Eric Wong]

Redundant since ("player: support guessing encodings for comments")
---
 lib/dtas/mlib.rb | 16 ++++------------
 1 file changed, 4 insertions(+), 12 deletions(-)

diff --git a/lib/dtas/mlib.rb b/lib/dtas/mlib.rb
index e217b59..d1707fb 100644
--- a/lib/dtas/mlib.rb
+++ b/lib/dtas/mlib.rb
@@ -106,18 +106,10 @@ def worker_work(job)
     return ignore(job) if tlen < 0
     tlen = tlen.round
     tmp = {}
-    found.comments.each do |tag, value|
-      tag_id = @tag_map[tag] or next
-      value.strip!
-
-      # FIXME: this fallback needs testing
-      [ Encoding::UTF_8, Encoding::ISO_8859_1 ].each do |enc|
-        value.force_encoding(enc)
-        if value.valid_encoding?
-          value.encode!(Encoding::UTF_8) if enc != Encoding::UTF_8
-          tmp[tag_id] = value
-          break
-        end
+    if comments = found.comments
+      comments.each do |tag, value|
+        tag_id = @tag_map[tag] or next
+        tmp[tag_id] = value if value.valid_encoding?
       end
     end
     @db.transaction do
-- 
EW

[PATCH 3/4] mlib: use flock to get around SQLite busy errors

[Eric Wong]

The SQLite busy waiting scheme isn't great for usability
and the busy timeout is done by sleep+backoff.  I prefer
to say we only support filesystems with flock() for our
little DB.
---
 lib/dtas/mlib.rb | 33 ++++++++++++++++++++++++++-------
 1 file changed, 26 insertions(+), 7 deletions(-)

diff --git a/lib/dtas/mlib.rb b/lib/dtas/mlib.rb
index d1707fb..3f5763f 100644
--- a/lib/dtas/mlib.rb
+++ b/lib/dtas/mlib.rb
@@ -10,6 +10,7 @@
 require_relative 'source/ff'
 require_relative 'source/splitfx'
 require 'socket'
+require 'tempfile'
 
 # For the DTAS Music Library, based on what MPD uses.
 class DTAS::Mlib # :nodoc:
@@ -62,15 +63,24 @@ def initialize(db)
     ]
   end
 
+  def synchronize
+    @lock.flock(File::LOCK_EX)
+    @db.transaction { yield }
+  ensure
+    @lock.flock(File::LOCK_UN)
+  end
+
   def init_suffixes
     `sox --help 2>/dev/null` =~ /\nAUDIO FILE FORMATS:\s*([^\n]+)/
     re = $1.split(/\s+/).map! { |x| Regexp.quote(x) }.join('|')
     @suffixes = Regexp.new("\\.(?:#{re}|yml)\\z", Regexp::IGNORECASE)
   end
 
-  def worker(todo)
+  def worker(todo, lock)
+    old_lock = @lock
+    @lock = lock
     @work.close
-    @db.tables # reconnect before chdir
+    synchronize { @db.tables } # reconnect before chdir
     @pwd = Dir.pwd.b
     begin
       buf = todo.recv(16384) # 4x bigger than PATH_MAX ought to be enough
@@ -84,7 +94,7 @@ def worker(todo)
   end
 
   def ignore(job)
-    @db.transaction do
+    synchronize do
       node_ensure(job.parent_id, job.path, DM_IGN, job.ctime)
     end
   end
@@ -112,7 +122,7 @@ def worker_work(job)
         tmp[tag_id] = value if value.valid_encoding?
       end
     end
-    @db.transaction do
+    synchronize do
       node_id = node_ensure(job.parent_id, path, tlen, job.ctime)[:id]
       vals = @db[:vals]
       comments = @db[:comments]
@@ -143,11 +153,20 @@ def update(path, opts = nil)
     @work and raise 'update already running'
     todo, @work = UNIXSocket.pair(:SOCK_SEQPACKET)
     @db.disconnect
-    jobs.times { |i| fork { worker(todo) } }
+
+    # like a Mutex between processes
+    @lock = Tempfile.new('dtas.mlib.lock')
+    jobs.times do |i|
+      lock = File.open(@lock.path, 'w')
+      fork { worker(todo, lock) }
+      lock.close
+    end
+    @lock.unlink
     todo.close
     scan_dir(path, st)
     @work.close
     Process.waitall
+    @lock.close
   ensure
     @work = nil
   end
@@ -235,7 +254,7 @@ def root_node
   end
 
   def dir_vivify(parts, ctime)
-    @db.transaction do
+    synchronize do
       dir = root_node
       last = parts.pop
       parts.each do |name|
@@ -289,7 +308,7 @@ def scan_dir(path, st, parent_id = nil)
       dir = dir_vivify(@pwd.split(%r{/+}n), st.ctime.to_i)
       dir_id = dir[:id]
 
-      @db.transaction do
+      synchronize do
         @db[:nodes].where(parent_id: dir_id).each do |node|
           File.exist?(node[:name]) or remove_entry(node)
         end
-- 
EW

[PATCH 4/4] mlib: ignore files with nil times

[Eric Wong]

It happens with some video files, apparently.
---
 lib/dtas/mlib.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/dtas/mlib.rb b/lib/dtas/mlib.rb
index 3f5763f..5395fa1 100644
--- a/lib/dtas/mlib.rb
+++ b/lib/dtas/mlib.rb
@@ -113,7 +113,7 @@ def worker_work(job)
     end
     return ignore(job) unless found
     tlen = found.duration
-    return ignore(job) if tlen < 0
+    return ignore(job) if tlen.nil? || tlen < 0
     tlen = tlen.round
     tmp = {}
     if comments = found.comments
-- 
EW