* [PATCH 1/4] mlib: compatibility with Sequel 5.x
2018-01-30 9:17 [PATCH 0/4] mlib: misc updates Eric Wong
@ 2018-01-30 9:17 ` Eric Wong
2018-01-30 9:17 ` [PATCH 2/4] mlib: remove redundant tag massaging and encoding Eric Wong
` (2 subsequent siblings)
3 siblings, 0 replies; 5+ messages in thread
From: Eric Wong @ 2018-01-30 9:17 UTC (permalink / raw)
To: dtas-all
Apparently some degree of thread-safety is being enforced;
not sure I agree, but oh well...
---
lib/dtas/mlib.rb | 15 ++++++---------
test/test_mlib.rb | 4 ++--
2 files changed, 8 insertions(+), 11 deletions(-)
diff --git a/lib/dtas/mlib.rb b/lib/dtas/mlib.rb
index 24393d5..e217b59 100644
--- a/lib/dtas/mlib.rb
+++ b/lib/dtas/mlib.rb
@@ -37,16 +37,13 @@ class DTAS::Mlib # :nodoc:
def initialize(db)
if String === db
+ require 'sequel'
+ opts = { single_threaded: true }
db = "sqlite://#{db}" unless db.include?('://')
- require 'sequel/no_core_ext'
- db = Sequel.connect(db, single_threaded: true)
- end
- if db.class.to_s.downcase.include?('sqlite')
- db.transaction_mode = :immediate
- db.synchronous = :off
- db.case_sensitive_like = false
- else
- warn 'non-SQLite databases may not work in the future'
+ opts[:transaction_mode] = :immediate
+ opts[:synchronous] = :off
+ opts[:case_sensitive_like] = false # only for 'search'
+ db = Sequel.connect(db, opts)
end
@db = db
@pwd = nil
diff --git a/test/test_mlib.rb b/test/test_mlib.rb
index a5c446f..0241314 100644
--- a/test/test_mlib.rb
+++ b/test/test_mlib.rb
@@ -4,7 +4,7 @@
require_relative 'helper'
begin
require 'dtas/mlib'
- require 'sequel/no_core_ext'
+ require 'sequel'
require 'sqlite3'
rescue LoadError => err
warn "skipping mlib test: #{err.message}"
@@ -13,7 +13,7 @@
class TestMlib < Testcase
def setup
- @db = Sequel.sqlite(':memory:')
+ @db = Sequel.sqlite(':memory:', case_sensitive_like: false)
end
def test_migrate
--
EW
^ permalink raw reply related [flat|nested] 5+ messages in thread* [PATCH 3/4] mlib: use flock to get around SQLite busy errors
2018-01-30 9:17 [PATCH 0/4] mlib: misc updates Eric Wong
2018-01-30 9:17 ` [PATCH 1/4] mlib: compatibility with Sequel 5.x Eric Wong
2018-01-30 9:17 ` [PATCH 2/4] mlib: remove redundant tag massaging and encoding Eric Wong
@ 2018-01-30 9:17 ` Eric Wong
2018-01-30 9:17 ` [PATCH 4/4] mlib: ignore files with nil times Eric Wong
3 siblings, 0 replies; 5+ messages in thread
From: Eric Wong @ 2018-01-30 9:17 UTC (permalink / raw)
To: dtas-all
The SQLite busy waiting scheme isn't great for usability
and the busy timeout is done by sleep+backoff. I prefer
to say we only support filesystems with flock() for our
little DB.
---
lib/dtas/mlib.rb | 33 ++++++++++++++++++++++++++-------
1 file changed, 26 insertions(+), 7 deletions(-)
diff --git a/lib/dtas/mlib.rb b/lib/dtas/mlib.rb
index d1707fb..3f5763f 100644
--- a/lib/dtas/mlib.rb
+++ b/lib/dtas/mlib.rb
@@ -10,6 +10,7 @@
require_relative 'source/ff'
require_relative 'source/splitfx'
require 'socket'
+require 'tempfile'
# For the DTAS Music Library, based on what MPD uses.
class DTAS::Mlib # :nodoc:
@@ -62,15 +63,24 @@ def initialize(db)
]
end
+ def synchronize
+ @lock.flock(File::LOCK_EX)
+ @db.transaction { yield }
+ ensure
+ @lock.flock(File::LOCK_UN)
+ end
+
def init_suffixes
`sox --help 2>/dev/null` =~ /\nAUDIO FILE FORMATS:\s*([^\n]+)/
re = $1.split(/\s+/).map! { |x| Regexp.quote(x) }.join('|')
@suffixes = Regexp.new("\\.(?:#{re}|yml)\\z", Regexp::IGNORECASE)
end
- def worker(todo)
+ def worker(todo, lock)
+ old_lock = @lock
+ @lock = lock
@work.close
- @db.tables # reconnect before chdir
+ synchronize { @db.tables } # reconnect before chdir
@pwd = Dir.pwd.b
begin
buf = todo.recv(16384) # 4x bigger than PATH_MAX ought to be enough
@@ -84,7 +94,7 @@ def worker(todo)
end
def ignore(job)
- @db.transaction do
+ synchronize do
node_ensure(job.parent_id, job.path, DM_IGN, job.ctime)
end
end
@@ -112,7 +122,7 @@ def worker_work(job)
tmp[tag_id] = value if value.valid_encoding?
end
end
- @db.transaction do
+ synchronize do
node_id = node_ensure(job.parent_id, path, tlen, job.ctime)[:id]
vals = @db[:vals]
comments = @db[:comments]
@@ -143,11 +153,20 @@ def update(path, opts = nil)
@work and raise 'update already running'
todo, @work = UNIXSocket.pair(:SOCK_SEQPACKET)
@db.disconnect
- jobs.times { |i| fork { worker(todo) } }
+
+ # like a Mutex between processes
+ @lock = Tempfile.new('dtas.mlib.lock')
+ jobs.times do |i|
+ lock = File.open(@lock.path, 'w')
+ fork { worker(todo, lock) }
+ lock.close
+ end
+ @lock.unlink
todo.close
scan_dir(path, st)
@work.close
Process.waitall
+ @lock.close
ensure
@work = nil
end
@@ -235,7 +254,7 @@ def root_node
end
def dir_vivify(parts, ctime)
- @db.transaction do
+ synchronize do
dir = root_node
last = parts.pop
parts.each do |name|
@@ -289,7 +308,7 @@ def scan_dir(path, st, parent_id = nil)
dir = dir_vivify(@pwd.split(%r{/+}n), st.ctime.to_i)
dir_id = dir[:id]
- @db.transaction do
+ synchronize do
@db[:nodes].where(parent_id: dir_id).each do |node|
File.exist?(node[:name]) or remove_entry(node)
end
--
EW
^ permalink raw reply related [flat|nested] 5+ messages in thread