diff options
Diffstat (limited to 'lib/dtas')
57 files changed, 688 insertions, 348 deletions
diff --git a/lib/dtas/buffer.rb b/lib/dtas/buffer.rb index 39070d7..0688af9 100644 --- a/lib/dtas/buffer.rb +++ b/lib/dtas/buffer.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2019 all contributors <dtas-all@nongnu.org> +# Copyright (C) all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true require 'io/wait' @@ -8,12 +8,15 @@ require_relative '../dtas' class DTAS::Buffer # :nodoc: begin raise LoadError, "no splice with _DTAS_POSIX" if ENV["_DTAS_POSIX"] - require 'sleepy_penguin' # splice is only in Linux for now... - SleepyPenguin.respond_to?(:splice) or - raise LoadError, 'sleepy_penguin 3.5+ required for splice', [] - require_relative 'buffer/splice' - include DTAS::Buffer::Splice - rescue LoadError + # splice is only in Linux for now + begin + require_relative 'buffer/splice' + include DTAS::Buffer::Splice + rescue LoadError + require_relative 'buffer/fiddle_splice' + include DTAS::Buffer::FiddleSplice + end + rescue LoadError, StandardError require_relative 'buffer/read_write' include DTAS::Buffer::ReadWrite end @@ -42,7 +45,7 @@ class DTAS::Buffer # :nodoc: def __dst_error(dst, e) warn "dropping #{dst.inspect} due to error: #{e.message} (#{e.class})" - dst.close unless dst.closed? + dst.close end # This will modify targets diff --git a/lib/dtas/buffer/fiddle_splice.rb b/lib/dtas/buffer/fiddle_splice.rb new file mode 100644 index 0000000..d9232cd --- /dev/null +++ b/lib/dtas/buffer/fiddle_splice.rb @@ -0,0 +1,217 @@ +# Copyright (C) 2013-2020 all contributors <dtas-all@nongnu.org> +# License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> +# frozen_string_literal: true +require 'io/nonblock' +require 'fiddle' # require_relative caller should expect LoadError +require_relative '../../dtas' +require_relative '../pipe' + +# Used by -player on Linux systems with the "splice" syscall +module DTAS::Buffer::FiddleSplice # :nodoc: + MAX_AT_ONCE = 4096 # page size in Linux + MAX_AT_ONCE_1 = 65536 + F_MOVE = 1 + F_NONBLOCK = 2 + + Splice = Fiddle::Function.new(DTAS.libc['splice'], [ + Fiddle::TYPE_INT, # int fd_in, + Fiddle::TYPE_VOIDP, # loff_t *off_in + Fiddle::TYPE_INT, # int fd_out + Fiddle::TYPE_VOIDP, # loff_t *off_out + Fiddle::TYPE_SIZE_T, # size_t len + Fiddle::TYPE_INT, # unsigned int flags + ], + Fiddle::TYPE_SSIZE_T) # ssize_t + + Tee = Fiddle::Function.new(DTAS.libc['tee'], [ + Fiddle::TYPE_INT, # int fd_in, + Fiddle::TYPE_INT, # int fd_out + Fiddle::TYPE_SIZE_T, # size_t len + Fiddle::TYPE_INT, # unsigned int flags + ], + Fiddle::TYPE_SSIZE_T) # ssize_t + + def _syserr(s, func) + raise "BUG: we should not encounter EOF on #{func}" if s == 0 + case errno = Fiddle.last_error + when Errno::EAGAIN::Errno + return :EAGAIN + when Errno::EPIPE::Errno + raise Errno::EPIPE.exception + when Errno::EINTR::Errno + return nil + else + raise SystemCallError, "#{func} error: #{errno}" + end + end + + def splice(src, dst, len, flags) + begin + s = Splice.call(src.fileno, nil, dst.fileno, nil, len, flags) + return s if s > 0 + sym = _syserr(s, 'splice') and return sym + end while true + end + + def tee(src, dst, len, flags = 0) + begin + s = Tee.call(src.fileno, dst.fileno, len, flags) + return s if s > 0 + sym = _syserr(s, 'tee') and return sym + end while true + end + + def buffer_size + @to_io.pipe_size + end + + # nil is OK, won't reset existing pipe, either... + def buffer_size=(bytes) + @to_io.pipe_size = bytes if bytes + @buffer_size = bytes + end + + # be sure to only call this with nil when all writers to @wr are done + def discard(bytes) + splice(@to_io, DTAS.null, bytes, 0) + end + + def broadcast_one(targets, limit = nil) + # single output is always non-blocking + limit ||= MAX_AT_ONCE_1 + s = splice(@to_io, targets[0], limit, F_MOVE|F_NONBLOCK) + if Symbol === s + targets # our one and only target blocked on write + else + @bytes_xfer += s + # s < limit means targets[0] is full + s < limit ? targets : :wait_readable + end + rescue Errno::EPIPE, IOError => e + __dst_error(targets[0], e) + targets.clear + nil # do not return error here, we already spewed an error message + end + + def __tee_in_full(src, dst, bytes) + rv = 0 + while bytes > 0 + s = tee(src, dst, bytes) + bytes -= s + rv += s + end + rv + end + + def __splice_in_full(src, dst, bytes, flags) + rv = 0 + while bytes > 0 + s = splice(src, dst, bytes, flags) + rv += s + bytes -= s + end + rv + end + + # returns the largest value we teed + def __broadcast_tee(blocked, targets, chunk_size) + most_teed = 0 + targets.delete_if do |dst| + begin + t = (dst.nonblock? || most_teed == 0) ? + tee(@to_io, dst, chunk_size, F_NONBLOCK) : + __tee_in_full(@to_io, dst, chunk_size) + if Integer === t + if t > most_teed + chunk_size = t if most_teed == 0 + most_teed = t + end + else + blocked << dst + end + false + rescue IOError, Errno::EPIPE => e + __dst_error(dst, e) + true + end + end + most_teed + end + + def broadcast_inf(targets, limit = nil) + if targets.all?(&:ready_write_optimized?) + blocked = [] + elsif targets.none?(&:nonblock?) + # if all targets are blocking, don't start until they're all writable + r = IO.select(nil, targets, nil, 0) or return targets + blocked = targets - r[1] + + # tell DTAS::UNIXServer#run_once to wait on the blocked targets + return blocked if blocked[0] + + # all writable, yay! + else + blocked = [] + end + + # don't pin too much on one target + bytes = limit || MAX_AT_ONCE + last = targets.pop # we splice to the last one, tee to the rest + + # this may return zero if all targets were non-blocking + most_teed = __broadcast_tee(blocked, targets, bytes) + + # don't splice more than the largest amount we successfully teed + bytes = most_teed if most_teed > 0 + + begin + targets << last + if last.nonblock? || most_teed == 0 + s = splice(@to_io, last, bytes, F_MOVE|F_NONBLOCK) + if Symbol === s + blocked << last + + # we accomplished nothing! + # If _all_ writers are blocked, do not discard data, + # stay blocked on :wait_writable + return blocked if most_teed == 0 + + # the tees targets win, drop data intended for last + if most_teed > 0 + discard(most_teed) + @bytes_xfer += most_teed + # do not watch for writability of last, last is non-blocking + return :wait_readable + end + end + else + # the blocking case is simple + s = __splice_in_full(@to_io, last, bytes, F_MOVE) + end + @bytes_xfer += s + + # if we can't splice everything + # discard it so the early targets do not get repeated data + if s < bytes && most_teed > 0 + discard(bytes - s) + end + :wait_readable + rescue IOError, Errno::EPIPE => e # last failed, drop it + __dst_error(last, e) + targets.pop # we're no longer a valid target + + if most_teed == 0 + # nothing accomplished, watch any targets + return blocked if blocked[0] + else + # some progress, discard the data we could not splice + @bytes_xfer += most_teed + discard(most_teed) + end + + # stop decoding if we're completely errored out + # returning nil will trigger close + return targets[0] ? :wait_readable : nil + end + end +end diff --git a/lib/dtas/buffer/read_write.rb b/lib/dtas/buffer/read_write.rb index 04856c7..8fdb25d 100644 --- a/lib/dtas/buffer/read_write.rb +++ b/lib/dtas/buffer/read_write.rb @@ -1,13 +1,12 @@ -# Copyright (C) 2013-2019 all contributors <dtas-all@nongnu.org> +# Copyright (C) all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true require 'io/nonblock' require_relative '../../dtas' require_relative '../pipe' -require_relative '../nonblock' -# compatibility code for systems lacking "splice" support via the -# "sleepy_penguin" 3.5+ RubyGem. Used only by -player +# compatibility code for non-Linux systems lacking "splice" support. +# Used only by -player module DTAS::Buffer::ReadWrite # :nodoc: MAX_AT_ONCE = 512 # min PIPE_BUF value in POSIX attr_accessor :buffer_size diff --git a/lib/dtas/buffer/splice.rb b/lib/dtas/buffer/splice.rb index 281ecfd..b9957ce 100644 --- a/lib/dtas/buffer/splice.rb +++ b/lib/dtas/buffer/splice.rb @@ -1,10 +1,12 @@ -# Copyright (C) 2013-2019 all contributors <dtas-all@nongnu.org> +# Copyright (C) 2013-2020 all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true require 'io/nonblock' require 'sleepy_penguin' require_relative '../../dtas' require_relative '../pipe' +SleepyPenguin.respond_to?(:splice) or + raise LoadError, 'sleepy_penguin 3.5+ required for splice', [] # Used by -player on Linux systems with the "sleepy_penguin" RubyGem installed module DTAS::Buffer::Splice # :nodoc: @@ -12,7 +14,6 @@ module DTAS::Buffer::Splice # :nodoc: MAX_AT_ONCE_1 = 65536 F_MOVE = SleepyPenguin::F_MOVE F_NONBLOCK = SleepyPenguin::F_NONBLOCK - TRY = { exception: false }.freeze def buffer_size @to_io.pipe_size @@ -32,12 +33,14 @@ module DTAS::Buffer::Splice # :nodoc: def broadcast_one(targets, limit = nil) # single output is always non-blocking limit ||= MAX_AT_ONCE_1 - s = SleepyPenguin.splice(@to_io, targets[0], limit, F_MOVE|F_NONBLOCK, TRY) + s = SleepyPenguin.splice(@to_io, targets[0], limit, F_MOVE|F_NONBLOCK, + exception: false) if Symbol === s targets # our one and only target blocked on write else @bytes_xfer += s - :wait_readable # we want to read more from @to_io soon + # s < limit means targets[0] is full + s < limit ? targets : :wait_readable end rescue Errno::EPIPE, IOError => e __dst_error(targets[0], e) @@ -71,7 +74,8 @@ module DTAS::Buffer::Splice # :nodoc: targets.delete_if do |dst| begin t = (dst.nonblock? || most_teed == 0) ? - SleepyPenguin.tee(@to_io, dst, chunk_size, F_NONBLOCK, TRY) : + SleepyPenguin.tee(@to_io, dst, chunk_size, F_NONBLOCK, + exception: false) : __tee_in_full(@to_io, dst, chunk_size) if Integer === t if t > most_teed @@ -119,7 +123,8 @@ module DTAS::Buffer::Splice # :nodoc: begin targets << last if last.nonblock? || most_teed == 0 - s = SleepyPenguin.splice(@to_io, last, bytes, F_MOVE|F_NONBLOCK, TRY) + s = SleepyPenguin.splice(@to_io, last, bytes, F_MOVE|F_NONBLOCK, + exception: false) if Symbol === s blocked << last diff --git a/lib/dtas/command.rb b/lib/dtas/command.rb index 5ac1eb7..116f880 100644 --- a/lib/dtas/command.rb +++ b/lib/dtas/command.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) 2013-2020 all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true require_relative 'serialize' diff --git a/lib/dtas/compat_onenine.rb b/lib/dtas/compat_onenine.rb deleted file mode 100644 index 39cc1ec..0000000 --- a/lib/dtas/compat_onenine.rb +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> -# License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> - -# Make Ruby 1.9.3 look like Ruby 2.0.0 to us -# This exists for Debian wheezy users using the stock Ruby 1.9.3 install. -# We'll drop this interface when Debian wheezy (7.0) becomes unsupported. -class String # :nodoc: - def b # :nodoc: - dup.force_encoding(Encoding::BINARY) - end -end unless String.method_defined?(:b) - -def IO # :nodoc: - def self.pipe # :nodoc: - super.each { |io| io.close_on_exec = true } - end -end if RUBY_VERSION.to_f <= 1.9 diff --git a/lib/dtas/cue_index.rb b/lib/dtas/cue_index.rb index 74d4098..9ba9334 100644 --- a/lib/dtas/cue_index.rb +++ b/lib/dtas/cue_index.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) 2013-2020 all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true require_relative '../dtas' diff --git a/lib/dtas/disclaimer.rb b/lib/dtas/disclaimer.rb index b5e0a57..91cfb7d 100644 --- a/lib/dtas/disclaimer.rb +++ b/lib/dtas/disclaimer.rb @@ -1,5 +1,5 @@ # :enddoc: -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) 2013-2020 all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true DTAS_PROGNAME = File.basename($0) diff --git a/lib/dtas/edit_client.rb b/lib/dtas/edit_client.rb index 82cc857..2bdc4d8 100644 --- a/lib/dtas/edit_client.rb +++ b/lib/dtas/edit_client.rb @@ -1,8 +1,7 @@ -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true require 'tempfile' -require 'yaml' require_relative 'unix_client' require_relative 'disclaimer' @@ -14,7 +13,7 @@ module DTAS::EditClient # :nodoc: v.empty? and next return v end - 'vi'.freeze + 'vi' end def client_socket diff --git a/lib/dtas/encoding.rb b/lib/dtas/encoding.rb index 613e376..bbc6076 100644 --- a/lib/dtas/encoding.rb +++ b/lib/dtas/encoding.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2018-2019 all contributors <dtas-all@nongnu.org> +# Copyright (C) 2018-2020 all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true @@ -11,15 +11,14 @@ module DTAS::Encoding # :nodoc: private def try_enc_harder(str, enc, old) # :nodoc: + begin + require 'charlock_holmes' + @charlock_holmes = CharlockHolmes::EncodingDetector.new + rescue LoadError + @charlock_holmes = false + end if @charlock_holmes.nil? + case @charlock_holmes - when nil - begin - require 'charlock_holmes' - @charlock_holmes = CharlockHolmes::EncodingDetector.new - rescue LoadError - warn "`charlock_holmes` gem not available for encoding detection" - @charlock_holmes = false - end when false enc_fallback(str, enc, old) else diff --git a/lib/dtas/fadefx.rb b/lib/dtas/fadefx.rb index 1a00653..0ec108c 100644 --- a/lib/dtas/fadefx.rb +++ b/lib/dtas/fadefx.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true require_relative '../dtas' @@ -95,7 +95,7 @@ class DTAS::FadeFX # :nodoc: def parse!(str) return nil if str.empty? type = "t" - str.sub!(/\A([a-z])/, "") and type = DTAS.dedupe_str($1) + str.sub!(/\A([a-z])/, "") and type = -$1 F.new(type, parse_time(str)) end end diff --git a/lib/dtas/format.rb b/lib/dtas/format.rb index f4ea102..2c26517 100644 --- a/lib/dtas/format.rb +++ b/lib/dtas/format.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) 2013-2020 all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true require_relative '../dtas' diff --git a/lib/dtas/mcache.rb b/lib/dtas/mcache.rb index b638a23..e0a39af 100644 --- a/lib/dtas/mcache.rb +++ b/lib/dtas/mcache.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true # encoding: binary @@ -13,16 +13,27 @@ class DTAS::Mcache def lookup(infile) bucket = infile.hash & @mask + st = nil if cur = @tbl[bucket] if cur[:infile] == infile && (DTAS.now - cur[:btime]) < @ttl - return cur + begin + st = File.stat(infile) + return cur if cur[:ctime] == st.ctime + rescue + end end end return unless block_given? @tbl[bucket] = begin cur = cur ? cur.clear : {} + begin + st ||= File.stat(infile) + cur[:ctime] = st.ctime + rescue + return + end if ret = yield(infile, cur) - ret[:infile] = infile.frozen? ? infile : infile.dup.freeze + ret[:infile] = infile.frozen? ? infile : -(infile.dup) ret[:btime] = DTAS.now end ret diff --git a/lib/dtas/mlib.rb b/lib/dtas/mlib.rb index e0f19ab..f99ed6a 100644 --- a/lib/dtas/mlib.rb +++ b/lib/dtas/mlib.rb @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- -# Copyright (C) 2015-2019 all contributors <dtas-all@nongnu.org> +# Copyright (C) all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true # @@ -129,9 +129,13 @@ class DTAS::Mlib # :nodoc: comments.where(q).delete tmp.each do |tid, val| v = vals[val: val] - q[:val_id] = v ? v[:id] : vals.insert(val: val) - q[:tag_id] = tid - comments.insert(q) + begin + q[:val_id] = v ? v[:id] : vals.insert(val: val) + q[:tag_id] = tid + comments.insert(q) + rescue => e + warn "E: #{e.message} (#{e.class}) q=#{q.inspect} val=#{val.inspect}" + end end end end @@ -197,9 +201,7 @@ class DTAS::Mlib # :nodoc: tag_id = tag_map[x] and tag_map["#{x}number"] = tag_id end @tag_rmap = tag_map.invert.freeze - tag_map.merge!(Hash[*(tag_map.map { |k,v| - [DTAS.dedupe_str(k.upcase), v] - }.flatten!)]) + tag_map.merge!(Hash[*(tag_map.map { |k,v| [-(k.upcase), v] }.flatten!)]) @tag_map = tag_map.freeze end @@ -214,12 +216,16 @@ class DTAS::Mlib # :nodoc: end end + def maybe_blob(path) + path.valid_encoding? ? path : Sequel.blob(path) + end + def scan_file(path, st, parent_id) return if @suffixes !~ path || st.size == 0 # no-op if no change unless @force - if node = @db[:nodes][name: path, parent_id: parent_id] + if node = @db[:nodes][name: maybe_blob(path), parent_id: parent_id] return if st.ctime.to_i == node[:ctime] || node[:tlen] == DM_IGN end end @@ -271,14 +277,16 @@ class DTAS::Mlib # :nodoc: node_id = node.delete(:id) @db[:nodes].where(id: node_id).update(node.merge(q)) node[:id] = node_id + rescue => e + warn "E: #{e.message} (#{e.class}) node=#{node.inspect}" end def node_lookup(parent_id, name) - @db[:nodes][name: name, parent_id: parent_id] + @db[:nodes][name: maybe_blob(name), parent_id: parent_id] end def node_ensure(parent_id, name, tlen, ctime = nil) - q = { name: name, parent_id: parent_id } + q = { name: maybe_blob(name), parent_id: parent_id } if node = @db[:nodes][q] node_update_maybe(node, tlen, ctime) else @@ -289,6 +297,8 @@ class DTAS::Mlib # :nodoc: node[:id] = @db[:nodes].insert(node) end node + rescue => e + warn "E: #{e.message} (#{e.class}) q=#{q.inspect}" end def cd(path) @@ -409,7 +419,7 @@ class DTAS::Mlib # :nodoc: return '/' if base == '' # root_node parent_id = node[:parent_id] base += '/' unless node[:tlen] >= 0 - ppath = cache[parent_id] and return DTAS.dedupe_str("#{ppath}/#{base}") + ppath = cache[parent_id] and return -"#{ppath}/#{base}" parts = [] begin node = @db[:nodes][id: node[:parent_id]] @@ -417,9 +427,9 @@ class DTAS::Mlib # :nodoc: parts.unshift node[:name] end while true parts.unshift('') - cache[parent_id] = DTAS.dedupe_str(parts.join('/')) + cache[parent_id] = -(parts.join('/')) parts << base - DTAS.dedupe_str(parts.join('/')) + -(parts.join('/')) end def emit_recurse(node, cache, cb) diff --git a/lib/dtas/mlib/migrations/0001_initial.rb b/lib/dtas/mlib/migrations/0001_initial.rb index 688b6a5..d3da6a3 100644 --- a/lib/dtas/mlib/migrations/0001_initial.rb +++ b/lib/dtas/mlib/migrations/0001_initial.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) 2015-2020 all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> Sequel.migration do diff --git a/lib/dtas/nonblock.rb b/lib/dtas/nonblock.rb deleted file mode 100644 index 504c8d2..0000000 --- a/lib/dtas/nonblock.rb +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright (C) 2015-2016 all contributors <dtas-all@nongnu.org> -# License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> - -class DTAS::Nonblock < IO # :nodoc: - if RUBY_VERSION.to_f <= 2.0 - EX = {}.freeze - def read_nonblock(len, buf = nil, opts = EX) - super(len, buf) - rescue IO::WaitReadable - raise if opts[:exception] - :wait_readable - rescue EOFError - raise if opts[:exception] - nil - end - - def write_nonblock(buf, opts = EX) - super(buf) - rescue IO::WaitWritable - raise if opts[:exception] - :wait_writable - end - end -end diff --git a/lib/dtas/parse_freq.rb b/lib/dtas/parse_freq.rb index afc9048..201c284 100644 --- a/lib/dtas/parse_freq.rb +++ b/lib/dtas/parse_freq.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) 2015-2020 all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true require_relative '../dtas' diff --git a/lib/dtas/parse_time.rb b/lib/dtas/parse_time.rb index f156c5c..73f5b5c 100644 --- a/lib/dtas/parse_time.rb +++ b/lib/dtas/parse_time.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) 2013-2020 all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true require_relative '../dtas' diff --git a/lib/dtas/partstats.rb b/lib/dtas/partstats.rb index 1037c87..061ff50 100644 --- a/lib/dtas/partstats.rb +++ b/lib/dtas/partstats.rb @@ -1,9 +1,8 @@ # -*- encoding: binary -*- -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true require_relative '../dtas' -require_relative 'xs' require_relative 'process' require_relative 'sigevent' @@ -12,7 +11,6 @@ require_relative 'sigevent' class DTAS::PartStats # :nodoc: CMD = 'sox "$INFILE" -n $TRIMFX $SOXFX stats $STATSOPTS' include DTAS::Process - include DTAS::SpawnFix attr_reader :key_idx attr_reader :key_width @@ -56,7 +54,7 @@ class DTAS::PartStats # :nodoc: rd, wr = IO.pipe env = opts[:env] env = env ? env.dup : {} - env["INFILE"] = xs(@infile) + env["INFILE"] = @infile env["TRIMFX"] = "trim #{trim_part.tbeg}s #{trim_part.tlen}s" opts = { pgroup: true, close_others: true, err: wr } pid = spawn(env, CMD, opts) @@ -173,7 +171,7 @@ becomes: else next end - key = DTAS.dedupe_str($1) + key = -$1 key_idx = @key_idx[key] parts = line.split(/\s+/) nshift.times { parts.shift } # remove stuff we don't need diff --git a/lib/dtas/pipe.rb b/lib/dtas/pipe.rb index 58d926c..a7b02b0 100644 --- a/lib/dtas/pipe.rb +++ b/lib/dtas/pipe.rb @@ -1,19 +1,19 @@ -# Copyright (C) 2013-2019 all contributors <dtas-all@nongnu.org> +# Copyright (C) all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true -begin - require 'sleepy_penguin' -rescue LoadError -end require_relative '../dtas' require_relative 'writable_iter' -require_relative 'nonblock' # pipe wrapper for -player sinks -class DTAS::Pipe < DTAS::Nonblock # :nodoc: +class DTAS::Pipe < IO # :nodoc: include DTAS::WritableIter attr_accessor :sink + if RUBY_PLATFORM =~ /linux/i && File.readable?('/proc/sys/fs/pipe-max-size') + F_SETPIPE_SZ = 1031 + F_GETPIPE_SZ = 1032 + end + def self.new _, w = rv = pipe w.writable_iter_init @@ -21,13 +21,16 @@ class DTAS::Pipe < DTAS::Nonblock # :nodoc: end def pipe_size=(nr) - defined?(SleepyPenguin::F_SETPIPE_SZ) and - fcntl(SleepyPenguin::F_SETPIPE_SZ, nr) + fcntl(F_SETPIPE_SZ, nr) if defined?(F_SETPIPE_SZ) + rescue Errno::EINVAL # old kernel + rescue Errno::EPERM + # resizes fail if Linux is close to the pipe limit for the user + # or if the user does not have permissions to resize end def pipe_size - fcntl(SleepyPenguin::F_GETPIPE_SZ) - end if defined?(SleepyPenguin::F_GETPIPE_SZ) + fcntl(F_GETPIPE_SZ) + end if defined?(F_GETPIPE_SZ) # avoid syscall, we never change IO#nonblock= directly def nonblock? diff --git a/lib/dtas/pipeline.rb b/lib/dtas/pipeline.rb index b900fee..1bebe87 100644 --- a/lib/dtas/pipeline.rb +++ b/lib/dtas/pipeline.rb @@ -1,12 +1,9 @@ -# Copyright (C) 2017-2019 all contributors <dtas-all@nongnu.org> +# Copyright (C) all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true require_relative '../dtas' -require_relative 'spawn_fix' module DTAS::Pipeline # :nodoc: - include DTAS::SpawnFix - # Process.spawn wrapper which supports running Proc-like objects in # a separate process, not just external commands. # Returns the pid of the spawned process diff --git a/lib/dtas/player.rb b/lib/dtas/player.rb index 37f2c96..6ea3aba 100644 --- a/lib/dtas/player.rb +++ b/lib/dtas/player.rb @@ -1,8 +1,8 @@ -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true -require 'yaml' require 'shellwords' +require 'yaml' require_relative '../dtas' require_relative 'xs' require_relative 'source' @@ -37,6 +37,7 @@ class DTAS::Player # :nodoc: @paused = false @format = DTAS::Format.new @bypass = [] # %w(rate bits channels) (not worth Hash overhead) + @bypass_next = nil # source_spec @sinks = {} # { user-defined name => sink } @targets = [] # order matters @@ -123,10 +124,6 @@ class DTAS::Player # :nodoc: rv end - def to_omap(hash) - YAML::Omap === hash ? hash : YAML::Omap.new.merge!(hash) - end - def self.load(hash) rv = new rv.instance_eval do @@ -157,7 +154,6 @@ class DTAS::Player # :nodoc: @source_map.each do |name, src| src_hsh = v[name] or next src.load!(src_hsh) - src.env = to_omap(src.env) end source_map_reload end @@ -168,9 +164,8 @@ class DTAS::Player # :nodoc: if sinks = hash["sinks"] sinks.each do |sink_hsh| - sink_hsh['name'] = DTAS.dedupe_str(sink_hsh['name']) + sink_hsh['name'] = -sink_hsh['name'] sink = DTAS::Sink.load(sink_hsh) - sink.env = to_omap(sink.env) @sinks[sink.name] = sink end end @@ -208,13 +203,13 @@ class DTAS::Player # :nodoc: command = msg.shift case command when "enq" - enq_handler(io, msg[0]) + enq_handler(io, -msg[0]) when "enq-cmd" - enq_handler(io, { "command" => msg[0]}) + enq_handler(io, { "command" => -msg[0]}) when "pause", "play", "play_pause" play_pause_handler(io, command) when "pwd" - io.emit(Dir.pwd) + io.emit(-Dir.pwd) else m = "dpc_#{command.tr('-', '_')}" __send__(m, io, msg) if respond_to?(m) @@ -282,7 +277,7 @@ class DTAS::Player # :nodoc: if deleted[0] warn("#{sink.name} died unexpectedly: #{status.inspect}") deleted.each { |t| drop_target(t) } - __current_drop unless @targets[0] + do_pause unless @targets[0] return # sink stays dead if it died unexpectedly end @@ -337,6 +332,7 @@ class DTAS::Player # :nodoc: # called when the player is leaving idle state def spawn_sinks(source_spec) + @bypass_next = nil return true if @targets[0] @sinks.each_value do |sink| sink.active or next @@ -398,6 +394,8 @@ class DTAS::Player # :nodoc: if ! @bypass.empty? && pending.respond_to?(:format) new_fmt = bypass_match!(@format.dup, pending.format) if new_fmt != @format + @bypass_next = source_spec + return if @sink_buf.inflight > 0 stop_sinks # we may fail to start below format_update!(new_fmt) end @@ -440,6 +438,7 @@ class DTAS::Player # :nodoc: end def stop_sinks + @bypass_next = nil @targets.each { |t| drop_target(t) }.clear end @@ -451,7 +450,7 @@ class DTAS::Player # :nodoc: # pull data from sink_buf into @targets, source feeds into sink_buf def sink_iter wait_iter = broadcast_iter(@sink_buf, @targets) - __current_drop if nil == wait_iter # sink error, stop source + do_pause if nil == wait_iter # sink error, stop source return wait_iter if @current # no source left to feed sink_buf, drain the remaining data @@ -464,7 +463,9 @@ class DTAS::Player # :nodoc: end # nothing left inflight, stop the sinks until we have a source + bn = @bypass_next stop_sinks + next_source(bn) if bn # are we restarting for bypass? :ignore end diff --git a/lib/dtas/player/client_handler.rb b/lib/dtas/player/client_handler.rb index 1e4ac96..3c5fe5d 100644 --- a/lib/dtas/player/client_handler.rb +++ b/lib/dtas/player/client_handler.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true require_relative '../xs' @@ -135,7 +135,7 @@ module DTAS::Player::ClientHandler # :nodoc: # or variable names. sink.valid_name?(name) or return io.emit("ERR sink name invalid") - sink.name = DTAS.dedupe_str(name) + sink.name = -name active_before = sink.active before = __sink_snapshot(sink) @@ -144,7 +144,7 @@ module DTAS::Player::ClientHandler # :nodoc: k, v = kv.split('=', 2) case k when %r{\Aenv\.([^=]+)\z} - sink.env[DTAS.dedupe_str($1)] = v + sink.env[$1] = v when %r{\Aenv#([^=]+)\z} v == nil or return io.emit("ERR unset env has no value") sink.env.delete($1) @@ -197,19 +197,20 @@ module DTAS::Player::ClientHandler # :nodoc: end end + def __offset_to_i(offset, src) + # either "999s" for 999 samples or HH:MM:SS for time + offset.sub!(/s\z/, '') ? offset.to_i : src.format.hhmmss_to_samples(offset) + end + def __offset_to_samples(offset) - offset.sub!(/s\z/, '') and return offset.to_i - @current.format.hhmmss_to_samples(offset) + __offset_to_i(offset, @current) end # returns seek offset as an Integer in sample count - def __seek_offset_adj(dir, offset) - if offset.sub!(/s\z/, '') - offset = offset.to_i - else # time - offset = @current.format.hhmmss_to_samples(offset) - end - n = __current_decoded_samples + (dir * offset) + def __seek_offset_adj(dir, offset, + src = @current, + current_decoded_samples = __current_decoded_samples) + n = current_decoded_samples + (dir * __offset_to_i(offset, src)) n = 0 if n < 0 "#{n}s" end @@ -391,15 +392,17 @@ module DTAS::Player::ClientHandler # :nodoc: end end + def __offset_direction(offset) + offset.sub!(/\A\+/, '') ? 1 : (offset.sub!(/\A-/, '') ? -1 : nil) + end + def dpc_seek(io, msg) offset = msg[0] or return io.emit('ERR usage: seek OFFSET') if @current if @current.respond_to?(:infile) begin - if offset.sub!(/\A\+/, '') - offset = __seek_offset_adj(1, offset) - elsif offset.sub!(/\A-/, '') - offset = __seek_offset_adj(-1, offset) + if direction = __offset_direction(offset) + offset = __seek_offset_adj(direction, offset) # else: pass to sox directly end rescue ArgumentError @@ -413,7 +416,12 @@ module DTAS::Player::ClientHandler # :nodoc: case file = @queue[0] when String @queue[0] = [ file, offset ] - when Array + when Array # offset already stored, adjust + if direction = __offset_direction(offset) + tmp = try_file(*file) + cur_off = __offset_to_i(file[1].dup, tmp) + offset = __seek_offset_adj(direction, offset, tmp, cur_off) + end file[1] = offset else return io.emit("ERR unseekable") @@ -556,7 +564,7 @@ module DTAS::Player::ClientHandler # :nodoc: rescue => e res = "ERR dumping to #{xs(sf.path)} #{e.message}" end - io.to_io.send(res, Socket::MSG_EOR) + io.to_io.send(res, 0) ensure exit!(0) end diff --git a/lib/dtas/process.rb b/lib/dtas/process.rb index 4caf96b..02bf77e 100644 --- a/lib/dtas/process.rb +++ b/lib/dtas/process.rb @@ -1,17 +1,15 @@ -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true require 'io/wait' require 'shellwords' require_relative '../dtas' require_relative 'xs' -require_relative 'nonblock' # process management helpers module DTAS::Process # :nodoc: PIDS = {} include DTAS::XS - include DTAS::SpawnFix def self.reaper begin @@ -89,12 +87,12 @@ module DTAS::Process # :nodoc: env = {} end buf = ''.b - r, w = DTAS::Nonblock.pipe + r, w = IO.pipe opts = opts.merge(out: w) r.binmode no_raise = opts.delete(:no_raise) if err_str = opts.delete(:err_str) - re, we = DTAS::Nonblock.pipe + re, we = IO.pipe re.binmode opts[:err] = we end diff --git a/lib/dtas/replaygain.rb b/lib/dtas/replaygain.rb index 116625e..5fa6dcf 100644 --- a/lib/dtas/replaygain.rb +++ b/lib/dtas/replaygain.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) 2013-2020 all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true diff --git a/lib/dtas/rg_state.rb b/lib/dtas/rg_state.rb index 6b2c718..9a44835 100644 --- a/lib/dtas/rg_state.rb +++ b/lib/dtas/rg_state.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true # @@ -72,7 +72,7 @@ class DTAS::RGState # :nodoc: when 1 then return 'gain 192' else val.abs <= 0.00000001 and return - DTAS.dedupe_str(sprintf('gain %0.8f', val)) + -sprintf('gain %0.8f', val) end end diff --git a/lib/dtas/serialize.rb b/lib/dtas/serialize.rb index fbed5af..d8331e6 100644 --- a/lib/dtas/serialize.rb +++ b/lib/dtas/serialize.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) 2013-2020 all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true diff --git a/lib/dtas/sigevent.rb b/lib/dtas/sigevent.rb index d4a96d7..16edacb 100644 --- a/lib/dtas/sigevent.rb +++ b/lib/dtas/sigevent.rb @@ -1,10 +1,13 @@ -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) 2013-2020 all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true begin raise LoadError, "no eventfd with _DTAS_POSIX" if ENV["_DTAS_POSIX"] - require 'sleepy_penguin' - require_relative 'sigevent/efd' + begin + require_relative 'sigevent/efd' + rescue LoadError + require_relative 'sigevent/fiddle_efd' + end rescue LoadError require_relative 'sigevent/pipe' end diff --git a/lib/dtas/sigevent/efd.rb b/lib/dtas/sigevent/efd.rb index 4be2c84..d13c32a 100644 --- a/lib/dtas/sigevent/efd.rb +++ b/lib/dtas/sigevent/efd.rb @@ -1,8 +1,10 @@ -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) 2013-2020 all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # used in various places for safe wakeups from IO.select via signals # This requires a modern Linux system and the "sleepy_penguin" RubyGem +require 'sleepy_penguin' + class DTAS::Sigevent < SleepyPenguin::EventFD # :nodoc: def self.new super(0, :CLOEXEC) diff --git a/lib/dtas/sigevent/fiddle_efd.rb b/lib/dtas/sigevent/fiddle_efd.rb new file mode 100644 index 0000000..8bfa332 --- /dev/null +++ b/lib/dtas/sigevent/fiddle_efd.rb @@ -0,0 +1,37 @@ +# Copyright (C) all contributors <dtas-all@nongnu.org> +# License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> +# frozen_string_literal: true + +# used in various places for safe wakeups from IO.select via signals +# This requires a modern GNU/Linux system with eventfd(2) support +require 'fiddle' +class DTAS::Sigevent # :nodoc: + + EventFD = Fiddle::Function.new(DTAS.libc['eventfd'], + [ Fiddle::TYPE_INT, Fiddle::TYPE_INT ], # initval, flags + Fiddle::TYPE_INT) # fd + + attr_reader :to_io + ONE = -([ 1 ].pack('Q')) + + def initialize + fd = EventFD.call(0, 02000000|00004000) # EFD_CLOEXEC|EFD_NONBLOCK + raise "eventfd failed: #{Fiddle.last_error}" if fd < 0 + @to_io = IO.for_fd(fd) + @buf = ''.b + end + + def signal + @to_io.syswrite(ONE) + end + + def readable_iter + @to_io.read_nonblock(8, @buf, exception: false) + yield self, nil # calls DTAS::Process.reaper + :wait_readable + end + + def close + @to_io.close + end +end diff --git a/lib/dtas/sigevent/pipe.rb b/lib/dtas/sigevent/pipe.rb index 921a5b3..6c3b83c 100644 --- a/lib/dtas/sigevent/pipe.rb +++ b/lib/dtas/sigevent/pipe.rb @@ -1,15 +1,14 @@ -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true # used in various places for safe wakeups from IO.select via signals -# A fallback for non-Linux systems lacking the "sleepy_penguin" RubyGem -require_relative '../nonblock' +# A fallback for non-Linux systems lacking the "splice" syscall class DTAS::Sigevent # :nodoc: attr_reader :to_io def initialize - @to_io, @wr = DTAS::Nonblock.pipe + @to_io, @wr = IO.pipe @rbuf = ''.b end diff --git a/lib/dtas/sink.rb b/lib/dtas/sink.rb index c481032..966bab4 100644 --- a/lib/dtas/sink.rb +++ b/lib/dtas/sink.rb @@ -1,7 +1,6 @@ -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true -require 'yaml' require_relative '../dtas' require_relative 'pipe' require_relative 'process' diff --git a/lib/dtas/source.rb b/lib/dtas/source.rb index 1944894..e3ca17e 100644 --- a/lib/dtas/source.rb +++ b/lib/dtas/source.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) 2013-2020 all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true require_relative '../dtas' diff --git a/lib/dtas/source/av.rb b/lib/dtas/source/av.rb index 0a9d39b..dcebcfd 100644 --- a/lib/dtas/source/av.rb +++ b/lib/dtas/source/av.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true require_relative '../../dtas' @@ -13,13 +13,12 @@ class DTAS::Source::Av # :nodoc: 'avconv -v error $SSPOS $PROBE -i "$INFILE" $AMAP -f sox - |' \ 'sox -p $SOXFMT - $TRIMFX $RGFX', - # this is above ffmpeg because this av is the Debian default and - # it's easier for me to test av than ff - "tryorder" => 1, + "tryorder" => 2, ) def initialize command_init(AV_DEFAULTS) + @mcache = nil @av_ff_probe = "avprobe" end diff --git a/lib/dtas/source/av_ff_common.rb b/lib/dtas/source/av_ff_common.rb index ae654ba..7f197e0 100644 --- a/lib/dtas/source/av_ff_common.rb +++ b/lib/dtas/source/av_ff_common.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true require_relative '../../dtas' @@ -7,10 +7,10 @@ require_relative '../replaygain' require_relative '../xs' require_relative 'file' -# Common code for libav (avconv/avprobe) and ffmpeg (and ffprobe) -# TODO: newer versions of both *probes support JSON, which will be easier to -# parse. However, the packaged libav version in Debian 7.0 does not -# support JSON, so we have an ugly parser... +# Common code for ffmpeg/ffprobe and the abandoned libav (avconv/avprobe). +# TODO: newer versions of both *probes support JSON, which will be easier +# to parse. libav is abandoned, nowadays, and Debian only packages +# ffmpeg+ffprobe nowadays. module DTAS::Source::AvFfCommon # :nodoc: include DTAS::Source::File include DTAS::XS @@ -21,10 +21,23 @@ module DTAS::Source::AvFfCommon # :nodoc: attr_reader :format attr_reader :duration + CACHE_KEYS = [ :@duration, :@probe_harder, :@comments, :@astreams, + :@format ].freeze + + def mcache_lookup(infile) + (@mcache ||= DTAS::Mcache.new).lookup(infile) do |input, dst| + tmp = source_file_dup(infile, nil, nil) + tmp.av_ff_ok? or return nil + CACHE_KEYS.each { |k| dst[k] = tmp.instance_variable_get(k) } + dst + end + end + def try(infile, offset = nil, trim = nil) - rv = source_file_dup(infile, offset, trim) - rv.av_ff_ok? or return - rv + ent = mcache_lookup(infile) or return + ret = source_file_dup(infile, offset, trim) + CACHE_KEYS.each { |k| ret.instance_variable_set(k, ent[k]) } + ret end def __parse_astream(cmd, stream) @@ -79,7 +92,7 @@ module DTAS::Source::AvFfCommon # :nodoc: err = "".b begin - s = qx(@env, cmd, err_str: err, no_raise: true) + s = qx(@env, cmd, err_str: err, no_raise: true, rlimit_cpu: [ 1, 2 ]) rescue Errno::ENOENT # avprobe/ffprobe not installed return false end @@ -104,13 +117,14 @@ module DTAS::Source::AvFfCommon # :nodoc: prev_cmd = cmd end while incomplete.compact[0] + enc = Encoding.default_external # typically Encoding::UTF_8 # old avprobe s.scan(%r{^\[FORMAT\]\n(.*?)\n\[/FORMAT\]\n}m) do |_| f = $1.dup f =~ /^duration=([\d\.]+)\s*$/nm and @duration = $1.to_f # TODO: multi-line/multi-value/repeated tags f.gsub!(/^TAG:([^=]+)=(.*)$/ni) { |_| - @comments[DTAS.dedupe_str($1.upcase)] = DTAS.dedupe_str($2) + @comments[-DTAS.try_enc($1.upcase, enc)] = $2 } end @@ -118,13 +132,22 @@ module DTAS::Source::AvFfCommon # :nodoc: s.scan(%r{^\[format\.tags\]\n(.*?)\n\n}m) do |_| f = $1.dup f.gsub!(/^([^=]+)=(.*)$/ni) { |_| - @comments[DTAS.dedupe_str($1.upcase)] = DTAS.dedupe_str($2) + @comments[-DTAS.try_enc($1.upcase, enc)] = $2 } end s.scan(%r{^\[format\]\n(.*?)\n\n}m) do |_| f = $1.dup f =~ /^duration=([\d\.]+)\s*$/nm and @duration = $1.to_f end + comments.each do |k,v| + v.chomp! + comments[k] = -DTAS.try_enc(v, enc) + end + + # ffprobe always uses "track", favor FLAC convention "TRACKNUMBER": + if @comments['TRACK'] && !@comments['TRACKNUMBER'] + @comments['TRACKNUMBER'] = @comments.delete('TRACK') + end ! @astreams.compact.empty? end @@ -186,7 +209,7 @@ module DTAS::Source::AvFfCommon # :nodoc: e["PROBE"] = @probe_harder ? @probe_harder.join(' ') : nil # make sure these are visible to the source command... - e["INFILE"] = xs(@infile) + e["INFILE"] = @infile e["AMAP"] = amap e["SSPOS"] = sspos e["RGFX"] = rg_state.effect(self) || nil diff --git a/lib/dtas/source/cmd.rb b/lib/dtas/source/cmd.rb index cdcd3b3..435ac07 100644 --- a/lib/dtas/source/cmd.rb +++ b/lib/dtas/source/cmd.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) 2013-2020 all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true require_relative '../../dtas' diff --git a/lib/dtas/source/common.rb b/lib/dtas/source/common.rb index a6d1a60..bdcb16d 100644 --- a/lib/dtas/source/common.rb +++ b/lib/dtas/source/common.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) 2013-2020 all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> module DTAS::Source::Common # :nodoc: attr_reader :dst_zero_byte # first byte this source object saw diff --git a/lib/dtas/source/ff.rb b/lib/dtas/source/ff.rb index 80436c7..c337b42 100644 --- a/lib/dtas/source/ff.rb +++ b/lib/dtas/source/ff.rb @@ -1,12 +1,10 @@ -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true require_relative '../../dtas' require_relative 'av_ff_common' # ffmpeg support -# note: only tested with the compatibility wrapper in the Debian 7.0 package -# (so still using avconv/avprobe) class DTAS::Source::Ff # :nodoc: include DTAS::Source::AvFfCommon @@ -15,12 +13,12 @@ class DTAS::Source::Ff # :nodoc: 'ffmpeg -v error $SSPOS $PROBE -i "$INFILE" $AMAP -f sox - |' \ 'sox -p $SOXFMT - $TRIMFX $RGFX', - # I haven't tested this much since av is in Debian stable and ff is not - "tryorder" => 2, + "tryorder" => 1, ) def initialize command_init(FF_DEFAULTS) + @mcache = nil @av_ff_probe = "ffprobe" end diff --git a/lib/dtas/source/file.rb b/lib/dtas/source/file.rb index 01ac998..88beb39 100644 --- a/lib/dtas/source/file.rb +++ b/lib/dtas/source/file.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) 2013-2020 all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true require_relative '../../dtas' diff --git a/lib/dtas/source/mp3gain.rb b/lib/dtas/source/mp3gain.rb index 3a7569d..7688822 100644 --- a/lib/dtas/source/mp3gain.rb +++ b/lib/dtas/source/mp3gain.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) 2013-2020 all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true require_relative '../process' diff --git a/lib/dtas/source/sox.rb b/lib/dtas/source/sox.rb index dc23c27..365c7b6 100644 --- a/lib/dtas/source/sox.rb +++ b/lib/dtas/source/sox.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2019 all contributors <dtas-all@nongnu.org> +# Copyright (C) all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true # encoding: binary @@ -24,7 +24,7 @@ class DTAS::Source::Sox # :nodoc: return if @last_failed == infile @last_failed = infile case msg - when Process::Status then msg = "failed with #{msg.exitstatus}" + when Process::Status then msg = "failed with #{msg.inspect}" when 0 then msg = 'detected zero samples' end warn("soxi #{infile}: #{msg}\n") @@ -39,7 +39,8 @@ class DTAS::Source::Sox # :nodoc: def mcache_lookup(infile) (@mcache ||= DTAS::Mcache.new).lookup(infile) do |input, dst| err = ''.b - out = qx(@env.dup, %W(soxi #{input}), err_str: err, no_raise: true) + out = qx(@env.dup, %W(soxi #{input}), err_str: err, no_raise: true, + rlimit_cpu: [ 1, 2 ]) return soxi_failed(infile, out) if Process::Status === out return soxi_failed(infile, err) if err =~ /soxi FAIL formats:/ out =~ /^Duration\s*:[^=]*= (\d+) samples /n @@ -56,14 +57,14 @@ class DTAS::Source::Sox # :nodoc: key = nil $1.split(/\n/n).each do |line| if line.sub!(/^([^=]+)=/ni, '') - key = DTAS.dedupe_str(DTAS.try_enc($1.upcase, enc)) + key = DTAS.try_enc($1.upcase, enc) end (comments[key] ||= ''.b) << "#{line}\n" unless line.empty? end comments.each do |k,v| v.chomp! DTAS.try_enc(v, enc) - comments[k] = DTAS.dedupe_str(v) + comments[k] = -v end end dst @@ -113,7 +114,7 @@ class DTAS::Source::Sox # :nodoc: def src_spawn(player_format, rg_state, opts) raise "BUG: #{self.inspect}#src_spawn called twice" if @to_io e = @env.merge!(player_format.to_env) - e["INFILE"] = xs(@infile) + e["INFILE"] = @infile # make sure these are visible to the "current" command... e["TRIMFX"] = trimfx diff --git a/lib/dtas/source/splitfx.rb b/lib/dtas/source/splitfx.rb index f746bee..2268404 100644 --- a/lib/dtas/source/splitfx.rb +++ b/lib/dtas/source/splitfx.rb @@ -1,7 +1,6 @@ -# Copyright (C) 2014-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true -require 'yaml' require_relative 'sox' require_relative '../splitfx' require_relative '../watchable' @@ -36,7 +35,7 @@ class DTAS::Source::SplitFX < DTAS::Source::Sox # :nodoc: sfx = DTAS::SplitFX.new Dir.chdir(File.dirname(ymlfile)) do # ugh - @ymlhash = YAML.load(buf) + @ymlhash = DTAS.yaml_load(buf) @ymlhash['tracks'] ||= [ "t 0 default" ] sfx.import(@ymlhash) sfx.infile.replace(File.expand_path(sfx.infile)) diff --git a/lib/dtas/spawn_fix.rb b/lib/dtas/spawn_fix.rb deleted file mode 100644 index a510a9e..0000000 --- a/lib/dtas/spawn_fix.rb +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> -# License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> -# workaround for older Rubies: https://bugs.ruby-lang.org/issues/8770 -module DTAS::SpawnFix # :nodoc: - def spawn(*args) - super(*args) - rescue Errno::EINTR - retry - end if RUBY_VERSION.to_f <= 2.1 -end diff --git a/lib/dtas/splitfx.rb b/lib/dtas/splitfx.rb index c0c7ac9..b94f54b 100644 --- a/lib/dtas/splitfx.rb +++ b/lib/dtas/splitfx.rb @@ -1,19 +1,18 @@ -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true require_relative '../dtas' require_relative 'format' require_relative 'process' -require_relative 'xs' require 'tempfile' # The backend for dtas-splitfx(1) command, but also supported by dtas-player # Unlike the stuff for dtas-player, dtas-splitfx is fairly tied to sox # (but we may still pipe to ecasound or anything else) class DTAS::SplitFX # :nodoc: - CMD = 'sox "$INFILE" $COMMENTS $OUTFMT $OUTDST $TRIMFX $FX $RATEFX $DITHERFX' + CMD = 'sox "$INFILE" $COMMENTS $OUTFMT $OUTDST $TRIMFX $FX' \ + ' $RATEFX $DITHERFX $STATS' include DTAS::Process - include DTAS::XS attr_reader :infile, :env, :command # for --trim on the command-line @@ -67,6 +66,7 @@ class DTAS::SplitFX # :nodoc: # $CHANNELS (input) # $BITS_PER_SAMPLE (input) def initialize + @tshift = 0 @env = {} @comments = {} @track_start = 1 @@ -115,7 +115,7 @@ class DTAS::SplitFX # :nodoc: end case v = hash["track_zpad"] - when Integer then @track_zpad = val + when Integer then @track_zpad = v else _bool(hash, "track_zpad") { |val| @track_zpad = val } end @@ -206,12 +206,13 @@ class DTAS::SplitFX # :nodoc: elsif outfmt.bits && outfmt.bits <= 16 env["DITHERFX"] = "dither -s" end + env['STATS'] = 'stats' if opts[:stats] comments = Tempfile.new(%W(dtas-splitfx-#{t.comments["TRACKNUMBER"]} .txt)) - comments.sync = true t.comments.each do |k,v| env[k] = v.to_s comments.puts("#{k}=#{v}") end + comments.flush env["COMMENTS"] = "--comment-file=#{comments.path}" infile_env(env, @infile) outarg = outfmt.to_sox_arg @@ -250,7 +251,10 @@ class DTAS::SplitFX # :nodoc: command = 'true' if opts[:dryrun] # still gotta fork # pgroup: false so Ctrl-C on command-line will immediately stop everything - [ dtas_spawn(env, command, pgroup: false), comments ] + o = { pgroup: false } + e = opts[:err_suffix] and + o[:err] = [ "#{env['OUTDIR']}#{env['TRACKNUMBER']}#{e}", 'a' ] + [ dtas_spawn(env, command, o), comments ] end def load_tracks!(hash) @@ -287,8 +291,9 @@ class DTAS::SplitFX # :nodoc: start_time = argv.shift title = argv.shift t = T.new - t.tbeg = @t2s.call(start_time) + t.tbeg = @t2s.call(start_time) + @tshift t.comments = @comments.dup + title.valid_encoding? or warn "#{title.inspect} encoding invalid" t.comments["TITLE"] = title t.env = @env.dup @@ -298,6 +303,7 @@ class DTAS::SplitFX # :nodoc: t.fade_in = $1.split(/\s+/) when %r{\Afade_out=(.+)\z} # $1 = "t 4" or just "4" t.fade_out = $1.split(/\s+/) + when %r{\Aenv\.([^=]+)=(.+)\z} then t.env[$1] = -$2 when %r{\A\.(\w+)=(.+)\z} then t.comments[$1] = $2 else raise ArgumentError, "unrecognized arg(s): #{xs(argv)}" @@ -306,11 +312,24 @@ class DTAS::SplitFX # :nodoc: prev = @tracks.last and prev.commit(t.tbeg) @tracks << t + when 'tshift' + tshift = argv.shift + argv.empty? or raise ArgumentError, 'tshift does not take extra args' + if tshift.sub!(/\A-=/, '') + @tshift = @tshift - @t2s.call(tshift) + elsif tshift.sub!(/\A\+=/, '') + @tshift = @tshift + @t2s.call(tshift) + elsif tshift.sub!(/\A-/, '') + @tshift = -@t2s.call(tshift) + else + tshift.sub!(/\A\+/, '') + @tshift = @t2s.call(tshift) + end when "skip" stop_time = argv.shift argv.empty? or raise ArgumentError, "skip does not take extra args" s = Skip.new - s.tbeg = @t2s.call(stop_time) + s.tbeg = @t2s.call(stop_time) + @tshift # s.comments = {} # s.env = {} prev = @tracks.last or raise ArgumentError, "no tracks to skip" @@ -319,7 +338,7 @@ class DTAS::SplitFX # :nodoc: when "stop" stop_time = argv.shift argv.empty? or raise ArgumentError, "stop does not take extra args" - samples = @t2s.call(stop_time) + samples = @t2s.call(stop_time) + @tshift prev = @tracks.last and prev.commit(samples) else raise ArgumentError, "unknown command: #{xs(cmd)}" @@ -355,9 +374,19 @@ class DTAS::SplitFX # :nodoc: @rate = opts[:rate] @bits = opts[:bits] trim = opts[:trim] and @tracks = [ UTrim.new(trim, @env, @comments) ] - + if trim && opts[:filter] + raise ArgumentError, 'trim and filter are mutually exclusive' + end fails = [] tracks = @tracks.dup + (opts[:filter] || []).each do |re| + field, val = re.split(/=/, 2) + if val + tracks.delete_if { |t| (t.comments[field] || '') !~ /#{val}/ } + else + tracks.delete_if { |t| t.comments.values.grep(/#{re}/).empty? } + end + end pids = {} jobs = opts[:jobs] || tracks.size # jobs == nil => everything at once if opts[:sox_pipe] @@ -383,7 +412,7 @@ class DTAS::SplitFX # :nodoc: @out.puts "DONE #{done[0].inspect}" if $DEBUG done[1].close! else - fails << [ t, status ] + fails << [ done[0], status ] end end @@ -401,10 +430,10 @@ class DTAS::SplitFX # :nodoc: end def infile_env(env, infile) - env["INFILE"] = xs(infile) + env["INFILE"] = infile dir, base = File.split(File.expand_path(infile)) - env["INDIR"] = xs(dir) - env["INBASE"] = xs(base) + env["INDIR"] = dir + env["INBASE"] = base end def expand_cmd(env, command) # for display purposes only diff --git a/lib/dtas/state_file.rb b/lib/dtas/state_file.rb index b577850..f16a866 100644 --- a/lib/dtas/state_file.rb +++ b/lib/dtas/state_file.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true require 'yaml' @@ -14,7 +14,7 @@ class DTAS::StateFile # :nodoc: end def tryload - YAML.load(IO.binread(@path)) if File.readable?(@path) + DTAS.yaml_load(IO.binread(@path)) if File.readable?(@path) end def dump(obj, force_fsync = false) diff --git a/lib/dtas/tfx.rb b/lib/dtas/tfx.rb index 2ccdcf1..80051e8 100644 --- a/lib/dtas/tfx.rb +++ b/lib/dtas/tfx.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) 2013-2020 all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true require_relative '../dtas' diff --git a/lib/dtas/track.rb b/lib/dtas/track.rb index 35a00c1..3f4b813 100644 --- a/lib/dtas/track.rb +++ b/lib/dtas/track.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) 2015-2020 all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true require_relative '../dtas' @@ -9,6 +9,6 @@ class DTAS::Track # :nodoc: def initialize(track_id, path) @track_id = track_id - @to_path = path + @to_path = -path end end diff --git a/lib/dtas/tracklist.rb b/lib/dtas/tracklist.rb index e6a8fb6..a7f4c15 100644 --- a/lib/dtas/tracklist.rb +++ b/lib/dtas/tracklist.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) 2013-2020 all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true require_relative '../dtas' diff --git a/lib/dtas/unix_accepted.rb b/lib/dtas/unix_accepted.rb index 4a01972..63d3ce0 100644 --- a/lib/dtas/unix_accepted.rb +++ b/lib/dtas/unix_accepted.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true require 'socket' @@ -10,23 +10,22 @@ class DTAS::UNIXAccepted # :nodoc: def initialize(sock) @to_io = sock - @send_buf = [] + @sbuf = [] end # public API (for DTAS::Player) # returns :wait_readable on success def emit(msg) - buffered = @send_buf.size - if buffered == 0 - case rv = sendmsg_nonblock(msg) + if @sbuf.empty? + case rv = @to_io.sendmsg_nonblock(msg, 0, exception: false) when :wait_writable - @send_buf << msg + @sbuf << msg rv else :wait_readable end - else # buffered > 0 - @send_buf << msg + else + @sbuf << msg :wait_writable end rescue => e @@ -35,11 +34,11 @@ class DTAS::UNIXAccepted # :nodoc: # flushes pending data if it got buffered def writable_iter - case sendmsg_nonblock(@send_buf[0]) + case @to_io.sendmsg_nonblock(@sbuf[0], 0, exception: false) when :wait_writable then return :wait_writable else - @send_buf.shift - @send_buf.empty? ? :wait_readable : :wait_writable + @sbuf.shift + @sbuf.empty? ? :wait_readable : :wait_writable end rescue => e e @@ -51,13 +50,13 @@ class DTAS::UNIXAccepted # :nodoc: # EOF, assume no spurious wakeups for SOCK_SEQPACKET return nil if nread == 0 - case msg = recv_nonblock(nread) + case msg = @to_io.recv_nonblock(nread, exception: false) when :wait_readable then return msg when '', nil then return nil # EOF else yield(self, msg) # DTAS::Player deals with this end - @send_buf.empty? ? :wait_readable : :wait_writable + @sbuf.empty? ? :wait_readable : :wait_writable rescue SystemCallError nil end @@ -69,28 +68,4 @@ class DTAS::UNIXAccepted # :nodoc: def closed? @to_io.closed? end - - if RUBY_VERSION.to_f >= 2.3 - def sendmsg_nonblock(msg) - @to_io.sendmsg_nonblock(msg, Socket::MSG_EOR, exception: false) - end - - def recv_nonblock(len) - @to_io.recv_nonblock(len, exception: false) - end - else - def sendmsg_nonblock(msg) - @to_io.sendmsg_nonblock(msg, Socket::MSG_EOR) - rescue IO::WaitWritable - :wait_writable - end - - def recv_nonblock(len) - @to_io.recv_nonblock(len) - rescue IO::WaitReadable - :wait_readable - rescue EOFError - nil - end - end end diff --git a/lib/dtas/unix_client.rb b/lib/dtas/unix_client.rb index aae8c9d..8c73b7d 100644 --- a/lib/dtas/unix_client.rb +++ b/lib/dtas/unix_client.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true require_relative '../dtas' @@ -24,7 +24,7 @@ class DTAS::UNIXClient # :nodoc: def req_start(args) args = xs(args) if Array === args - @to_io.send(args, Socket::MSG_EOR) + @to_io.send(args, 0) end def req_ok(args, timeout = nil) @@ -39,7 +39,7 @@ class DTAS::UNIXClient # :nodoc: end def res_wait(timeout = nil) - IO.select([@to_io], nil, nil, timeout) + @to_io.wait_readable(timeout) nr = @to_io.nread nr > 0 or raise EOFError, "unexpected EOF from server" @to_io.recv(nr) diff --git a/lib/dtas/unix_server.rb b/lib/dtas/unix_server.rb index ccfa662..60ab86c 100644 --- a/lib/dtas/unix_server.rb +++ b/lib/dtas/unix_server.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true require 'socket' @@ -59,7 +59,7 @@ class DTAS::UNIXServer # :nodoc: def readable_iter # we do not do anything with the block passed to us - case rv = accept_nonblock + case rv = @to_io.accept_nonblock(exception: false) when :wait_readable then return rv else @readers[DTAS::UNIXAccepted.new(rv[0])] = true @@ -114,16 +114,4 @@ class DTAS::UNIXServer # :nodoc: wait_ctl(io, io.readable_iter { |_io, msg| yield(_io, msg) }) end end - - if RUBY_VERSION.to_f >= 2.3 - def accept_nonblock - @to_io.accept_nonblock(exception: false) - end - else - def accept_nonblock - @to_io.accept_nonblock - rescue Errno::EAGAIN, Errno::ECONNABORTED, Errno::EPROTO - :wait_readable - end - end end diff --git a/lib/dtas/util.rb b/lib/dtas/util.rb index acfcafe..a74c14e 100644 --- a/lib/dtas/util.rb +++ b/lib/dtas/util.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) 2013-2020 all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true require_relative '../dtas' diff --git a/lib/dtas/watchable.rb b/lib/dtas/watchable.rb index d0f37af..445bf98 100644 --- a/lib/dtas/watchable.rb +++ b/lib/dtas/watchable.rb @@ -1,71 +1,71 @@ -# Copyright (C) 2013-2019 all contributors <dtas-all@nongnu.org> +# Copyright (C) all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true +require_relative '../dtas' begin -require 'sleepy_penguin' + module DTAS::Watchable # :nodoc: + module InotifyCommon # :nodoc: + FLAGS = 8 | 128 # IN_CLOSE_WRITE | IN_MOVED_TO -# used to restart DTAS::Source::SplitFX processing in dtas-player -# if the YAML file is edited -module DTAS::Watchable # :nodoc: - class InotifyReadableIter < SleepyPenguin::Inotify # :nodoc: - def self.new - super(:CLOEXEC) - end - - FLAGS = CLOSE_WRITE | MOVED_TO - - def readable_iter - or_call = false - while event = take(true) # drain the buffer - w = @watches[event.wd] or next - if (event.mask & FLAGS) != 0 && w[event.name] - or_call = true + def readable_iter + or_call = false + while event = take(true) # drain the buffer + w = @watches[event.wd] or next + if (event.mask & FLAGS) != 0 && w[event.name] + or_call = true + end + end + if or_call + @on_readable.call + :delete + else + :wait_readable end end - if or_call - @on_readable.call - :delete - else - :wait_readable - end - end - # we must watch the directory, since - def watch_files(paths, blk) - @watches = {} # wd -> { basename -> true } - @on_readable = blk - @dir2wd = {} - Array(paths).each do |path| - watchdir, watchbase = File.split(File.expand_path(path)) - begin - wd = @dir2wd[watchdir] ||= add_watch(watchdir, FLAGS) - m = @watches[wd] ||= {} - m[watchbase] = true - rescue SystemCallError => e - warn "#{watchdir.dump}: #{e.message} (#{e.class})" + # we must watch the directory, since + def watch_files(paths, blk) + @watches = {} # wd -> { basename -> true } + @on_readable = blk + @dir2wd = {} + Array(paths).each do |path| + watchdir, watchbase = File.split(File.expand_path(path)) + begin + wd = @dir2wd[watchdir] ||= add_watch(watchdir, FLAGS) + m = @watches[wd] ||= {} + m[watchbase] = true + rescue SystemCallError => e + warn "#{watchdir.dump}: #{e.message} (#{e.class})" + end end end - end - end + end # module InotifyCommon - def watch_begin(blk) - @ino = InotifyReadableIter.new - @ino.watch_files(@watch_extra << @infile, blk) - @ino - end + begin + require_relative 'watchable/inotify' + rescue LoadError + # TODO: support kevent + require_relative 'watchable/fiddle_ino' + end - def watch_extra(paths) - @ino.watch_extra(paths) - end + def watch_begin(blk) + @ino = DTAS::Watchable::InotifyReadableIter.new + @ino.watch_files(@watch_extra << @infile, blk) + @ino + end - # Closing the inotify descriptor (instead of using inotify_rm_watch) - # is cleaner because it avoids EINVAL on race conditions in case - # a directory is deleted: https://lkml.org/lkml/2007/7/9/3 - def watch_end(srv) - srv.wait_ctl(@ino, :delete) - @ino = @ino.close - end -end + def watch_extra(paths) + @ino.watch_extra(paths) + end -rescue LoadError -end + # Closing the inotify descriptor (instead of using inotify_rm_watch) + # is cleaner because it avoids EINVAL on race conditions in case + # a directory is deleted: https://lkml.org/lkml/2007/7/9/3 + def watch_end(srv) + srv.wait_ctl(@ino, :delete) + @ino = @ino.close + end + end # module DTAS::Watchable +rescue LoadError, StandardError => e + warn "#{e.message} (#{e.class})" +end # begin diff --git a/lib/dtas/watchable/fiddle_ino.rb b/lib/dtas/watchable/fiddle_ino.rb new file mode 100644 index 0000000..3ec72a1 --- /dev/null +++ b/lib/dtas/watchable/fiddle_ino.rb @@ -0,0 +1,78 @@ +# Copyright (C) all contributors <dtas-all@nongnu.org> +# License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> +# frozen_string_literal: true +require 'fiddle' + +# used to restart DTAS::Source::SplitFX processing in dtas-player +# if the YAML file is edited +class DTAS::Watchable::InotifyReadableIter # :nodoc: + include DTAS::Watchable::InotifyCommon + + Inotify_init = Fiddle::Function.new(DTAS.libc['inotify_init1'], + [ Fiddle::TYPE_INT ], + Fiddle::TYPE_INT) + + Inotify_add_watch = Fiddle::Function.new(DTAS.libc['inotify_add_watch'], + [ Fiddle::TYPE_INT, Fiddle::TYPE_VOIDP, Fiddle::TYPE_INT ], + Fiddle::TYPE_INT) + + # IO.select compatibility + attr_reader :to_io #:nodoc: + + def initialize # :nodoc: + fd = Inotify_init.call(02000000 | 04000) # CLOEXEC | NONBLOCK + raise "inotify_init failed: #{Fiddle.last_error}" if fd < 0 + @to_io = IO.for_fd(fd) + @buf = ''.b + @q = [] + end + + # struct inotify_event { + # int wd; /* Watch descriptor */ + # uint32_t mask; /* Mask describing event */ + # uint32_t cookie; /* Unique cookie associating related + # events (for rename(2)) */ + # uint32_t len; /* Size of name field */ + # char name[]; /* Optional null-terminated name */ + InotifyEvent = Struct.new(:wd, :mask, :cookie, :len, :name) # :nodoc: + + def take(nonblock) # :nodoc: + event = @q.pop and return event + case rv = @to_io.read_nonblock(16384, @buf, exception: false) + when :wait_readable, nil + return + else + until rv.empty? + hdr = rv.slice!(0,16) + name = nil + wd, mask, cookie, len = res = hdr.unpack('iIII') + wd && mask && cookie && len or + raise "bogus inotify_event #{res.inspect} hdr=#{hdr.inspect}" + if len > 0 + name = rv.slice!(0, len) + name.size == len or raise "short name #{name.inspect} != #{len}" + name.sub!(/\0+\z/, '') or + raise "missing: `\\0', inotify_event.name=#{name.inspect}" + name = -name + end + ie = InotifyEvent.new(wd, mask, cookie, len, name) + if event + @q << ie + else + event = ie + end + end # /until rv.empty? + return event + end while true + end + + def add_watch(watchdir, flags) + wd = Inotify_add_watch.call(@to_io.fileno, watchdir, flags) + raise "inotify_add_watch failed: #{Fiddle.last_error}" if wd < 0 + wd + end + + def close + @to_io = @to_io.close if @to_io + end +end diff --git a/lib/dtas/watchable/inotify.rb b/lib/dtas/watchable/inotify.rb new file mode 100644 index 0000000..36b5746 --- /dev/null +++ b/lib/dtas/watchable/inotify.rb @@ -0,0 +1,13 @@ +# Copyright (C) 2013-2020 all contributors <dtas-all@nongnu.org> +# License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> +# frozen_string_literal: true +require 'sleepy_penguin' + +# used to restart DTAS::Source::SplitFX processing in dtas-player +# if the YAML file is edited +class DTAS::Watchable::InotifyReadableIter < SleepyPenguin::Inotify # :nodoc: + include DTAS::Watchable::InotifyCommon + def self.new + super(:CLOEXEC) + end +end diff --git a/lib/dtas/writable_iter.rb b/lib/dtas/writable_iter.rb index 24d1cee..caf6850 100644 --- a/lib/dtas/writable_iter.rb +++ b/lib/dtas/writable_iter.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) 2013-2020 all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true require_relative '../dtas' diff --git a/lib/dtas/xs.rb b/lib/dtas/xs.rb index 6836172..d4586a3 100644 --- a/lib/dtas/xs.rb +++ b/lib/dtas/xs.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org> +# Copyright (C) 2013-2020 all contributors <dtas-all@nongnu.org> # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> # frozen_string_literal: true require_relative '../dtas' |