1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
|
#!/usr/bin/env ruby
# Copyright (C) 2015-2016 all contributors <dtas-all@nongnu.org>
# License: GPL-3.0+ (https://www.gnu.org/licenses/gpl-3.0.txt)
# frozen_string_literal: true
usage = "#$0 SOURCE DESTINATION"
# We could use the equivalent sox command here, but some folks working on
# dtas is more likely to write patches for sox (and thus introduce bugs
# into it), so we'll use sndfile-cmp as it lives in a different source tree
%w(sndfile-cmp sox).each do |cmd|
`which #{cmd} 2>/dev/null`.chomp.empty? and abort "#{cmd} not found in PATH"
end
RUBY_PLATFORM =~ /linux/ or
warn "#$0 is unproven without Linux kernel fadvise semantics"
have_advise = IO.instance_methods.include?(:advise)
have_advise or warn "#$0 does not work reliably without IO#advise support"
require 'shellwords'
require 'fileutils'
require 'find'
require 'optparse'
Thread.abort_on_exception = true
dry_run = false
silent = false
type = 'flac'
jobs = 1
repeat = 1
stats = false
keep_going = false
compression = []
OptionParser.new('', 24, ' ') do |op|
op.banner = usage
op.on('-t', '--type [TYPE]', 'FILE-TYPE (default: flac)') { |t| type = t }
op.on('-C', '--compression [FACTOR]', 'compression factor for sox') { |c|
compression = [ '-C', c ]
}
op.on('-j', '--jobs [JOBS]', Integer) { |j| jobs = j }
op.on('-S', '--stats', 'save stats on the file') { stats = true }
op.on('-k', '--keep-going', 'continue after error') { keep_going = true }
op.on('-n', '--dry-run', 'only print commands, do not run them') do
dry_run = true
end
op.on('-r', '--repeat [COUNT]', 'number of times to check', Integer) do |r|
repeat = r
end
op.on('-s', '--quiet', '--silent') { silent = true }
op.on('-h', '--help') do
puts(op.to_s)
exit
end
op.parse!(ARGV)
end
dst = ARGV.pop
src = ARGV.dup
FileUtils.mkpath(dst) unless File.exist?(dst)
src_files = Hash.new { |h,dest_dir| h[dest_dir] = [] }
src.each do |s|
src_st = File.stat(s)
if src_st.directory?
Find.find(s) do |path|
File.file?(path) or next
dir = File.dirname(path)
dir_st = File.stat(dir)
if dir_st.ino == src_st.ino && dir_st.dev == src_st.dev
src_files['.'] << path
else
dir = File.basename(File.dirname(path))
src_files[dir] << path
end
end
else
src_files['.'] << s
end
end
pairs = []
type = ".#{type}" unless type.start_with?('.')
src_files.each do |dir, files|
dir = dir == '.' ? dst : File.join(dst, dir)
if dry_run || !silent
puts "mkdir -p #{Shellwords.escape(dir)}"
end
FileUtils.mkpath(dir) unless dry_run
files.each do |path|
base = File.basename(path).sub(/\.[^\.]+\z/, type)
out = File.join(dir, base)
pairs << [ path, out ]
end
end
mtx = Mutex.new # protects fails and pairs
fails = []
mismatches = []
on_fail = lambda do |job, status|
mtx.synchronize do
pairs.clear unless keep_going
fails << [ job, status ]
end
end
on_mismatch = lambda do |job, status|
mtx.synchronize do
mismatches << [ job, status ]
end
end
exiting = false
%w(INT TERM).each do |s|
trap(s) do
warn "Caught SIG#{s}, stopping gracefully..."
exiting = true
trap(s, 'DEFAULT') # non-graceful if signaled again
end
end
thrs = jobs.times.map do |i|
Thread.new do
while job = mtx.synchronize { pairs.shift }
break if exiting
input, output = *job
unless system('soxi', '-s', input, out: IO::NULL, err: IO::NULL)
warn "skipping #{input.inspect}, not an audio file"
next
end
stats_out = "#{output.sub(/\.[\.]+\z/, '')}.stats" if stats
if dry_run || !silent
names = job.map { |x| Shellwords.escape(x) }
cmd = [ 'sox', *names ]
if stats
cmd << 'stats'
cmd << "2>#{Shellwords.escape(stats_out)}"
end
puts cmd.join(' ')
cmpcmd = "sndfile-cmp #{names[0]} #{names[1]}"
if dry_run
puts cmpcmd
next
end
end
cmd = [ 'sox', input, *compression, output ]
if stats
cmd << 'stats'
cmd = [ *cmd, { err: stats_out } ]
end
system(*cmd) or on_fail.call(job, $?)
# clear kernel caches, this relies on Linux behavior
repeat.times do
if have_advise
th = Thread.new { File.open(input) { |fp| fp.advise(:dontneed) } }
File.open(output, 'ab') do |fp|
fp.fsync
fp.advise(:dontneed)
end
th.join
end
puts cmpcmd unless silent
system('sndfile-cmp', input, output) or on_mismatch.call(job, $?)
end
st = File.stat(input)
File.utime(st.atime, st.mtime, output)
end
end
end
thrs.each(&:join)
ok = true
fails.each do |job, status|
$stderr.puts "#{job.inspect} failed: #{status.inspect}"
ok = false
end
mismatches.each do |job, status|
$stderr.puts "#{job.inspect} mismatched: #{status.inspect}"
ok = false
end
exit ok
|