about summary refs log tree commit homepage
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/dtas.rb2
-rw-r--r--lib/dtas/encoding.rb58
-rw-r--r--lib/dtas/source/sox.rb4
3 files changed, 63 insertions, 1 deletions
diff --git a/lib/dtas.rb b/lib/dtas.rb
index ac416d7..3c2cdb4 100644
--- a/lib/dtas.rb
+++ b/lib/dtas.rb
@@ -42,3 +42,5 @@ end
 
 require_relative 'dtas/compat_onenine'
 require_relative 'dtas/spawn_fix'
+require_relative 'dtas/encoding'
+DTAS.extend(DTAS::Encoding)
diff --git a/lib/dtas/encoding.rb b/lib/dtas/encoding.rb
new file mode 100644
index 0000000..71c877f
--- /dev/null
+++ b/lib/dtas/encoding.rb
@@ -0,0 +1,58 @@
+# Copyright (C) 2018 all contributors <dtas-all@nongnu.org>
+# License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
+# frozen_string_literal: true
+
+# This module gets included in DTAS
+module DTAS::Encoding # :nodoc:
+  def self.extended(mod)
+    mod.instance_eval { @charlock_holmes = nil}
+  end
+
+private
+
+  def try_enc_harder(str, enc, old) # :nodoc:
+    case @charlock_holmes
+    when nil
+      begin
+        require 'charlock_holmes'
+        @charlock_holmes = CharlockHolmes::EncodingDetector.new
+      rescue LoadError
+        warn "`charlock_holmes` gem not available for encoding detection"
+        @charlock_holmes = false
+      end
+    when false
+      enc_fallback(str, enc, old)
+    else
+      res = @charlock_holmes.detect(str)
+      if det = res[:ruby_encoding]
+        str.force_encoding(det)
+        warn "charlock_holmes detected #{str.inspect} as #{det}..."
+        str.valid_encoding? or enc_fallback(str, det, old)
+      else
+        enc_fallback(str, enc, old)
+      end
+    end
+    str
+  end
+
+  def enc_fallback(str, enc, old) # :nodoc:
+    str.force_encoding(old)
+    warn "could not detect encoding for #{str.inspect} (not #{enc})"
+  end
+
+public
+
+  def try_enc(str, enc, harder = true) # :nodoc:
+    old = str.encoding
+    return str if old == enc
+    str.force_encoding(enc)
+    unless str.valid_encoding?
+      if harder
+        try_enc_harder(str, enc, old)
+      else
+        enc_fallback(str, enc, old)
+      end
+    end
+    str
+  end
+end
diff --git a/lib/dtas/source/sox.rb b/lib/dtas/source/sox.rb
index f702b41..5e967c1 100644
--- a/lib/dtas/source/sox.rb
+++ b/lib/dtas/source/sox.rb
@@ -50,17 +50,19 @@ class DTAS::Source::Sox # :nodoc:
       out =~ /^Sample Rate\s*:\s*(\d+)/n and dst['rate'] = $1.to_i
       out =~ /^Precision\s*:\s*(\d+)-bit/n and dst['bits'] = $1.to_i
 
+      enc = Encoding.default_external # typically Encoding::UTF_8
       if out =~ /\nComments\s*:[ \t]*\n?(.*)\z/mn
         comments = dst['comments'] = {}
         key = nil
         $1.split(/\n/n).each do |line|
           if line.sub!(/^([^=]+)=/ni, '')
-            key = DTAS.dedupe_str($1.upcase)
+            key = DTAS.dedupe_str(DTAS.try_enc($1.upcase, enc))
           end
           (comments[key] ||= ''.b) << "#{line}\n" unless line.empty?
         end
         comments.each do |k,v|
           v.chomp!
+          DTAS.try_enc(v, enc)
           comments[k] = DTAS.dedupe_str(v)
         end
       end