[PATCH] lib/net/*: recycle and clear short-lived read buffers

[PATCH] lib/net/*: recycle and clear short-lived read buffers

[Eric Wong]

Using a parallel Net::HTTP downloader, this reduced memory usage
from around 120MB to 27MB on my 32-bit x86 system.

* lib/net/protocol.rb (rbuf_fill): accept destination buffer, clear if used
  (read): reuse result of rbuf_consume for rbuf_fill
  (read_all): ditto
  (read_until): ditto
* lib/net/ftp.rb (rbuf_fill): update to match

Test script I used:

  require 'net/http'
  require 'uri'
  require 'digest/sha1'
  url = 'http://80x24.org/git-i-forgot-to-pack/objects/pack/pack-97b25a76c03b489d4cbbd85b12d0e1ad28717e55.idx'
  uri = URI(url)
  use_ssl = "https" == uri.scheme
  thrs = 30.times.map do
    Thread.start do
      cur = Thread.current.object_id
      Net::HTTP.start(uri.host, uri.port, use_ssl: use_ssl) do |http|
        req = Net::HTTP::Get.new(uri)
        http.request(req) do |res|
          dig = Digest::SHA1.new
          res.read_body do |buf|
            dig.update(buf)
            #buf.clear # most Ruby programmers don't do this :<
          end
          warn "#{Time.now} #{cur} #{dig.hexdigest}\n"
        end
      end
      :done
    end
  end

  p thrs.map(&:value)
---
 lib/net/ftp.rb      |  4 ++--
 lib/net/protocol.rb | 15 +++++++++------
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/lib/net/ftp.rb b/lib/net/ftp.rb
index 601d2cb4c3..af163885bc 100644
--- a/lib/net/ftp.rb
+++ b/lib/net/ftp.rb
@@ -1478,11 +1478,11 @@ def send(mesg, flags, dest = nil)
 
         private
 
-        def rbuf_fill
+        def rbuf_fill(dst)
           if @is_shutdown
             raise EOFError, "shutdown has been called"
           else
-            super
+            super(dst)
           end
         end
       end
diff --git a/lib/net/protocol.rb b/lib/net/protocol.rb
index 518b92c4ab..f492298a03 100644
--- a/lib/net/protocol.rb
+++ b/lib/net/protocol.rb
@@ -121,10 +121,11 @@ def read(len, dest = ''.dup, ignore_eof = false)
         while read_bytes + @rbuf.size < len
           dest << (s = rbuf_consume(@rbuf.size))
           read_bytes += s.size
-          rbuf_fill
+          rbuf_fill(s)
         end
         dest << (s = rbuf_consume(len - read_bytes))
         read_bytes += s.size
+        s.clear
       rescue EOFError
         raise unless ignore_eof
       end
@@ -139,7 +140,7 @@ def read_all(dest = ''.dup)
         while true
           dest << (s = rbuf_consume(@rbuf.size))
           read_bytes += s.size
-          rbuf_fill
+          rbuf_fill(s)
         end
       rescue EOFError
         ;
@@ -151,7 +152,7 @@ def read_all(dest = ''.dup)
     def readuntil(terminator, ignore_eof = false)
       begin
         until idx = @rbuf.index(terminator)
-          rbuf_fill
+          rbuf_fill(''.b)
         end
         return rbuf_consume(idx + terminator.size)
       rescue EOFError
@@ -168,10 +169,12 @@ def readline
 
     BUFSIZE = 1024 * 16
 
-    def rbuf_fill
-      case rv = @io.read_nonblock(BUFSIZE, exception: false)
+    def rbuf_fill(dst)
+      case rv = @io.read_nonblock(BUFSIZE, dst, exception: false)
       when String
-        return @rbuf << rv
+        @rbuf << rv
+        dst.clear
+        return
       when :wait_readable
         @io.to_io.wait_readable(@read_timeout) or raise Net::ReadTimeout
         # continue looping
-- 
EW