avoid excessive garbage on uploads with Ruby 2.2+

This is a workaround for <https://bugs.ruby-lang.org/issues/13085>
since we use non-blocking sockets anyways.

2 files changed, 46 insertions, 0 deletions

diff --git a/lib/mogilefs/socket_common.rb b/lib/mogilefs/socket_common.rb
index 5ba8c10..9383731 100644
--- a/lib/mogilefs/socket_common.rb
+++ b/lib/mogilefs/socket_common.rb
@@ -56,4 +56,17 @@ module MogileFS::SocketCommon
   def readpartial(size, buf = "", timeout = 5)
     timed_read(size, buf, timeout) or raise EOFError, "end of file reached"
   end
+
+  # workaround for https://bugs.ruby-lang.org/issues/13085
+  # (excessive garbage from IO#write)
+  # XXX maybe this can be fixed for Ruby 2.5 final, but maybe not:
+  # Update this when Ruby 2.5 is released on 2017-12-25
+  if defined?(RUBY_ENGINE) && RUBY_ENGINE == 'ruby' &&
+     RUBY_VERSION.to_f >= 2.2 && RUBY_VERSION.to_f <= 2.5
+    def write(buf)
+      # Blocking TCP writes would error out long before one day,
+      # and MogileFS won't allow file creations which take over a day.
+      timed_write(buf, 86400)
+    end
+  end
 end
diff --git a/test/test_cmogstored.rb b/test/test_cmogstored.rb
index 586000b..0dd920a 100644
--- a/test/test_cmogstored.rb
+++ b/test/test_cmogstored.rb
@@ -27,6 +27,25 @@ class Test_cmogstored < Test::Unit::TestCase
     assert_equal "a\nb\nc\nd\ne\n", client.get_file_data("puts")
   end
 
+  def test_garbage
+    add_host_device_domain
+    client = MogileFS::MogileFS.new :hosts => @hosts, :domain => @domain,
+                                    :timeout => 60
+    nr = 1024 * 1024 * 1024
+    client.new_file('giant', :largefile => :stream,
+                    :content_length => nr) do |io|
+      assert_instance_of MogileFS::NewFile::Stream, io
+      zero = Zero.new
+      before = GC.count
+      wr = IO.copy_stream(zero, io, nr)
+      after = GC.count
+      assert_equal nr, wr
+      assert_in_delta before, after, 1
+    end
+  end if IO.respond_to?(:copy_stream) && defined?(Zero) &&
+         GC.respond_to?(:count) && defined?(RUBY_ENGINE) &&
+         RUBY_ENGINE == 'ruby' && ENV['TEST_EXPENSIVE'].to_i != 0
+
   def test_stream_new_file
     add_host_device_domain
     client = MogileFS::MogileFS.new :hosts => @hosts, :domain => @domain
@@ -146,3 +165,17 @@ class Test_cmogstored < Test::Unit::TestCase
     end
   end
 end if `which cmogstored`.chomp.size > 0
+
+# The goal of this is to use a synthetic (non-IO) reader
+# to trigger the read/write loop of IO.copy_stream,
+# bypassing in-kernel mechanisms like sendfile for zero copy,
+# so we wrap the /dev/zero IO object:
+class Zero
+  def initialize
+    @in = File.open('/dev/zero', 'rb')
+  end
+
+  def read(len, buf)
+    @in.read(len, buf)
+  end
+end if File.readable?('/dev/zero')