http_file: wait for the destination server to respond

Uploading large files can cause the destination to take
equally long to respond, either because of extra I/O
needed to verify or because we've hit fsync(2) or similar.

1 files changed, 16 insertions, 1 deletions

diff --git a/lib/mogilefs/http_file.rb b/lib/mogilefs/http_file.rb
index 1611a5d..b08abaa 100644
--- a/lib/mogilefs/http_file.rb
+++ b/lib/mogilefs/http_file.rb
@@ -20,7 +20,19 @@ class MogileFS::HTTPFile < StringIO
   end
   class NonRetryableError < MogileFS::Error; end
 
+  # :stopdoc:
   MD5_TRAILER_NODES = {} # :nodoc: # EXPERIMENTAL
+  class << self
+    attr_accessor :response_timeout_cb
+  end
+
+  @response_timeout_cb = lambda do |elapsed_time, bytes_uploaded|
+    mbytes_uploaded = bytes_uploaded / (1024.0 * 1024.0)
+    # assumes worst case is 10M/s on the remote storage disk
+    t = mbytes_uploaded * 10 + elapsed_time
+    t < 5 ? 5 : t
+  end
+  # :startdoc:
 
   ##
   # The URI this file will be stored to.
@@ -94,6 +106,7 @@ class MogileFS::HTTPFile < StringIO
   # Writes an HTTP PUT request to +sock+ to upload the file and
   # returns file size if the socket finished writing
   def upload(devid, uri) # :nodoc:
+    start = Time.now
     sock = MogileFS::Socket.tcp(uri.host, uri.port)
     file_size = length
 
@@ -119,7 +132,9 @@ class MogileFS::HTTPFile < StringIO
       request_put(sock, uri, file_size, self)
     end
 
-    case line = sock.timed_read(23, "")
+    tout = self.class.response_timeout_cb.call(Time.now - start, file_size)
+
+    case line = sock.timed_read(23, "", tout)
     when %r{^HTTP/\d\.\d\s+(2\d\d)\s} # success!
       file_size
     when nil