list_keys_verbose: pipeline retries on all socket errors

We'll try to retry for all transient issues that aren't
our fault

2 files changed, 51 insertions, 2 deletions

diff --git a/lib/mogilefs/mogilefs.rb b/lib/mogilefs/mogilefs.rb
index 73827c3..79b4895 100644
--- a/lib/mogilefs/mogilefs.rb
+++ b/lib/mogilefs/mogilefs.rb
@@ -276,7 +276,11 @@ class MogileFS::MogileFS < MogileFS::Client
         paths = get_paths(key, args)
         block.call(key, paths_size(paths), paths.size)
       end
-    rescue MogileFS::PipelineError
+    rescue MogileFS::PipelineError, SystemCallError,
+           MogileFS::RequestTruncatedError,
+           MogileFS::UnreadableSocketError,
+           MogileFS::InvalidResponseError, # truncated response
+           MogileFS::Timeout
       @backend.shutdown
       keys = ordered - ready.keys
       retry
diff --git a/test/test_mogilefs.rb b/test/test_mogilefs.rb
index 143e187..7b231fa 100644
--- a/test/test_mogilefs.rb
+++ b/test/test_mogilefs.rb
@@ -670,7 +670,7 @@ class TestMogileFS__MogileFS < TestMogileFS
       sock.close
   end
 
-  def test_list_keys_verbose_retry # implementation detail
+  def test_list_keys_verbose_retry_eof # implementation detail
     received = []
     sock = TCPServer.new("127.0.0.1", 0)
     nargs = { :hosts => [ "127.0.0.1:#{sock.addr[1]}" ], :domain => "foo" }
@@ -714,6 +714,51 @@ class TestMogileFS__MogileFS < TestMogileFS
       sock.close
   end
 
+  def test_list_keys_verbose_retry_truncated # implementation detail
+    received = []
+    sock = TCPServer.new("127.0.0.1", 0)
+    nargs = { :hosts => [ "127.0.0.1:#{sock.addr[1]}" ], :domain => "foo" }
+    c = MogileFS::MogileFS.new(nargs)
+    th = Thread.new do
+      a = sock.accept
+      %w(a b c d e).each do |key|
+        line = a.gets
+        cmd, args = line.split(/\s+/, 2)
+        args = c.backend.url_decode(args.strip)
+        assert_equal "file_info", cmd
+        assert_equal key, args["key"]
+      end
+      out = { "length" => 3, "devcount" => 6 }
+      out["key"] = "a"
+      a.write "OK #{c.backend.url_encode(out)}\r\n"
+      out["key"] = "b"
+      a.write "OK #{c.backend.url_encode(out)}"
+      a.close # trigger EOF
+
+      a = sock.accept # client will retry
+      %w(b c d e).each do |key|
+        line = a.gets
+        cmd, args = line.split(/\s+/, 2)
+        args = c.backend.url_decode(args.strip)
+        assert_equal "file_info", cmd
+        assert_equal key, args["key"]
+        out["key"] = key
+        a.write "OK #{c.backend.url_encode(out)}\r\n"
+      end
+      a.close
+    end
+
+    blk = lambda do |key, length, devcount|
+      received << [ key, length, devcount ]
+    end
+    c.list_keys_verbose(%w(a b c d e), blk)
+    th.join
+    received.map! { |(key,_,_)| key }
+    assert_equal %w(a b c d e), received
+    ensure
+      sock.close
+  end
+
   def test_sleep
     @backend.sleep = {}
     assert_nothing_raised do