finalize and document improved new_file API

5 files changed, 107 insertions, 12 deletions

diff --git a/.document b/.document
index 91c2780..db7b04c 100644
--- a/.document
+++ b/.document
@@ -10,3 +10,5 @@ lib/mogilefs/admin.rb
 lib/mogilefs/backend.rb
 lib/mogilefs/client.rb
 lib/mogilefs/mogilefs.rb
+lib/mogilefs/new_file.rb
+lib/mogilefs/new_file/writer.rb
diff --git a/lib/mogilefs/mogilefs.rb b/lib/mogilefs/mogilefs.rb
index b192605..8a35168 100644
--- a/lib/mogilefs/mogilefs.rb
+++ b/lib/mogilefs/mogilefs.rb
@@ -125,16 +125,30 @@ class MogileFS::MogileFS < MogileFS::Client
     get_paths(key, *args).map! { |path| URI.parse(path) }
   end
 
-  # Creates a new file +key+ in +klass+.  +bytes+ is currently unused.
-  # Consider using store_file instead of this method for large files.
-  # This requires a block passed to it and operates like File.open.
-  # This atomically replaces existing data stored as +key+ when
+  # Creates a new file +key+ in the domain of this object.
+  #
+  # +bytes+ is the expected size of the file if known in advance
+  #
+  # It operates like File.open(..., "w") and may take an optional
+  # block, yielding an IO-like object with support for the methods
+  # documented in MogileFS::NewFile::Writer.
+  #
+  # This atomically replaces existing data stored as +key+
+  # when the block exits or when the returned object is closed.
   #
   # +args+ may contain the following options:
-  # * :content_length => Integer
-  # * :largefile => [ :stream, :content_range, :tempfile ]
-  #    (see # MogileFS::NewFile)
-  # * :content_md5 => String, Proc, or :trailer
+  #
+  # [:content_length => Integer]
+  #
+  #   This has the same effect as the (deprecated) +bytes+ parameter.
+  #
+  # [ :largefile => :stream, :content_range or :tempfile ]
+  #
+  #   See MogileFS::NewFile for more information on this
+  #
+  # [ :class => String]
+  #
+  #   The MogileFS storage class of the object.
   def new_file(key, args = nil, bytes = nil) # :yields: file
     raise MogileFS::ReadOnlyError if readonly?
     opts = { :key => key, :multi_dest => 1 }
@@ -167,7 +181,7 @@ class MogileFS::MogileFS < MogileFS::Client
 
     case (dests[0][1] rescue nil)
     when %r{\Ahttp://}
-      http_file = MogileFS::NewFile[opts[:largefile]].new(dests, opts)
+      http_file = MogileFS::NewFile.new(dests, opts)
       if block_given?
         yield http_file
         return http_file.commit # calls create_close
diff --git a/lib/mogilefs/new_file.rb b/lib/mogilefs/new_file.rb
index e99a023..baac6e2 100644
--- a/lib/mogilefs/new_file.rb
+++ b/lib/mogilefs/new_file.rb
@@ -1,8 +1,67 @@
 # -*- encoding: binary -*-
+#
+# The MogileFS::MogileFS#new_file method is enhanced in v3.1.0+
+# to support the :largefile parameter.  While we have always
+# supported large files via the "store_file" method, streaming
+# large amounts of content of an unknown length required the use
+# of awkward APIs.
+#
+# It is possible to stream large content of known length any WebDAV server.
+# One example of this is for mirroring a file from an existing HTTP server without download downloading
+
+#
+#   uri = URI('http://example.com/large_file')
+#   Net::HTTP.start(uri.host, uri.port) do |http|
+#     req = Net::HTTP::Get.new(uri.request_uri)
+#
+#     http.request(req) do |response|
+#       if len = response.content_length
+#         io = mg.new_file('key', :largefile => true, :content_length => len)
+#       else
+#         warn "trying to upload with Transfer-Encoding: chunked"
+#         warn "this is not supported by all WebDAV servers"
+#         io = mg.new_file('key', :largefile => :chunked)
+#       end
+#       response.read_body { |buf| io.write(buf) }
+#       io.close
+#     end
+#   end
+
+#   nf = mg.new_file("key", :largefile => stream, :content_length => )
+#   nf.write(buf)
+#   nf.close
+#
+# If your WebDAV servers have chunked PUT support (e.g. Perlbal), you can
+# stream a file of unknown length using "Transfer-Encoding: chunked".
+#
+#   nf = mg.new_file("key", :largefile => :stream)
+#   nf.write "hello"
+#   nf.write ...
+#   nf.close
+#
+# If your WebDAV server has partial PUT support (e.g Apache), you can
+# you can use multiple PUT requests with "Content-Range" support.
+# This method is slower than Transfer-Encoding: chunked.
+#
+#   nf = mg.new_file("key", :largefile => :content_range)
+#   nf.write "hello"
+#   nf.write ...
+#   nf.close
+#
+# Finally, if your WebDAV servers does not support either partial nor
+# nor chunked PUTs, you must buffer a file of unknown length using a
+# Tempfile:
+#
+#   nf = mg.new_file("key", :largefile => :tempfile)
+#   nf.write "hello"
+#   nf.close
+#
 module MogileFS::NewFile
 
   # avoiding autoload for new code since it's going away in Ruby...
-  def self.[](largefile) # :nodoc:
+  def self.new(dests, opts) # :nodoc:
+    largefile = opts[:largefile]
+    largefile = :stream if largefile && opts[:content_length]
     require "mogilefs/new_file/#{largefile}" if Symbol === largefile
     case largefile
     when nil, false
@@ -15,7 +74,7 @@ module MogileFS::NewFile
       Tempfile
     else
       raise ArgumentError, "largefile: #{largefile.inspect} not understood"
-    end
+    end.new(dests, opts)
   end
 end
 
diff --git a/lib/mogilefs/new_file/writer.rb b/lib/mogilefs/new_file/writer.rb
index 994545b..d58a4e6 100644
--- a/lib/mogilefs/new_file/writer.rb
+++ b/lib/mogilefs/new_file/writer.rb
@@ -1,7 +1,11 @@
 # -*- encoding: binary -*-
-# here are internal implementation details, do not use them in your code
 #
+# All objects yielded or returned by MogileFS::MogileFS#new_file should
+# conform to this interface (based on existing IO methods).  These objects
+# should be considered write-only.
 module MogileFS::NewFile::Writer
+
+  # see IO#puts
   def puts(*args)
     args.each do |obj|
       write(obj)
@@ -10,11 +14,13 @@ module MogileFS::NewFile::Writer
     nil
   end
 
+  # see IO#putc
   def putc(ch)
     write(ch.respond_to?(:chr) ? ch.chr : ch[0])
     ch
   end
 
+  # see IO#print
   def print(*args)
     args = [ $_ ] unless args[0]
     write(args.shift)
@@ -26,16 +32,24 @@ module MogileFS::NewFile::Writer
     nil
   end
 
+  # see IO#printf
   def printf(*args)
     write(sprintf(*args))
     nil
   end
 
+  # see IO#<<
   def <<(str)
     write(str)
     self
   end
 
+  # This will issue the +create_close+ command to the MogileFS tracker
+  # and finalize the creation of a new file.  This returns +nil+ on
+  # success and will raise IOError if called twice.  For non-streaming
+  # implementations, this will initiate and finalize the upload.
+  #
+  # see IO#close
   def close
     commit
     nil
diff --git a/test/test_mogilefs_integration.rb b/test/test_mogilefs_integration.rb
index b6a181a..2ce8159 100644
--- a/test/test_mogilefs_integration.rb
+++ b/test/test_mogilefs_integration.rb
@@ -190,6 +190,12 @@ class TestMogileFSIntegration < TestMogIntegration
     assert_nothing_raised { rv.write "GOOD" }
     assert_raises(MogileFS::SizeMismatchError) { rv.close }
     assert_equal "HELLO", @client.get_file_data("a")
+
+    rv = @client.new_file("large", :content_length => 6, :largefile => true)
+    assert_instance_of MogileFS::NewFile::Stream, rv
+    assert_equal 6, rv.write("HIHIHI")
+    assert_nil rv.close
+    assert_equal "HIHIHI", @client.get_file_data("large")
   end
 
   def test_new_file_content_md5