From b69111a6573826d7c4fc3edd4fcc514d0eb425e9 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Fri, 9 Dec 2011 18:22:01 -0800 Subject: finalize and document improved new_file API --- .document | 2 ++ lib/mogilefs/mogilefs.rb | 32 ++++++++++++++------ lib/mogilefs/new_file.rb | 63 +++++++++++++++++++++++++++++++++++++-- lib/mogilefs/new_file/writer.rb | 16 +++++++++- test/test_mogilefs_integration.rb | 6 ++++ 5 files changed, 107 insertions(+), 12 deletions(-) diff --git a/.document b/.document index 91c2780..db7b04c 100644 --- a/.document +++ b/.document @@ -10,3 +10,5 @@ lib/mogilefs/admin.rb lib/mogilefs/backend.rb lib/mogilefs/client.rb lib/mogilefs/mogilefs.rb +lib/mogilefs/new_file.rb +lib/mogilefs/new_file/writer.rb diff --git a/lib/mogilefs/mogilefs.rb b/lib/mogilefs/mogilefs.rb index b192605..8a35168 100644 --- a/lib/mogilefs/mogilefs.rb +++ b/lib/mogilefs/mogilefs.rb @@ -125,16 +125,30 @@ class MogileFS::MogileFS < MogileFS::Client get_paths(key, *args).map! { |path| URI.parse(path) } end - # Creates a new file +key+ in +klass+. +bytes+ is currently unused. - # Consider using store_file instead of this method for large files. - # This requires a block passed to it and operates like File.open. - # This atomically replaces existing data stored as +key+ when + # Creates a new file +key+ in the domain of this object. + # + # +bytes+ is the expected size of the file if known in advance + # + # It operates like File.open(..., "w") and may take an optional + # block, yielding an IO-like object with support for the methods + # documented in MogileFS::NewFile::Writer. + # + # This atomically replaces existing data stored as +key+ + # when the block exits or when the returned object is closed. # # +args+ may contain the following options: - # * :content_length => Integer - # * :largefile => [ :stream, :content_range, :tempfile ] - # (see # MogileFS::NewFile) - # * :content_md5 => String, Proc, or :trailer + # + # [:content_length => Integer] + # + # This has the same effect as the (deprecated) +bytes+ parameter. + # + # [ :largefile => :stream, :content_range or :tempfile ] + # + # See MogileFS::NewFile for more information on this + # + # [ :class => String] + # + # The MogileFS storage class of the object. def new_file(key, args = nil, bytes = nil) # :yields: file raise MogileFS::ReadOnlyError if readonly? opts = { :key => key, :multi_dest => 1 } @@ -167,7 +181,7 @@ class MogileFS::MogileFS < MogileFS::Client case (dests[0][1] rescue nil) when %r{\Ahttp://} - http_file = MogileFS::NewFile[opts[:largefile]].new(dests, opts) + http_file = MogileFS::NewFile.new(dests, opts) if block_given? yield http_file return http_file.commit # calls create_close diff --git a/lib/mogilefs/new_file.rb b/lib/mogilefs/new_file.rb index e99a023..baac6e2 100644 --- a/lib/mogilefs/new_file.rb +++ b/lib/mogilefs/new_file.rb @@ -1,8 +1,67 @@ # -*- encoding: binary -*- +# +# The MogileFS::MogileFS#new_file method is enhanced in v3.1.0+ +# to support the :largefile parameter. While we have always +# supported large files via the "store_file" method, streaming +# large amounts of content of an unknown length required the use +# of awkward APIs. +# +# It is possible to stream large content of known length any WebDAV server. +# One example of this is for mirroring a file from an existing HTTP server without download downloading + +# +# uri = URI('http://example.com/large_file') +# Net::HTTP.start(uri.host, uri.port) do |http| +# req = Net::HTTP::Get.new(uri.request_uri) +# +# http.request(req) do |response| +# if len = response.content_length +# io = mg.new_file('key', :largefile => true, :content_length => len) +# else +# warn "trying to upload with Transfer-Encoding: chunked" +# warn "this is not supported by all WebDAV servers" +# io = mg.new_file('key', :largefile => :chunked) +# end +# response.read_body { |buf| io.write(buf) } +# io.close +# end +# end + +# nf = mg.new_file("key", :largefile => stream, :content_length => ) +# nf.write(buf) +# nf.close +# +# If your WebDAV servers have chunked PUT support (e.g. Perlbal), you can +# stream a file of unknown length using "Transfer-Encoding: chunked". +# +# nf = mg.new_file("key", :largefile => :stream) +# nf.write "hello" +# nf.write ... +# nf.close +# +# If your WebDAV server has partial PUT support (e.g Apache), you can +# you can use multiple PUT requests with "Content-Range" support. +# This method is slower than Transfer-Encoding: chunked. +# +# nf = mg.new_file("key", :largefile => :content_range) +# nf.write "hello" +# nf.write ... +# nf.close +# +# Finally, if your WebDAV servers does not support either partial nor +# nor chunked PUTs, you must buffer a file of unknown length using a +# Tempfile: +# +# nf = mg.new_file("key", :largefile => :tempfile) +# nf.write "hello" +# nf.close +# module MogileFS::NewFile # avoiding autoload for new code since it's going away in Ruby... - def self.[](largefile) # :nodoc: + def self.new(dests, opts) # :nodoc: + largefile = opts[:largefile] + largefile = :stream if largefile && opts[:content_length] require "mogilefs/new_file/#{largefile}" if Symbol === largefile case largefile when nil, false @@ -15,7 +74,7 @@ module MogileFS::NewFile Tempfile else raise ArgumentError, "largefile: #{largefile.inspect} not understood" - end + end.new(dests, opts) end end diff --git a/lib/mogilefs/new_file/writer.rb b/lib/mogilefs/new_file/writer.rb index 994545b..d58a4e6 100644 --- a/lib/mogilefs/new_file/writer.rb +++ b/lib/mogilefs/new_file/writer.rb @@ -1,7 +1,11 @@ # -*- encoding: binary -*- -# here are internal implementation details, do not use them in your code # +# All objects yielded or returned by MogileFS::MogileFS#new_file should +# conform to this interface (based on existing IO methods). These objects +# should be considered write-only. module MogileFS::NewFile::Writer + + # see IO#puts def puts(*args) args.each do |obj| write(obj) @@ -10,11 +14,13 @@ module MogileFS::NewFile::Writer nil end + # see IO#putc def putc(ch) write(ch.respond_to?(:chr) ? ch.chr : ch[0]) ch end + # see IO#print def print(*args) args = [ $_ ] unless args[0] write(args.shift) @@ -26,16 +32,24 @@ module MogileFS::NewFile::Writer nil end + # see IO#printf def printf(*args) write(sprintf(*args)) nil end + # see IO#<< def <<(str) write(str) self end + # This will issue the +create_close+ command to the MogileFS tracker + # and finalize the creation of a new file. This returns +nil+ on + # success and will raise IOError if called twice. For non-streaming + # implementations, this will initiate and finalize the upload. + # + # see IO#close def close commit nil diff --git a/test/test_mogilefs_integration.rb b/test/test_mogilefs_integration.rb index b6a181a..2ce8159 100644 --- a/test/test_mogilefs_integration.rb +++ b/test/test_mogilefs_integration.rb @@ -190,6 +190,12 @@ class TestMogileFSIntegration < TestMogIntegration assert_nothing_raised { rv.write "GOOD" } assert_raises(MogileFS::SizeMismatchError) { rv.close } assert_equal "HELLO", @client.get_file_data("a") + + rv = @client.new_file("large", :content_length => 6, :largefile => true) + assert_instance_of MogileFS::NewFile::Stream, rv + assert_equal 6, rv.write("HIHIHI") + assert_nil rv.close + assert_equal "HIHIHI", @client.get_file_data("large") end def test_new_file_content_md5 -- cgit v1.2.3-24-ge0c7