From d6b2624ed1f4502d499dad93ee44a716b5c56f08 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Sat, 5 Nov 2011 01:59:27 +0000 Subject: rename httpfile => http_file Should be easier to read this way --- Manifest.txt | 2 +- lib/mogilefs.rb | 2 +- lib/mogilefs/http_file.rb | 108 ++++++++++++++++++++++++++++++++++++++++++++++ lib/mogilefs/httpfile.rb | 108 ---------------------------------------------- lib/mogilefs/mogilefs.rb | 10 ++--- 5 files changed, 115 insertions(+), 115 deletions(-) create mode 100644 lib/mogilefs/http_file.rb delete mode 100644 lib/mogilefs/httpfile.rb diff --git a/Manifest.txt b/Manifest.txt index 2426585..2413ce7 100644 --- a/Manifest.txt +++ b/Manifest.txt @@ -11,7 +11,7 @@ lib/mogilefs/admin.rb lib/mogilefs/backend.rb lib/mogilefs/bigfile.rb lib/mogilefs/client.rb -lib/mogilefs/httpfile.rb +lib/mogilefs/http_file.rb lib/mogilefs/mogilefs.rb lib/mogilefs/mysql.rb lib/mogilefs/pool.rb diff --git a/lib/mogilefs.rb b/lib/mogilefs.rb index 02075bb..00f3ab7 100644 --- a/lib/mogilefs.rb +++ b/lib/mogilefs.rb @@ -33,7 +33,7 @@ module MogileFS end require 'mogilefs/backend' -require 'mogilefs/httpfile' +require 'mogilefs/http_file' require 'mogilefs/client' require 'mogilefs/bigfile' require 'mogilefs/mogilefs' diff --git a/lib/mogilefs/http_file.rb b/lib/mogilefs/http_file.rb new file mode 100644 index 0000000..22692e7 --- /dev/null +++ b/lib/mogilefs/http_file.rb @@ -0,0 +1,108 @@ +# -*- encoding: binary -*- +# here are internal implementation details, do not use them in your code +require 'stringio' +require 'uri' +require 'mogilefs/backend' +require 'mogilefs/util' + +## +# HTTPFile wraps up the new file operations for storing files onto an HTTP +# storage node. +# +# You really don't want to create an HTTPFile by hand. Instead you want to +# create a new file using MogileFS::MogileFS.new_file. +# +class MogileFS::HTTPFile < StringIO + include MogileFS::Util + + class EmptyResponseError < MogileFS::Error; end + class BadResponseError < MogileFS::Error; end + class UnparseableResponseError < MogileFS::Error; end + class NoStorageNodesError < MogileFS::Error + def message; 'Unable to open socket to storage node'; end + end + + ## + # The URI this file will be stored to. + + attr_reader :uri + + attr_reader :devid + + ## + # The big_io name in case we have file > 256M + + attr_accessor :big_io + + attr_accessor :streaming_io + + ## + # Creates a new HTTPFile with MogileFS-specific data. Use + # MogileFS::MogileFS#new_file instead of this method. + + def initialize(dests, content_length) + super "" + @streaming_io = @big_io = @uri = @devid = nil + @dests = dests + @tried = {} + end + + ## + # Writes an HTTP PUT request to +sock+ to upload the file and + # returns file size if the socket finished writing + def upload(devid, uri) # :nodoc: + file_size = length + sock = MogileFS::Socket.tcp(uri.host, uri.port) + + if @streaming_io + file_size = @streaming_io.length + sock.write("PUT #{uri.request_uri} HTTP/1.0\r\n" \ + "Content-Length: #{file_size}\r\n\r\n") + @streaming_io.call(Proc.new do |data_to_write| + sock.write(data_to_write) + end) + elsif @big_io + # Don't try to run out of memory + File.open(@big_io, "rb") do |fp| + file_size = fp.stat.size + sock.write("PUT #{uri.request_uri} HTTP/1.0\r\n" \ + "Content-Length: #{file_size}\r\n\r\n") + copy_stream(fp, sock) + end + else + sock.write("PUT #{uri.request_uri} HTTP/1.0\r\n" \ + "Content-Length: #{length}\r\n\r\n#{string}") + end + + case line = sock.timed_read(23, "") + when %r{^HTTP/\d\.\d\s+(2\d\d)\s} # success! + file_size + when nil + raise EmptyResponseError, 'Unable to read response line from server' + when %r{^HTTP/\d\.\d\s+(\d+)} + raise BadResponseError, "HTTP response status from upload: #$1" + else + raise UnparseableResponseError, "Response line not understood: #{line}" + end + ensure + sock.close if sock && ! sock.closed? + end + + def commit + errors = nil + @dests.each do |devid, path| + begin + uri = URI.parse(path) + bytes_uploaded = upload(devid, uri) + @devid, @uri = devid, uri + return bytes_uploaded + rescue => e + errors ||= [] + errors << "#{path} failed with #{e.message} (#{e.class})" + end + end + + raise NoStorageNodesError, + "all paths failed with PUT: #{errors.join(', ')}", [] + end +end diff --git a/lib/mogilefs/httpfile.rb b/lib/mogilefs/httpfile.rb deleted file mode 100644 index 22692e7..0000000 --- a/lib/mogilefs/httpfile.rb +++ /dev/null @@ -1,108 +0,0 @@ -# -*- encoding: binary -*- -# here are internal implementation details, do not use them in your code -require 'stringio' -require 'uri' -require 'mogilefs/backend' -require 'mogilefs/util' - -## -# HTTPFile wraps up the new file operations for storing files onto an HTTP -# storage node. -# -# You really don't want to create an HTTPFile by hand. Instead you want to -# create a new file using MogileFS::MogileFS.new_file. -# -class MogileFS::HTTPFile < StringIO - include MogileFS::Util - - class EmptyResponseError < MogileFS::Error; end - class BadResponseError < MogileFS::Error; end - class UnparseableResponseError < MogileFS::Error; end - class NoStorageNodesError < MogileFS::Error - def message; 'Unable to open socket to storage node'; end - end - - ## - # The URI this file will be stored to. - - attr_reader :uri - - attr_reader :devid - - ## - # The big_io name in case we have file > 256M - - attr_accessor :big_io - - attr_accessor :streaming_io - - ## - # Creates a new HTTPFile with MogileFS-specific data. Use - # MogileFS::MogileFS#new_file instead of this method. - - def initialize(dests, content_length) - super "" - @streaming_io = @big_io = @uri = @devid = nil - @dests = dests - @tried = {} - end - - ## - # Writes an HTTP PUT request to +sock+ to upload the file and - # returns file size if the socket finished writing - def upload(devid, uri) # :nodoc: - file_size = length - sock = MogileFS::Socket.tcp(uri.host, uri.port) - - if @streaming_io - file_size = @streaming_io.length - sock.write("PUT #{uri.request_uri} HTTP/1.0\r\n" \ - "Content-Length: #{file_size}\r\n\r\n") - @streaming_io.call(Proc.new do |data_to_write| - sock.write(data_to_write) - end) - elsif @big_io - # Don't try to run out of memory - File.open(@big_io, "rb") do |fp| - file_size = fp.stat.size - sock.write("PUT #{uri.request_uri} HTTP/1.0\r\n" \ - "Content-Length: #{file_size}\r\n\r\n") - copy_stream(fp, sock) - end - else - sock.write("PUT #{uri.request_uri} HTTP/1.0\r\n" \ - "Content-Length: #{length}\r\n\r\n#{string}") - end - - case line = sock.timed_read(23, "") - when %r{^HTTP/\d\.\d\s+(2\d\d)\s} # success! - file_size - when nil - raise EmptyResponseError, 'Unable to read response line from server' - when %r{^HTTP/\d\.\d\s+(\d+)} - raise BadResponseError, "HTTP response status from upload: #$1" - else - raise UnparseableResponseError, "Response line not understood: #{line}" - end - ensure - sock.close if sock && ! sock.closed? - end - - def commit - errors = nil - @dests.each do |devid, path| - begin - uri = URI.parse(path) - bytes_uploaded = upload(devid, uri) - @devid, @uri = devid, uri - return bytes_uploaded - rescue => e - errors ||= [] - errors << "#{path} failed with #{e.message} (#{e.class})" - end - end - - raise NoStorageNodesError, - "all paths failed with PUT: #{errors.join(', ')}", [] - end -end diff --git a/lib/mogilefs/mogilefs.rb b/lib/mogilefs/mogilefs.rb index 267c896..d17d6ec 100644 --- a/lib/mogilefs/mogilefs.rb +++ b/lib/mogilefs/mogilefs.rb @@ -111,14 +111,14 @@ class MogileFS::MogileFS < MogileFS::Client when nil, '' then raise MogileFS::EmptyPathError when /^http:\/\// then - httpfile = MogileFS::HTTPFile.new(dests, bytes) - yield httpfile - rv = httpfile.commit + http_file = MogileFS::HTTPFile.new(dests, bytes) + yield http_file + rv = http_file.commit @backend.create_close(:fid => res['fid'], - :devid => httpfile.devid, + :devid => http_file.devid, :domain => @domain, :key => key, - :path => httpfile.uri.to_s, + :path => http_file.uri.to_s, :size => rv) rv else -- cgit v1.2.3-24-ge0c7