Branching mogilefs-client to version 1.2.1 v1.2.1

From p4 revision #3338

git-svn-id: http://seattlerb.rubyforge.org/svn/mogilefs-client/1.2.1@378 d2e05cf2-00e0-46e5-a3de-bbee4d6b9404

19 files changed, 2019 insertions, 0 deletions

diff --git a/History.txt b/History.txt
new file mode 100644
index 0000000..930c062
--- /dev/null
+++ b/History.txt
@@ -0,0 +1,11 @@
+= 1.2.1
+
+* Switched to Hoe.
+* Moved to p4.
+* Fixed bug #7273 in HTTP mode of client where data would not get
+  returned.  Submitted by Matthew Willson.
+
+= 1.2.0
+
+* Changes lost to time.
+
diff --git a/LICENSE.txt b/LICENSE.txt
new file mode 100644
index 0000000..31e4c06
--- /dev/null
+++ b/LICENSE.txt
@@ -0,0 +1,27 @@
+Copyright 2005 Eric Hodel, The Robot Co-op.  All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+3. Neither the names of the authors nor the names of their contributors
+   may be used to endorse or promote products derived from this software
+   without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS
+OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
diff --git a/Manifest.txt b/Manifest.txt
new file mode 100644
index 0000000..00300f0
--- /dev/null
+++ b/Manifest.txt
@@ -0,0 +1,19 @@
+History.txt
+LICENSE.txt
+Manifest.txt
+README.txt
+Rakefile
+lib/mogilefs.rb
+lib/mogilefs/admin.rb
+lib/mogilefs/backend.rb
+lib/mogilefs/client.rb
+lib/mogilefs/httpfile.rb
+lib/mogilefs/mogilefs.rb
+lib/mogilefs/nfsfile.rb
+lib/mogilefs/pool.rb
+test/setup.rb
+test/test_admin.rb
+test/test_backend.rb
+test/test_client.rb
+test/test_mogilefs.rb
+test/test_pool.rb
diff --git a/README.txt b/README.txt
new file mode 100644
index 0000000..a96b399
--- /dev/null
+++ b/README.txt
@@ -0,0 +1,66 @@
+= mogilefs-client
+
+A Ruby MogileFS client
+
+Rubyforge Project:
+
+http://rubyforge.org/projects/seattlerb/
+
+Documentation:
+
+http://seattlerb.org/mogilefs-client
+
+File bugs:
+
+http://rubyforge.org/tracker/?func=add&group_id=1513&atid=5921
+
+== About
+
+A Ruby MogileFS client.  MogileFS is a distributed filesystem written
+by Danga Interactive.  This client supports NFS and HTTP modes.
+
+For information on MogileFS see:
+
+http://danga.com/mogilefs/
+
+== Installing mogilefs-client
+
+First you need a MogileFS setup.  You can find information on how to do that at the above URL.
+
+Then install the gem:
+
+  $ sudo gem install mogilefs-client
+
+== Using mogilefs-client
+
+  # Create a new instance that will communicate with these trackers:
+  hosts = %w[192.168.1.69:6001 192.168.1.70:6001]
+  mg = MogileFS::MogileFS.new(:domain => 'test', :hosts => hosts
+                              :root => '/mnt/mogilefs')
+  
+  # Stores "A bunch of text to store" into 'some_key' with a class of 'text'.
+  mg.store_content 'some_key', 'text', "A bunch of text to store"
+  
+  # Retrieve data from 'some_key'
+  data = mg.get_file_data 'some_key'
+  
+  # Store the contents of 'image.jpeg' into the key 'my_image' with a class of
+  # 'image'.
+  mg.store_file 'my_image', 'image', 'image.jpeg'
+  
+  # Store the contents of 'image.jpeg' into the key 'my_image' with a class of
+  # 'image' using an open IO.
+  File.open 'image.jpeg' do |fp|
+    mg.store_file 'my_image', 'image', fp
+  end
+  
+  # Remove the key 'my_image' and 'some_key'.
+  mg.delete 'my_image'
+  mg.delete 'some_key'
+
+== WARNING!
+
+This client is only known to work in NFS mode.  HTTP mode is implemented but
+only lightly tested in production environments.  If you find a bug,
+please report it on the Rubyforge project.
+
diff --git a/Rakefile b/Rakefile
new file mode 100644
index 0000000..9ac5f00
--- /dev/null
+++ b/Rakefile
@@ -0,0 +1,18 @@
+require 'rubygems'
+require 'hoe'
+
+$:.unshift 'lib'
+require 'mogilefs'
+
+Hoe.new 'mogilefs-client', MogileFS::VERSION do |p|
+  p.rubyforge_name = 'seattlerb'
+  p.author = 'Eric Hodel'
+  p.email = 'drbrain@segment7.net'
+  p.summary = p.paragraphs_of('README.txt', 1).first
+  p.description = p.paragraphs_of('README.txt', 9).first
+  p.url = p.paragraphs_of('README.txt', 5).first
+  p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")
+end
+
+# vim: syntax=Ruby
+
diff --git a/lib/mogilefs.rb b/lib/mogilefs.rb
new file mode 100644
index 0000000..8bf3fb6
--- /dev/null
+++ b/lib/mogilefs.rb
@@ -0,0 +1,26 @@
+##
+# MogileFS is a Ruby client for Danga Interactive's open source distributed
+# filesystem.
+#
+# To read more about Danga's MogileFS: http://danga.com/mogilefs/
+
+module MogileFS
+
+  VERSION = '1.2.1'
+
+  ##
+  # Raised when a socket remains unreadable for too long.
+
+  class UnreadableSocketError < RuntimeError; end
+
+end
+
+require 'socket'
+
+require 'mogilefs/backend'
+require 'mogilefs/nfsfile'
+require 'mogilefs/httpfile'
+require 'mogilefs/client'
+require 'mogilefs/mogilefs'
+require 'mogilefs/admin'
+
diff --git a/lib/mogilefs/admin.rb b/lib/mogilefs/admin.rb
new file mode 100644
index 0000000..c03b394
--- /dev/null
+++ b/lib/mogilefs/admin.rb
@@ -0,0 +1,298 @@
+require 'mogilefs/client'
+
+##
+# A MogileFS Administration Client
+
+class MogileFS::Admin < MogileFS::Client
+
+  ##
+  # Enumerates fids using #list_fids.
+
+  def each_fid
+    low = 0
+    high = nil
+
+    max = get_stats('fids')['fids']['max']
+
+    0.step max, 100 do |high|
+      fids = list_fids low, high
+      fids.each { |fid| yield fid }
+      low = high + 1
+    end
+  end
+
+  ##
+  # Returns an Array of host status Hashes.  If +hostid+ is given only that
+  # host is returned.
+  #
+  #   admin.get_hosts 1
+  #
+  # Returns:
+  #
+  #   [{"status"=>"alive",
+  #     "http_get_port"=>"",
+  #     "http_port"=>"",
+  #     "hostid"=>"1",
+  #     "hostip"=>"",
+  #     "hostname"=>"rur-1",
+  #     "remoteroot"=>"/mnt/mogilefs/rur-1",
+  #     "altip"=>"",
+  #     "altmask"=>""}]
+
+  def get_hosts(hostid = nil)
+    args = hostid ? { :hostid => hostid } : {}
+    res = @backend.get_hosts args
+    return clean('hosts', 'host', res)
+  end
+
+  ##
+  # Returns an Array of device status Hashes.  If devid is given only that
+  # device is returned.
+  #
+  #   admin.get_devices 1
+  #
+  # Returns:
+  #
+  #   [{"status"=>"alive",
+  #     "mb_asof"=>"",
+  #     "mb_free"=>"0",
+  #     "devid"=>"1",
+  #     "hostid"=>"1",
+  #     "mb_used"=>"",
+  #     "mb_total"=>""}]
+
+  def get_devices(devid = nil)
+    args = devid ? { :devid => devid } : {}
+    res = @backend.get_devices args
+    return clean('devices', 'dev', res)
+  end
+
+  ##
+  # Returns an Array of fid Hashes from +from_fid+ to +to_fid+.
+  #
+  #   admin.list_fids 0, 100
+  #
+  # Returns:
+  #
+  #   [{"fid"=>"99",
+  #     "class"=>"normal",
+  #     "domain"=>"test",
+  #     "devcount"=>"2",
+  #     "length"=>"4",
+  #     "key"=>"file_key"},
+  #    {"fid"=>"82",
+  #     "class"=>"normal",
+  #     "devcount"=>"2",
+  #     "domain"=>"test",
+  #     "length"=>"9",
+  #     "key"=>"new_new_key"}]
+
+  def list_fids(from_fid, to_fid)
+    res = @backend.list_fids :from => from_fid, :to => to_fid
+    return clean('fid_count', 'fid_', res)
+  end
+
+  ##
+  # Returns a statistics structure representing the state of mogilefs.
+  #
+  #   admin.get_stats
+  #
+  # Returns:
+  #
+  #   {"fids"=>{"max"=>"99", "count"=>"2"},
+  #    "device"=>
+  #     [{"status"=>"alive", "files"=>"2", "id"=>"1", "host"=>"rur-1"},
+  #      {"status"=>"alive", "files"=>"2", "id"=>"2", "host"=>"rur-2"}],
+  #    "replication"=>
+  #     [{"files"=>"2", "class"=>"normal", "devcount"=>"2", "domain"=>"test"}],
+  #    "file"=>[{"files"=>"2", "class"=>"normal", "domain"=>"test"}]}
+
+  def get_stats(type = 'all')
+    res = @backend.stats type => 1
+    stats = {}
+
+    stats['device'] = clean 'devicescount', 'devices', res, false
+    stats['file'] = clean 'filescount', 'files', res, false
+    stats['replication'] = clean 'replicationcount', 'replication', res, false
+
+    if res['fidmax'] or res['fidcount'] then
+      stats['fids'] = {
+        'max' => res['fidmax'].to_i,
+        'count' => res['fidcount'].to_i
+      }
+    end
+
+    stats.delete 'device' if stats['device'].empty?
+    stats.delete 'file' if stats['file'].empty?
+    stats.delete 'replication' if stats['replication'].empty?
+
+    return stats
+  end
+
+  ##
+  # Returns the domains present in the mogilefs.
+  #
+  #   admin.get_domains
+  #
+  # Returns:
+  #
+  #   {"test"=>{"normal"=>3, "default"=>2}}
+
+  def get_domains
+    res = @backend.get_domains
+
+    domains = {}
+    (1..res['domains'].to_i).each do |i|
+      domain = clean "domain#{i}classes", "domain#{i}class", res, false
+      domain = domain.map { |d| [d.values.first, d.values.last.to_i] }
+      domains[res["domain#{i}"]] = Hash[*domain.flatten]
+    end
+
+    return domains
+  end
+
+  ##
+  # Creates a new domain named +domain+.  Returns nil if creation failed.
+
+  def create_domain(domain)
+    raise 'readonly mogilefs' if readonly?
+    res = @backend.create_domain :domain => domain
+    return res['domain'] unless res.nil?
+  end
+
+  ##
+  # Deletes +domain+.  Returns true if successful, false if not.
+
+  def delete_domain(domain)
+    raise 'readonly mogilefs' if readonly?
+    res = @backend.delete_domain :domain => domain
+    return !res.nil?
+  end
+
+  ##
+  # Creates a new class in +domain+ named +klass+ with files replicated to
+  # +mindevcount+ devices.  Returns nil on failure.
+
+  def create_class(domain, klass, mindevcount)
+    return modify_class(domain, klass, mindevcount, :create)
+  end
+
+  ##
+  # Updates class +klass+ in +domain+ to be replicated to +mindevcount+
+  # devices.  Returns nil on failure.
+
+  def update_class(domain, klass, mindevcount)
+    return modify_class(domain, klass, mindevcount, :update)
+  end
+
+  ##
+  # Removes class +klass+ from +domain+.  Returns true if successful, false if
+  # not.
+
+  def delete_class(domain, klass)
+    res = @backend.delete_class :domain => domain, :class => klass
+    return !res.nil?
+  end
+
+  ##
+  # Creates a new host named +host+.  +args+ must contain :ip and :port.
+  # Returns true if successful, false if not.
+
+  def create_host(host, args = {})
+    raise ArgumentError, "Must specify ip and port" unless \
+      args.include? :ip and args.include? :port
+
+    return modify_host(host, args, 'create')
+  end
+
+  ##
+  # Updates +host+ with +args+.  Returns true if successful, false if not.
+
+  def update_host(host, args = {})
+    return modify_host(host, args, 'update')
+  end
+
+  ##
+  # Deletes host +host+.  Returns nil on failure.
+
+  def delete_host(host)
+    raise 'readonly mogilefs' if readonly?
+    res = @backend.delete_host :host => host
+    return !res.nil?
+  end
+
+  ##
+  # Changes the device status of +device+ on +host+ to +state+ which can be
+  # 'alive', 'down', or 'dead'.
+
+  def change_device_state(host, device, state)
+    raise 'readonly mogilefs' if readonly?
+    res = @backend.set_state :host => host, :device => device, :state => state
+    return !res.nil?
+  end
+
+  protected unless defined? $TESTING
+
+  ##
+  # Modifies +klass+ on +domain+ to store files on +mindevcount+ devices via
+  # +action+.  Returns the class name if successful, nil if not.
+
+  def modify_class(domain, klass, mindevcount, action)
+    raise 'readonly mogilefs' if readonly?
+    res = @backend.send("#{action}_class", :domain => domain, :class => klass,
+                                          :mindevcount => mindevcount)
+
+    return res['class'] unless res.nil?
+  end
+
+  ##
+  # Modifies +host+ using +args+ via +action+.  Returns true if successful,
+  # false if not.
+
+  def modify_host(host, args = {}, action = 'create')
+    args[:host] = host
+    res = @backend.send "#{action}_host", args
+    return !res.nil?
+  end
+
+  ##
+  # Turns the response +res+ from the backend into an Array of Hashes from 1
+  # to res[+count+].  If +underscore+ is true then a '_' character is assumed
+  # between the prefix and the hash key value.
+  #
+  #   res = {"host1_remoteroot"=>"/mnt/mogilefs/rur-1",
+  #          "host1_hostname"=>"rur-1",
+  #          "host1_hostid"=>"1",
+  #          "host1_http_get_port"=>"",
+  #          "host1_altip"=>"",
+  #          "hosts"=>"1",
+  #          "host1_hostip"=>"",
+  #          "host1_http_port"=>"",
+  #          "host1_status"=>"alive",
+  #          "host1_altmask"=>""}
+  #   admin.clean 'hosts', 'host', res
+  # 
+  # Returns:
+  # 
+  #   [{"status"=>"alive",
+  #     "http_get_port"=>"",
+  #     "http_port"=>"",
+  #     "hostid"=>"1",
+  #     "hostip"=>"",
+  #     "hostname"=>"rur-1",
+  #     "remoteroot"=>"/mnt/mogilefs/rur-1",
+  #     "altip"=>"",
+  #     "altmask"=>""}]
+
+  def clean(count, prefix, res, underscore = true)
+    underscore = underscore ? '_' : ''
+    return (1..res[count].to_i).map do |i|
+      dev = res.select { |k,_| k =~ /^#{prefix}#{i}#{underscore}/ }.map do |k,v|
+        [k.sub(/^#{prefix}#{i}#{underscore}/, ''), v]
+      end
+      Hash[*dev.flatten]
+    end
+  end
+
+end
+
diff --git a/lib/mogilefs/backend.rb b/lib/mogilefs/backend.rb
new file mode 100644
index 0000000..04708e9
--- /dev/null
+++ b/lib/mogilefs/backend.rb
@@ -0,0 +1,222 @@
+require 'socket'
+require 'thread'
+require 'mogilefs'
+
+##
+# MogileFS::Backend communicates with the MogileFS trackers.
+
+class MogileFS::Backend
+
+  ##
+  # Adds MogileFS commands +names+.
+
+  def self.add_command(*names)
+    names.each do |name|
+      define_method name do |*args|
+        do_request name, args.first || {}
+      end
+    end
+  end
+
+  ##
+  # The last error
+  #--
+  # TODO Use Exceptions
+
+  attr_reader :lasterr
+
+  ##
+  # The string attached to the last error
+  #--
+  # TODO Use Exceptions
+
+  attr_reader :lasterrstr
+
+  ##
+  # Creates a new MogileFS::Backend.
+  #
+  # :hosts is a required argument and must be an Array containing one or more
+  # 'hostname:port' pairs as Strings.
+  #
+  # :timeout adjusts the request timeout before an error is returned.
+
+  def initialize(args)
+    @hosts = args[:hosts]
+    raise ArgumentError, "must specify at least one host" unless @hosts
+    raise ArgumentError, "must specify at least one host" if @hosts.empty?
+    unless @hosts == @hosts.select { |h| h =~ /:\d+$/ } then
+      raise ArgumentError, ":hosts must be in 'host:port' form"
+    end
+
+    @mutex = Mutex.new
+    @timeout = args[:timeout] || 3
+    @socket = nil
+    @lasterr = nil
+    @lasterrstr = nil
+
+    @dead = {}
+  end
+
+  ##
+  # Closes this backend's socket.
+
+  def shutdown
+    @socket.close unless @socket.nil? or @socket.closed?
+    @socket = nil
+  end
+
+  # MogileFS::MogileFS commands
+
+  add_command :create_open
+  add_command :create_close
+  add_command :get_paths
+  add_command :delete
+  add_command :sleep
+  add_command :rename
+  add_command :list_keys
+
+  # MogileFS::Backend commands
+  
+  add_command :get_hosts
+  add_command :get_devices
+  add_command :list_fids
+  add_command :stats
+  add_command :get_domains
+  add_command :create_domain
+  add_command :delete_domain
+  add_command :create_class
+  add_command :update_class
+  add_command :delete_class
+  add_command :create_host
+  add_command :update_host
+  add_command :delete_host
+  add_command :set_state
+
+  private unless defined? $TESTING
+
+  ##
+  # Returns a new TCPSocket connected to +port+ on +host+.
+
+  def connect_to(host, port)
+    return TCPSocket.new(host, port)
+  end
+
+  ##
+  # Performs the +cmd+ request with +args+.
+
+  def do_request(cmd, args)
+    @mutex.synchronize do
+      request = make_request cmd, args
+
+      begin
+        bytes_sent = socket.send request, 0
+      rescue SystemCallError
+        @socket = nil
+        raise "couldn't connect to mogilefsd backend"
+      end
+
+      unless bytes_sent == request.length then
+        raise "request truncated (sent #{bytes_sent} expected #{request.length})"
+      end
+
+      readable?
+
+      return parse_response(socket.gets)
+    end
+  end
+
+  ##
+  # Makes a new request string for +cmd+ and +args+.
+
+  def make_request(cmd, args)
+    return "#{cmd} #{url_encode args}\r\n"
+  end
+
+  ##
+  # Turns the +line+ response from the server into a Hash of options, an
+  # error, or raises, as appropriate.
+
+  def parse_response(line)
+    if line =~ /^ERR\s+(\w+)\s*(.*)/ then
+      @lasterr = $1
+      @lasterrstr = $2 ? url_unescape($2) : nil
+      return nil
+    end
+
+    return url_decode($1) if line =~ /^OK\s+\d*\s*(\S*)/
+
+    raise "Invalid response from server: #{line.inspect}"
+  end
+
+  ##
+  # Raises if the socket does not become readable in +@timeout+ seconds.
+
+  def readable?
+    found = select [socket], nil, nil, @timeout
+    if found.nil? or found.empty? then
+      peer = (@socket ? "#{@socket.peeraddr[3]}:#{@socket.peeraddr[1]} " : nil)
+      raise MogileFS::UnreadableSocketError, "#{peer}never became readable"
+    end
+    return true
+  end
+
+  ##
+  # Returns a socket connected to a MogileFS tracker.
+
+  def socket
+    return @socket if @socket and not @socket.closed?
+
+    now = Time.now
+
+    @hosts.sort_by { rand(3) - 1 }.each do |host|
+      next if @dead.include? host and @dead[host] > now - 5
+
+      begin
+        @socket = connect_to(*host.split(':'))
+      rescue SystemCallError
+        @dead[host] = now
+        next
+      end
+
+      return @socket
+    end
+
+    raise "couldn't connect to mogilefsd backend"
+  end
+
+  ##
+  # Turns a url params string into a Hash.
+
+  def url_decode(str)
+    pairs = str.split('&').map do |pair|
+      pair.split('=', 2).map { |v| url_unescape v }
+    end
+
+    return Hash[*pairs.flatten]
+  end
+
+  ##
+  # Turns a Hash (or Array of pairs) into a url params string.
+
+  def url_encode(params)
+    return params.map do |k,v|
+      "#{url_escape k.to_s}=#{url_escape v.to_s}"
+    end.join("&")
+  end
+
+  ##
+  # Escapes naughty URL characters.
+
+  def url_escape(str)
+    return str.gsub(/([^\w\,\-.\/\\\: ])/) { "%%%02x" % $1[0] }.tr(' ', '+')
+  end
+
+  ##
+  # Unescapes naughty URL characters.
+
+  def url_unescape(str)
+    return str.gsub(/%([a-f0-9][a-f0-9])/i) { [$1.to_i(16)].pack 'C' }.tr('+', ' ')
+  end
+
+end
+
diff --git a/lib/mogilefs/client.rb b/lib/mogilefs/client.rb
new file mode 100644
index 0000000..75d35d2
--- /dev/null
+++ b/lib/mogilefs/client.rb
@@ -0,0 +1,65 @@
+require 'mogilefs/backend'
+
+##
+# MogileFS::Client is the MogileFS client base class.  Concrete clients like
+# MogileFS::MogileFS and MogileFS::Admin are implemented atop this one to do
+# real work.
+
+class MogileFS::Client
+
+  ##
+  # The backend connection for this client
+
+  attr_reader :backend
+
+  attr_accessor :hosts if defined? $TESTING
+
+  ##
+  # Creates a new Client.  See MogileFS::Backend#initialize for how to specify
+  # hosts.  If :readonly is set to true, the client will not modify anything
+  # on the server.
+  #
+  #   MogileFS::Client.new :hosts => ['kaa:6001', 'ziz:6001'], :readonly => true
+
+  def initialize(args)
+    @hosts = args[:hosts]
+    @readonly = args[:readonly] ? true : false
+    @timeout = args[:timeout]
+
+    reload
+  end
+
+  ##
+  # Creates a new MogileFS::Backend.
+
+  def reload
+    @backend = MogileFS::Backend.new :hosts => @hosts, :timeout => @timeout
+  end
+
+  ##
+  # The last error reported by the backend.
+  #--
+  # TODO use Exceptions
+
+  def err
+    @backend.lasterr
+  end
+
+  ##
+  # The last error message reported by the backend.
+  #--
+  # TODO use Exceptions
+
+  def errstr
+    @backend.lasterrstr
+  end
+
+  ##
+  # Is this a read-only client?
+
+  def readonly?
+    return @readonly
+  end
+
+end
+
diff --git a/lib/mogilefs/httpfile.rb b/lib/mogilefs/httpfile.rb
new file mode 100644
index 0000000..13788aa
--- /dev/null
+++ b/lib/mogilefs/httpfile.rb
@@ -0,0 +1,144 @@
+require 'fcntl'
+require 'socket'
+require 'stringio'
+require 'uri'
+require 'mogilefs/backend'
+
+##
+# HTTPFile wraps up the new file operations for storing files onto an HTTP
+# storage node.
+#
+# You really don't want to create an HTTPFile by hand.  Instead you want to
+# create a new file using MogileFS::MogileFS.new_file.
+#
+# WARNING! HTTP mode is completely untested as I cannot make it work on
+# FreeBSD.  Please send patches/tests if you find bugs.
+#--
+# TODO dup'd content in MogileFS::NFSFile
+
+class MogileFS::HTTPFile < StringIO
+
+  ##
+  # The path this file will be stored to.
+
+  attr_reader :path
+
+  ##
+  # The key for this file.  This key won't represent a real file until you've
+  # called #close.
+
+  attr_reader :key
+
+  ##
+  # The class of this file.
+
+  attr_reader :class
+
+  ##
+  # Works like File.open.  Use MogileFS::MogileFS#new_file instead of this
+  # method.
+
+  def self.open(*args)
+    fp = new(*args)
+
+    return fp unless block_given?
+
+    begin
+      yield fp
+    ensure
+      fp.close
+    end
+  end
+
+  ##
+  # Creates a new HTTPFile with MogileFS-specific data.  Use
+  # MogileFS::MogileFS#new_file instead of this method.
+
+  def initialize(mg, fid, path, devid, klass, key, dests, content_length)
+    super ''
+    @mg = mg
+    @fid = fid
+    @path = path
+    @devid = devid
+    @klass = klass
+    @key = key
+
+    @dests = dests.map { |(_,u)| URI.parse u }
+    @tried = {}
+
+    @socket = nil
+  end
+
+  ##
+  # Closes the file handle and marks it as closed in MogileFS.
+
+  def close
+    connect_socket
+
+    @socket.write "PUT #{@path.request_uri} HTTP/1.0\r\nContent-length: #{length}\r\n\r\n#{string}"
+
+    if connected? then
+      line = @socket.gets
+      raise 'Unable to read response line from server' if line.nil?
+
+      if line =~ %r%^HTTP/\d+\.\d+\s+(\d+)% then
+        status = Integer $1
+        case status
+        when 200..299 then # success!
+        else
+          found_header = false
+          body = []
+          line = @socket.gets
+          until line.nil? do
+            line.strip
+            found_header = true if line.nil?
+            next unless found_header
+            body << " #{line}"
+          end
+          body = body[0, 512] if body.length > 512
+          raise "HTTP response status from upload: #{body}"
+        end
+      else
+        raise "Response line not understood: #{line}"
+      end
+      @socket.close
+    end
+
+    @mg.backend.create_close(:fid => @fid, :devid => @devid,
+                             :domain => @mg.domain, :key => @key,
+                             :path => @path, :size => length)
+    return nil
+  end
+
+  private
+
+  def connected?
+    return !(@socket.nil? or @socket.closed?)
+  end
+
+  def connect_socket
+    return @socket if connected?
+
+    next_path
+
+    if @path.nil? then
+      @tried.clear
+      next_path
+      raise 'Unable to open socket to storage node' if @path.nil?
+    end
+
+    @socket = TCPSocket.new @path.host, @path.port
+  end
+
+  def next_path
+    @path = nil
+    @dests.each do |dest|
+      unless @tried.include? dest then
+        @path = dest
+        return
+      end
+    end
+  end
+
+end
+
diff --git a/lib/mogilefs/mogilefs.rb b/lib/mogilefs/mogilefs.rb
new file mode 100644
index 0000000..b6776ce
--- /dev/null
+++ b/lib/mogilefs/mogilefs.rb
@@ -0,0 +1,233 @@
+require 'open-uri'
+require 'timeout'
+
+require 'mogilefs/client'
+require 'mogilefs/nfsfile'
+
+##
+# Timeout error class.
+
+class MogileFS::Timeout < Timeout::Error; end
+
+##
+# MogileFS File manipulation client.
+
+class MogileFS::MogileFS < MogileFS::Client
+
+  ##
+  # The path to the local MogileFS mount point if you are using NFS mode.
+
+  attr_reader :root
+
+  ##
+  # The domain of keys for this MogileFS client.
+
+  attr_reader :domain
+
+  ##
+  # Creates a new MogileFS::MogileFS instance.  +args+ must include a key
+  # :domain specifying the domain of this client.  A key :root will be used to
+  # specify the root of the NFS file system.
+
+  def initialize(args = {})
+    @domain = args[:domain]
+    @root = args[:root]
+
+    raise ArgumentError, "you must specify a domain" unless @domain
+
+    super
+  end
+
+  ##
+  # Enumerates keys starting with +key+.
+
+  def each_key(prefix)
+    after = nil
+
+    keys, after = list_keys prefix
+
+    until keys.empty? do
+      keys.each { |k| yield k }
+      keys, after = list_keys prefix, after
+    end
+
+    return nil
+  end
+
+  ##
+  # Retrieves the contents of +key+.
+
+  def get_file_data(key)
+    paths = get_paths key
+
+    return nil unless paths
+
+    paths.each do |path|
+      next unless path
+      case path
+      when /^http:\/\// then
+        begin
+          path = URI.parse path
+          data = timeout(5, MogileFS::Timeout) { path.read }
+          return data
+        rescue MogileFS::Timeout
+          next
+        end
+      else
+        next unless File.exist? path
+        return File.read(path)
+      end
+    end
+
+    return nil
+  end
+
+  ##
+  # Get the paths for +key+.
+
+  def get_paths(key, noverify = true, zone = nil)
+    noverify = noverify ? 1 : 0
+    res = @backend.get_paths(:domain => @domain, :key => key,
+                             :noverify => noverify, :zone => zone)
+    
+    return nil if res.nil? and @backend.lasterr == 'unknown_key'
+    paths = (1..res['paths'].to_i).map { |i| res["path#{i}"] }
+    return paths if paths.empty?
+    return paths if paths.first =~ /^http:\/\//
+    return paths.map { |path| File.join @root, path }
+  end
+
+  ##
+  # Creates a new file +key+ in +klass+.  +bytes+ is currently unused.
+  #
+  # The +block+ operates like File.open.
+
+  def new_file(key, klass, bytes = 0, &block) # :yields: file
+    raise 'readonly mogilefs' if readonly?
+
+    res = @backend.create_open(:domain => @domain, :class => klass,
+                               :key => key, :multi_dest => 1)
+
+    raise "#{@backend.lasterr}: #{@backend.lasterrstr}" if res.nil? # HACK
+
+    dests = nil
+
+    if res.include? 'dev_count' then # HACK HUH?
+      dests = (1..res['dev_count'].to_i).map do |i|
+        [res["devid_#{i}"], res["path_#{i}"]]
+      end
+    else
+      # 0x0040:  d0e4 4f4b 2064 6576 6964 3d31 2666 6964  ..OK.devid=1&fid
+      # 0x0050:  3d33 2670 6174 683d 6874 7470 3a2f 2f31  =3&path=http://1
+      # 0x0060:  3932 2e31 3638 2e31 2e37 323a 3735 3030  92.168.1.72:7500
+      # 0x0070:  2f64 6576 312f 302f 3030 302f 3030 302f  /dev1/0/000/000/
+      # 0x0080:  3030 3030 3030 3030 3033 2e66 6964 0d0a  0000000003.fid..
+
+      dests = [[res['devid'], res['path']]]
+    end
+
+    dest = dests.first
+    devid, path = dest
+
+    case path
+    when /^http:\/\// then
+      MogileFS::HTTPFile.open(self, res['fid'], path, devid, klass, key,
+                              dests, bytes, &block)
+    else
+      MogileFS::NFSFile.open(self, res['fid'], path, devid, klass, key, &block)
+    end
+  end
+
+  ##
+  # Copies the contents of +file+ into +key+ in class +klass+.  +file+ can be
+  # either a file name or an object that responds to #read.
+
+  def store_file(key, klass, file)
+    raise 'readonly mogilefs' if readonly?
+
+    new_file key, klass do |mfp|
+      if file.respond_to? :read then
+        return copy(file, mfp)
+      else
+        return File.open(file) { |fp| copy(fp, mfp) }
+      end
+    end
+  end
+
+  ##
+  # Stores +content+ into +key+ in class +klass+.
+
+  def store_content(key, klass, content)
+    raise 'readonly mogilefs' if readonly?
+
+    new_file key, klass do |mfp|
+      mfp << content
+    end
+
+    return content.length
+  end
+
+  ##
+  # Removes +key+.
+
+  def delete(key)
+    raise 'readonly mogilefs' if readonly?
+
+    res = @backend.delete :domain => @domain, :key => key
+
+    if res.nil? and @backend.lasterr != 'unknown_key' then
+      raise "unable to delete #{key}: #{@backend.lasterr}"
+    end
+  end
+
+  ##
+  # Sleeps +duration+.
+
+  def sleep(duration)
+    @backend.sleep :duration => duration
+  end
+
+  ##
+  # Renames a key +from+ to key +to+.
+
+  def rename(from, to)
+    raise 'readonly mogilefs' if readonly?
+
+    res = @backend.rename :domain => @domain, :from_key => from, :to_key => to
+
+    if res.nil? and @backend.lasterr != 'unknown_key' then
+      raise "unable to rename #{from_key} to #{to_key}: #{@backend.lasterr}"
+    end
+  end
+
+  ##
+  # Lists keys starting with +prefix+ follwing +after+ up to +limit+.  If
+  # +after+ is nil the list starts at the beginning.
+
+  def list_keys(prefix, after = nil, limit = 1000)
+    res = @backend.list_keys(:domain => domain, :prefix => prefix,
+                             :after => after, :limit => limit)
+
+    return nil if res.nil?
+
+    keys = (1..res['key_count'].to_i).map { |i| res["key_#{i}"] }
+
+    return keys, res['next_after']
+  end
+
+  private
+
+  def copy(from, to) # HACK use FileUtils
+    bytes = 0
+
+    until from.eof? do
+      chunk = from.read 8192
+      to.write chunk
+      bytes += chunk.length
+    end
+
+    return bytes
+  end
+
+end
+
diff --git a/lib/mogilefs/nfsfile.rb b/lib/mogilefs/nfsfile.rb
new file mode 100644
index 0000000..d4040c2
--- /dev/null
+++ b/lib/mogilefs/nfsfile.rb
@@ -0,0 +1,81 @@
+require 'mogilefs/backend'
+
+##
+# NFSFile wraps up the new file operations for storing files onto an NFS
+# storage node.
+#
+# You really don't want to create an NFSFile by hand.  Instead you want to
+# create a new file using MogileFS::MogileFS.new_file.
+
+class MogileFS::NFSFile < File
+
+  ##
+  # The path of this file not including the local mount point.
+
+  attr_reader :path
+
+  ##
+  # The key for this file.  This key won't represent a real file until you've
+  # called #close.
+
+  attr_reader :key
+
+  ##
+  # The class of this file.
+
+  attr_reader :class
+
+  class << self
+
+    ##
+    # Wraps up File.new with MogileFS-specific data.  Use
+    # MogileFS::MogileFS#new_file instead of this method.
+
+    def new(mg, fid, path, devid, klass, key)
+      fp = super join(mg.root, path), 'w+'
+      fp.send :setup, mg, fid, path, devid, klass, key
+      return fp
+    end
+
+    ##
+    # Wraps up File.open with MogileFS-specific data.  Use
+    # MogileFS::MogileFS#new_file instead of this method.
+
+    def open(mg, fid, path, devid, klass, key, &block)
+      fp = new mg, fid, path, devid, klass, key
+
+      return fp if block.nil?
+
+      begin
+        yield fp
+      ensure
+        fp.close
+      end
+    end
+
+  end
+
+  ##
+  # Closes the file handle and marks it as closed in MogileFS.
+
+  def close
+    super
+    @mg.backend.create_close(:fid => @fid, :devid => @devid,
+                             :domain => @mg.domain, :key => @key,
+                             :path => @path)
+    return nil
+  end
+
+  private
+
+  def setup(mg, fid, path, devid, klass, key)
+    @mg = mg
+    @fid = fid
+    @path = path
+    @devid = devid
+    @klass = klass
+    @key = key
+  end
+
+end
+
diff --git a/lib/mogilefs/pool.rb b/lib/mogilefs/pool.rb
new file mode 100644
index 0000000..edca70f
--- /dev/null
+++ b/lib/mogilefs/pool.rb
@@ -0,0 +1,50 @@
+require 'thread'
+require 'mogilefs'
+
+class MogileFS::Pool
+
+  class BadObjectError < RuntimeError; end
+
+  def initialize(klass, *args)
+    @args = args
+    @klass = klass
+    @queue = Queue.new
+    @objects = []
+  end
+
+  def get
+    begin
+      object = @queue.pop true
+    rescue ThreadError
+      object = @klass.new(*@args)
+      @objects << object
+    end
+    return object
+  end
+
+  def put(o)
+    raise BadObjectError unless @objects.include? o
+    @queue.push o
+    purge
+  end
+
+  def use
+    object = get
+    yield object
+  ensure
+    put object
+  end
+
+  def purge
+    return if @queue.length < 5
+    begin
+      until @queue.length <= 2 do
+        obj = @queue.pop true
+        @objects.delete obj
+      end
+    rescue ThreadError
+    end
+  end
+
+end
+
diff --git a/test/setup.rb b/test/setup.rb
new file mode 100644
index 0000000..86f0f28
--- /dev/null
+++ b/test/setup.rb
@@ -0,0 +1,54 @@
+require 'test/unit'
+
+$TESTING = true
+
+require 'mogilefs'
+
+class FakeBackend
+
+  attr_reader :lasterr, :lasterrstr
+
+  def initialize
+    @responses = Hash.new { |h,k| h[k] = [] }
+    @lasterr = nil
+    @lasterrstr = nil
+  end
+
+  def method_missing(meth, *args)
+    meth = meth.to_s
+    if meth =~ /(.*)=$/ then
+      @responses[$1] << args.first
+    else
+      response = @responses[meth].shift
+      case response
+      when Array then
+        @lasterr = response.first
+        @lasterrstr = response.last
+        return nil
+      end
+      return response
+    end
+  end
+
+end
+
+class MogileFS::Client
+  attr_writer :readonly
+end
+
+class TestMogileFS < Test::Unit::TestCase
+
+  def setup
+    return if self.class == TestMogileFS
+    @root = '/mogilefs/test'
+    @client = @klass.new :hosts => ['kaa:6001'], :domain => 'test',
+                                  :root => @root
+    @backend = FakeBackend.new
+    @client.instance_variable_set '@backend', @backend
+  end
+
+  def test_nothing
+  end
+
+end
+
diff --git a/test/test_admin.rb b/test/test_admin.rb
new file mode 100644
index 0000000..9227cfe
--- /dev/null
+++ b/test/test_admin.rb
@@ -0,0 +1,174 @@
+require 'test/setup'
+
+class TestMogileFS__Admin < TestMogileFS
+
+  def setup
+    @klass = MogileFS::Admin
+    super
+  end
+
+  def test_clean
+    res = {"host1_remoteroot"=>"/mnt/mogilefs/rur-1",
+           "host1_hostname"=>"rur-1",
+           "host1_hostid"=>"1",
+           "host1_http_get_port"=>"",
+           "host1_altip"=>"",
+           "hosts"=>"1",
+           "host1_hostip"=>"",
+           "host1_http_port"=>"",
+           "host1_status"=>"alive",
+           "host1_altmask"=>""}
+    actual = @client.clean 'hosts', 'host', res
+  
+    expected = [{"status"=>"alive",
+                 "http_get_port"=>"",
+                 "http_port"=>"",
+                 "hostid"=>"1",
+                 "hostip"=>"",
+                 "hostname"=>"rur-1",
+                 "remoteroot"=>"/mnt/mogilefs/rur-1",
+                 "altip"=>"",
+                 "altmask"=>""}]
+
+    assert_equal expected, actual
+  end
+
+  def test_each_fid
+    @backend.stats = {
+      'fidmax' => '182',
+      'fidcount' => '2',
+    }
+
+    @backend.list_fids = {
+      'fid_count' => '1',
+      'fid_1_fid' => '99',
+      'fid_1_class' => 'normal',
+      'fid_1_devcount' => '2',
+      'fid_1_domain' => 'test',
+      'fid_1_key' => 'file_key',
+      'fid_1_length' => '4',
+    }
+
+    @backend.list_fids = {
+      'fid_count' => '1',
+      'fid_1_fid' => '182',
+      'fid_1_class' => 'normal',
+      'fid_1_devcount' => '2',
+      'fid_1_domain' => 'test',
+      'fid_1_key' => 'new_new_key',
+      'fid_1_length' => '9',
+    }
+
+    fids = []
+    @client.each_fid { |fid| fids << fid }
+
+    expected = [
+      { "fid"      => "99",
+        "class"    => "normal",
+        "domain"   => "test",
+        "devcount" => "2",
+        "length"   => "4",
+        "key"      => "file_key" },
+      { "fid"      => "182",
+        "class"    => "normal",
+        "devcount" => "2",
+        "domain"   => "test",
+        "length"   => "9",
+        "key"      => "new_new_key" },
+    ]
+
+    assert_equal expected, fids
+  end
+
+  def test_get_domains
+    @backend.get_domains = {
+      'domains' => 2,
+      'domain1' => 'test',
+      'domain2' => 'images',
+      'domain1classes' => '1',
+      'domain2classes' => '2',
+      'domain1class1name' => 'default',
+      'domain1class1mindevcount' => '2',
+      'domain2class1name' => 'default',
+      'domain2class1mindevcount' => '2',
+      'domain2class2name' => 'resize',
+      'domain2class2mindevcount' => '1',
+    }
+
+    expected = {
+      'test'   => { 'default' => 2, },
+      'images' => { 'default' => 2, 'resize' => 1 },
+    }
+
+    assert_equal expected, @client.get_domains
+  end
+
+  def disabled_test_get_stats
+    @backend.stats = {}
+
+    expected = {
+      'fids' => { 'max' => '99', 'count' => '2' },
+      'device' => [
+        { 'status' => 'alive', 'files' => '2', 'id' => '1', 'host' => 'rur-1' },
+        { 'status' => 'alive', 'files' => '2', 'id' => '2', 'host' => 'rur-2' }
+      ],
+      'replication' => [
+        { 'files' => '2', 'class' => 'normal', 'devcount' => '2',
+          'domain' => 'test' }
+      ],
+      'file' => [{ 'files' => '2', 'class' => 'normal', 'domain' => 'test' }]
+    }
+
+    assert_equal 
+  end
+
+  def test_get_stats_fids
+    @backend.stats = {
+      'fidmax' => 99,
+      'fidcount' => 2,
+    }
+
+    expected = {
+      'fids' => { 'max' => 99, 'count' => 2 },
+    }
+
+    assert_equal expected, @client.get_stats('all')
+  end
+
+  def test_list_fids
+    @backend.list_fids = {
+      'fid_count' => '2',
+      'fid_1_fid' => '99',
+      'fid_1_class' => 'normal',
+      'fid_1_devcount' => '2',
+      'fid_1_domain' => 'test',
+      'fid_1_key' => 'file_key',
+      'fid_1_length' => '4',
+      'fid_2_fid' => '82',
+      'fid_2_class' => 'normal',
+      'fid_2_devcount' => '2',
+      'fid_2_domain' => 'test',
+      'fid_2_key' => 'new_new_key',
+      'fid_2_length' => '9',
+    }
+
+    expected = [
+      { "fid"      => "99",
+        "class"    => "normal",
+        "domain"   => "test",
+        "devcount" => "2",
+        "length"   => "4",
+        "key"      => "file_key" },
+      { "fid"      => "82",
+        "class"    => "normal",
+        "devcount" => "2",
+        "domain"   => "test",
+        "length"   => "9",
+        "key"      => "new_new_key" },
+    ]
+
+    assert_equal expected, @client.list_fids(0, 100)
+  end
+
+end
+
diff --git a/test/test_backend.rb b/test/test_backend.rb
new file mode 100644
index 0000000..7fa9a7c
--- /dev/null
+++ b/test/test_backend.rb
@@ -0,0 +1,220 @@
+require 'test/unit'
+
+$TESTING = true
+
+require 'mogilefs/backend'
+
+class MogileFS::Backend
+
+  attr_accessor :hosts
+  attr_reader :timeout, :dead
+  attr_writer :lasterr, :lasterrstr, :socket
+
+end
+
+class FakeSocket
+
+  def initialize
+    @closed = false
+  end
+
+  def closed?
+    @closed
+  end
+
+  def close
+    @closed = true
+    return nil
+  end
+
+  def peeraddr
+    ['AF_INET', 6001, 'localhost', '127.0.0.1']
+  end
+
+end
+
+class TestBackend < Test::Unit::TestCase
+
+  def setup
+    @backend = MogileFS::Backend.new :hosts => ['localhost:1']
+  end
+
+  def test_initialize
+    assert_raises ArgumentError do MogileFS::Backend.new end
+    assert_raises ArgumentError do MogileFS::Backend.new :hosts => [] end
+    assert_raises ArgumentError do MogileFS::Backend.new :hosts => [''] end
+
+    assert_equal ['localhost:1'], @backend.hosts
+    assert_equal 3, @backend.timeout
+    assert_equal nil, @backend.lasterr
+    assert_equal nil, @backend.lasterrstr
+    assert_equal({}, @backend.dead)
+
+    @backend = MogileFS::Backend.new :hosts => ['localhost:6001'], :timeout => 1
+    assert_equal 1, @backend.timeout
+  end
+
+  def test_do_request
+    socket_request = ''
+    socket = Object.new
+    def socket.closed?() false end
+    def socket.send(request, flags) return request.length end
+    def @backend.select(*args) return [true] end
+    def socket.gets() return 'OK 1 you=win' end
+
+    @backend.instance_variable_set '@socket', socket
+
+    assert_equal({'you' => 'win'},
+                 @backend.do_request('go!', { 'fight' => 'team fight!' }))
+  end
+
+  def test_do_request_send_error
+    socket_request = ''
+    socket = Object.new
+    def socket.closed?() false end
+    def socket.send(request, flags) raise SystemCallError, 'dummy' end
+
+    @backend.instance_variable_set '@socket', socket
+
+    assert_raises RuntimeError do
+      @backend.do_request 'go!', { 'fight' => 'team fight!' }
+    end
+
+    assert_equal nil, @backend.instance_variable_get('@socket')
+  end
+
+  def test_do_request_truncated
+    socket_request = ''
+    socket = Object.new
+    def socket.closed?() false end
+    def socket.send(request, flags) return request.length - 1 end
+
+    @backend.instance_variable_set '@socket', socket
+
+    assert_raises RuntimeError do
+      @backend.do_request 'go!', { 'fight' => 'team fight!' }
+    end
+  end
+
+  def test_make_request
+    assert_equal "go! fight=team+fight%21\r\n",
+                 @backend.make_request('go!', { 'fight' => 'team fight!' })
+  end
+
+  def test_parse_response
+    assert_equal({'foo' => 'bar', 'baz' => 'hoge'},
+                 @backend.parse_response('OK 1 foo=bar&baz=hoge'))
+    assert_equal nil, @backend.parse_response('ERR you totally suck')
+    assert_equal 'you', @backend.lasterr
+    assert_equal 'totally suck', @backend.lasterrstr
+
+    assert_raises RuntimeError do
+      @backend.parse_response 'garbage'
+    end
+  end
+
+  def test_readable_eh_readable
+    socket = Object.new
+    def socket.closed?() false end
+    def @backend.select(*args) return [true] end
+    @backend.instance_variable_set '@socket', socket
+
+    assert_equal true, @backend.readable?
+  end
+
+  def test_readable_eh_not_readable
+    socket = FakeSocket.new
+    def socket.closed?() false end
+    def @backend.select(*args) return [] end
+    @backend.instance_variable_set '@socket', socket
+
+    begin
+      @backend.readable?
+    rescue MogileFS::UnreadableSocketError => e
+      assert_equal '127.0.0.1:6001 never became readable', e.message
+    rescue Exception
+      flunk "MogileFS::UnreadableSocketError not raised"
+    else
+      flunk "MogileFS::UnreadableSocketError not raised"
+    end
+  end
+
+  def test_socket
+    assert_equal({}, @backend.dead)
+    assert_raises RuntimeError do @backend.socket end
+    assert_equal(['localhost:1'], @backend.dead.keys)
+  end
+
+  def test_socket_robust
+    @backend.hosts = ['localhost:6001', 'localhost:6002']
+    def @backend.connect_to(host, port)
+      @first = (defined? @first) ? false : true
+      raise Errno::ECONNREFUSED if @first
+    end
+
+    assert_equal({}, @backend.dead)
+    @backend.socket
+    assert_equal false, @backend.dead.keys.empty?
+  end
+
+  def test_shutdown
+    fake_socket = FakeSocket.new
+    @backend.socket = fake_socket
+    assert_equal fake_socket, @backend.socket
+    @backend.shutdown
+    assert_equal nil, @backend.instance_variable_get(:@socket)
+  end
+
+  def test_url_decode
+    assert_equal({"\272z" => "\360opy", "f\000" => "\272r"},
+                 @backend.url_decode("%baz=%f0opy&f%00=%bar"))
+  end
+
+  def test_url_encode
+    params = [["f\000", "\272r"], ["\272z", "\360opy"]]
+    assert_equal "f%00=%bar&%baz=%f0opy", @backend.url_encode(params)
+  end
+
+  def test_url_escape # \n for unit_diff
+    actual = (0..255).map { |c| @backend.url_escape c.chr }.join "\n"
+
+    expected = []
+    expected.push(*(0..0x1f).map { |c| "%%%0.2x" % c })
+    expected << '+'
+    expected.push(*(0x21..0x2b).map { |c| "%%%0.2x" % c })
+    expected.push(*%w[, - . /])
+    expected.push(*('0'..'9'))
+    expected.push(*%w[: %3b %3c %3d %3e %3f %40])
+    expected.push(*('A'..'Z'))
+    expected.push(*%w[%5b \\ %5d %5e _ %60])
+    expected.push(*('a'..'z'))
+    expected.push(*(0x7b..0xff).map { |c| "%%%0.2x" % c })
+
+    expected = expected.join "\n"
+
+    assert_equal expected, actual
+  end
+
+  def test_url_unescape
+    input = []
+    input.push(*(0..0x1f).map { |c| "%%%0.2x" % c })
+    input << '+'
+    input.push(*(0x21..0x2b).map { |c| "%%%0.2x" % c })
+    input.push(*%w[, - . /])
+    input.push(*('0'..'9'))
+    input.push(*%w[: %3b %3c %3d %3e %3f %40])
+    input.push(*('A'..'Z'))
+    input.push(*%w[%5b \\ %5d %5e _ %60])
+    input.push(*('a'..'z'))
+    input.push(*(0x7b..0xff).map { |c| "%%%0.2x" % c })
+
+    actual = input.map { |c| @backend.url_unescape c }.join "\n"
+
+    expected = (0..255).map { |c| c.chr }.join "\n"
+    expected.sub! '+', ' '
+
+    assert_equal expected, actual
+  end
+
+end
+
diff --git a/test/test_client.rb b/test/test_client.rb
new file mode 100644
index 0000000..93d1522
--- /dev/null
+++ b/test/test_client.rb
@@ -0,0 +1,53 @@
+require 'test/unit'
+
+$TESTING = true
+
+require 'mogilefs'
+
+class TestClient < Test::Unit::TestCase
+
+  def setup
+    @client = MogileFS::Client.new :hosts => ['kaa:6001']
+  end
+
+  def test_initialize
+    client = MogileFS::Client.new :hosts => ['kaa:6001']
+    assert_not_nil client
+    assert_instance_of MogileFS::Backend, client.backend
+    assert_equal ['kaa:6001'], client.hosts
+
+    client = MogileFS::Client.new :hosts => ['kaa:6001'], :timeout => 5
+    assert_equal 5, client.backend.timeout
+  end
+
+  def test_err
+    @client.backend.lasterr = 'you'
+    assert_equal 'you', @client.err
+  end
+
+  def test_errstr
+    @client.backend.lasterrstr = 'totally suck'
+    assert_equal 'totally suck', @client.errstr
+  end
+
+  def test_reload
+    orig_backend = @client.backend
+
+    @client.hosts = ['ziz:6001']
+    @client.reload
+
+    assert_not_same @client.backend, orig_backend
+    assert_equal ['ziz:6001'], @client.backend.hosts
+  end
+
+  def test_readonly_eh_readonly
+    client = MogileFS::Client.new :hosts => ['kaa:6001'], :readonly => true
+    assert_equal true, client.readonly?
+  end
+
+  def test_readonly_eh_readwrite
+    assert_equal false, @client.readonly?
+  end
+
+end
+
diff --git a/test/test_mogilefs.rb b/test/test_mogilefs.rb
new file mode 100644
index 0000000..0850d34
--- /dev/null
+++ b/test/test_mogilefs.rb
@@ -0,0 +1,160 @@
+require 'test/setup'
+
+class URI::HTTP
+
+  class << self
+    attr_accessor :read_data
+  end
+
+  def read
+    self.class.read_data.shift
+  end
+
+end
+
+class TestMogileFS__MogileFS < TestMogileFS
+
+  def setup
+    @klass = MogileFS::MogileFS
+    super
+  end
+
+  def test_initialize
+    assert_equal 'test', @client.domain
+    assert_equal '/mogilefs/test', @client.root
+
+    assert_raises ArgumentError do
+      MogileFS::MogileFS.new :hosts => ['kaa:6001'], :root => '/mogilefs/test'
+    end
+  end
+
+  def test_get_file_data_http
+    URI::HTTP.read_data = %w[data!]
+
+    path1 = 'http://rur-1/dev1/0/000/000/0000000062.fid'
+    path2 = 'http://rur-2/dev2/0/000/000/0000000062.fid'
+
+    @backend.get_paths = { 'paths' => 2, 'path1' => path1, 'path2' => path2 }
+
+    assert_equal 'data!', @client.get_file_data('key')
+  end
+
+  def test_get_paths
+    path1 = 'rur-1/dev1/0/000/000/0000000062.fid'
+    path2 = 'rur-2/dev2/0/000/000/0000000062.fid'
+
+    @backend.get_paths = { 'paths' => 2, 'path1' => path1, 'path2' => path2 }
+
+    expected = ["#{@root}/#{path1}", "#{@root}/#{path2}"]
+
+    assert_equal expected, @client.get_paths('key').sort
+  end
+
+  def test_get_paths_unknown_key
+    @backend.get_paths = ['unknown_key', '']
+
+    assert_equal nil, @client.get_paths('key')
+  end
+
+  def test_delete_existing
+    @backend.delete = { }
+    assert_nothing_raised do
+      @client.delete 'no_such_key'
+    end
+  end
+
+  def test_delete_nonexisting
+    @backend.delete = 'unknown_key', ''
+    assert_nothing_raised do
+      assert_equal nil, @client.delete('no_such_key')
+    end
+  end
+
+  def test_delete_readonly
+    @client.readonly = true
+    assert_raises RuntimeError do
+      @client.delete 'no_such_key'
+    end
+  end
+
+  def test_each_key
+    @backend.list_keys = { 'key_count' => 2, 'next_after' => 'new_key_2',
+                           'key_1' => 'new_key_1', 'key_2' => 'new_key_2' }
+    @backend.list_keys = { 'key_count' => 2, 'next_after' => 'new_key_4',
+                           'key_1' => 'new_key_3', 'key_2' => 'new_key_4' }
+    @backend.list_keys = { 'key_count' => 0, 'next_after' => 'new_key_4' }
+    keys = []
+    @client.each_key 'new' do |key|
+      keys << key
+    end
+
+    assert_equal %w[new_key_1 new_key_2 new_key_3 new_key_4], keys
+  end
+
+  def test_list_keys
+    @backend.list_keys = { 'key_count' => 2, 'next_after' => 'new_key_2',
+                           'key_1' => 'new_key_1', 'key_2' => 'new_key_2' }
+
+    keys, next_after = @client.list_keys 'new'
+    assert_equal ['new_key_1', 'new_key_2'], keys.sort
+    assert_equal 'new_key_2', next_after
+  end
+
+  def test_new_file_http
+    @client.readonly = true
+    assert_raises RuntimeError do
+      @client.new_file 'new_key', 'test'
+    end
+  end
+
+  def test_new_file_readonly
+    @client.readonly = true
+    assert_raises RuntimeError do
+      @client.new_file 'new_key', 'test'
+    end
+  end
+
+  def test_store_content_readonly
+    @client.readonly = true
+    assert_raises RuntimeError do
+      @client.store_content 'new_key', 'test', nil
+    end
+  end
+
+  def test_store_file_readonly
+    @client.readonly = true
+    assert_raises RuntimeError do
+      @client.store_file 'new_key', 'test', nil
+    end
+  end
+
+  def test_rename_existing
+    @backend.rename = {}
+    assert_nothing_raised do
+      assert_equal(nil, @client.rename('from_key', 'to_key'))
+    end
+  end
+
+  def test_rename_nonexisting
+    @backend.rename = 'unknown_key', ''
+    assert_nothing_raised do
+      assert_equal(nil, @client.rename('from_key', 'to_key'))
+    end
+  end
+
+  def test_rename_readonly
+    @client.readonly = true
+    assert_raises RuntimeError do
+      @client.rename 'new_key', 'test'
+    end
+  end
+
+  def test_sleep
+    @backend.sleep = {}
+    assert_nothing_raised do
+      assert_equal({}, @client.sleep(2))
+    end
+  end
+
+end
+
diff --git a/test/test_pool.rb b/test/test_pool.rb
new file mode 100644
index 0000000..1a64752
--- /dev/null
+++ b/test/test_pool.rb
@@ -0,0 +1,98 @@
+require 'test/unit'
+
+$TESTING = true
+
+require 'mogilefs/pool'
+
+class MogileFS::Pool
+
+  attr_reader :objects, :queue
+
+end
+
+class Resource; end
+
+class ResourceWithArgs
+
+  def initialize(args)
+  end
+
+end
+
+class TestPool < Test::Unit::TestCase
+
+  def setup
+    @pool = MogileFS::Pool.new Resource
+  end
+
+  def test_get
+    o1 = @pool.get
+    o2 = @pool.get
+    assert_kind_of Resource, o1
+    assert_kind_of Resource, o2
+    assert_not_equal o1, o2
+  end
+
+  def test_get_with_args
+    @pool = MogileFS::Pool.new ResourceWithArgs, 'my arg'
+    o = @pool.get
+    assert_kind_of ResourceWithArgs, o
+  end
+
+  def test_put
+    o = @pool.get
+    @pool.put o
+
+    assert_raises(MogileFS::Pool::BadObjectError) { @pool.put nil }
+    assert_raises(MogileFS::Pool::BadObjectError) { @pool.put Resource.new }
+  end
+
+  def test_put_destroy
+    objs = (0...7).map { @pool.get } # pool full
+
+    assert_equal 7, @pool.objects.length
+    assert_equal 0, @pool.queue.length
+
+    4.times { @pool.put objs.shift }
+
+    assert_equal 7, @pool.objects.length
+    assert_equal 4, @pool.queue.length
+
+    @pool.put objs.shift # trip threshold
+
+    assert_equal 4, @pool.objects.length
+    assert_equal 2, @pool.queue.length
+
+    @pool.put objs.shift # don't need to remove any more
+
+    assert_equal 4, @pool.objects.length
+    assert_equal 3, @pool.queue.length
+
+    @pool.put objs.shift until objs.empty?
+
+    assert_equal 4, @pool.objects.length
+    assert_equal 4, @pool.queue.length
+  end
+
+  def test_use
+    val = @pool.use { |o| assert_kind_of Resource, o }
+    assert_equal nil, val, "Don't return object from pool"
+  end
+
+  def test_use_with_exception
+    @pool.use { |o| raise } rescue nil
+    assert_equal 1, @pool.queue.length, "Resource not returned to pool"
+  end
+
+  def test_use_reuse
+    o1 = nil
+    o2 = nil
+
+    @pool.use { |o| o1 = o }
+    @pool.use { |o| o2 = o }
+
+    assert_equal o1, o2, "Objects must be reused"
+  end
+
+end
+