From d5f49b90ec7461b07e88eb2e13f9b510190be256 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Thu, 1 Dec 2011 03:03:52 +0000 Subject: remove mogstored_rack example It's now a separate project: http://bogomips.org/mogstored_rack/ --- examples/mogstored_rack.rb | 188 --------------------------------------- test/test_mogstored_rack.rb | 7 +- test/test_unit_mogstored_rack.rb | 72 --------------- 3 files changed, 2 insertions(+), 265 deletions(-) delete mode 100644 examples/mogstored_rack.rb delete mode 100644 test/test_unit_mogstored_rack.rb diff --git a/examples/mogstored_rack.rb b/examples/mogstored_rack.rb deleted file mode 100644 index 7ac90d6..0000000 --- a/examples/mogstored_rack.rb +++ /dev/null @@ -1,188 +0,0 @@ -# -*- encoding: binary -*- -require 'digest/md5' -require 'rack' - -# Rack application for handling HTTP PUT/DELETE/MKCOL operations needed -# for a MogileFS storage server. GET requests are handled by -# Rack::File and Rack::Head _must_ be in the middleware stack for -# mogilefsd fsck to work properly with keepalive. -# -# Usage in rackup config file (config.ru): -# -# require "./mogstored_rack" -# use Rack::Head -# run MogstoredRack.new("/var/mogdata") -class MogstoredRack - class ContentMD5 < Digest::MD5 - def content_md5 - [ digest ].pack("m").strip! - end - end - - def initialize(root, opts = {}) - @root = File.expand_path(root) - @io_size = opts[:io_size] || 0x100000 - @rack_file = (opts[:app] || Rack::File.new(@root)) - @fsync = !! opts[:fsync] - @creat_perms = opts[:creat_perms] || (~File.umask & 0666) - @mkdir_perms = opts[:mkdir_perms] || (~File.umask & 0777) - @reread_verify = !! opts[:reread_verify] - @open_flags = opts[:open_flags] || 0 - @open_flags |= IO::RDWR | IO::CREAT | IO::EXCL - end - - def tmpfile(basename, dir) - t = Time.now.utc.strftime("%Y%m%d%H%M%S") - seq = 0 - begin - fp = File.open("#{dir}/#{basename}.#{t}.#{seq}.tmp", @open_flags, 0600) - rescue Errno::EEXIST - retry if (seq += 1) < 10 - raise - end - fp.binmode - fp.sync = true - fp - end - - def call(env) - case env["REQUEST_METHOD"] - when "GET", "HEAD" - case env["PATH_INFO"] - when "/" - r(200, "") # MogileFS seems to need this... - else - @rack_file.call(env) - end - when "PUT" - put(env) - when "DELETE" - delete(env) - when "MKCOL" - mkcol(env) - else - r(405, "unsupported method", env) - end - rescue Errno::EPERM, Errno::EACCES => err - r(403, "#{err.message} (#{err.class})", env) - rescue => err - r(500, "#{err.message} (#{err.class})", env) - end - - def mkcol(env) - path = server_path(env) or return r(400) - Dir.mkdir(path, @mkdir_perms) - r(204) - rescue Errno::EEXIST # succeed (204) on race condition - File.directory?(path) ? r(204) : r(409) - end - - def delete(env) - path = server_path(env) or return r(400) - File.exist?(path) or return r(404) - File.directory?(path) ? Dir.rmdir(path) : File.unlink(path) - r(204) - rescue Errno::ENOENT # return 404 on race condition - File.exist?(path) ? r(500) : r(404) - end - - def put(env) - path = server_path(env) or return r(400) - dir = File.dirname(path) - File.directory?(dir) or return r(403) - - tmp = tmpfile(File.basename(path), dir) - buf = "" - received = put_loop(env["rack.input"], tmp, buf) - err = content_md5_fail?(env, received) and return err - if @reread_verify && err = reread_md5_fail?(env, tmp, received, buf) - return err - end - tmp.chmod(@creat_perms) - File.rename(tmp.path, path) - fsync(dir, tmp) if @fsync - resp = r(201) - resp[1]["X-Received-Content-MD5"] = received - resp - rescue - File.unlink(tmp.path) if tmp && File.exist?(tmp.path) - raise - ensure - tmp.close if tmp - end - - def put_loop(src, dst, buf) - md5 = ContentMD5.new - while src.read(@io_size, buf) - md5.update(buf) - dst.write(buf) - end - md5.content_md5 - end - - def server_path(env) - path = env['PATH_INFO'].squeeze('/') - path.split(%r{/}).include?("..") and return false - "#@root#{path}" - end - - # returns a plain-text HTTP response - def r(code, msg = nil, env = nil) - if env && logger = env["rack.logger"] - logger.warn("#{env['REQUEST_METHOD']} #{env['PATH_INFO']} " \ - "#{code} #{msg.inspect}") - end - if Rack::Utils::STATUS_WITH_NO_ENTITY_BODY.include?(code) - [ code, {}, [] ] - else - msg ||= Rack::Utils::HTTP_STATUS_CODES[code] || "" - msg += "\n" if msg.size > 0 - [ code, - { 'Content-Type' => 'text/plain', 'Content-Length' => msg.size.to_s }, - [ msg ] ] - end - end - - # Tries to detect filesystem/disk corruption. - # Unfortunately, posix_fadvise(2)/IO#advise is only advisory and - # can't guarantee we're not just reading the data in the kernel - # page cache. - def reread_md5_fail?(env, tmp, received, buf) - # try to force a reread from the storage device, not cache - tmp.fsync - tmp.rewind - tmp.advise(:dontneed) rescue nil # only in Ruby 1.9.3 and only advisory - - md5 = ContentMD5.new - while tmp.read(0x4000, buf) - md5.update(buf) - end - reread = md5.content_md5 - reread == received and return false # success - r(500, "reread MD5 mismatch\n" \ - "received: #{received}\n" \ - " reread: #{reread}", env) - end - - # Tries to detect network corruption by verifying the client-supplied - # Content-MD5 is correct. It's highly unlikely the MD5 can be corrupted - # in a way that also allows corrupt data to pass through. - # - # The Rainbows!/Unicorn HTTP servers will populate the HTTP_CONTENT_MD5 - # field in +env+ after env["rack.input"] is fully-consumed. Clients - # may also send Content-MD5 as a header and this will still work. - def content_md5_fail?(env, received) - expected = env["HTTP_CONTENT_MD5"] or return false - expected = expected.strip - expected == received and return false # success - r(400, "Content-MD5 mismatch\n" \ - "expected: #{expected}\n" \ - "received: #{received}", env) - end - - # fsync each and every directory component above us on the same device - def fsync(dir, tmp) - tmp.fsync - File.open(dir) { |io| io.fsync } - end -end diff --git a/test/test_mogstored_rack.rb b/test/test_mogstored_rack.rb index 491f372..f4bb0ca 100644 --- a/test/test_mogstored_rack.rb +++ b/test/test_mogstored_rack.rb @@ -47,12 +47,8 @@ EOF # I would use Rainbows! + *Threads + Ruby 1.9.3 in production def unicorn_setup - examples_dir = Dir.pwd + "/examples" - assert File.directory?(examples_dir) @ru = Tempfile.new(%w(mogstored_rack .ru)) @ru.write <true)) - all_methods(req) - end - - def test_reread_verify - app = MogstoredRack.new(@docroot, :reread_verify=>true) - req = Rack::MockRequest.new(app) - all_methods(req) - end - - def all_methods(req) - assert_equal 200, req.get("/").status - assert ! File.directory?("#@docroot/dev666") - assert_equal 204, req.request("MKCOL", "/dev666").status - assert File.directory?("#@docroot/dev666") - - io = StringIO.new("HELLO") - r = req.request("PUT", "/dev666/666.fid", :input => io) - assert_equal 201, r.status - assert_equal "HELLO", IO.read("#@docroot/dev666/666.fid") - - # invalid MD5 - io = StringIO.new("WORLD") - md5 = [ Digest::MD5.new.digest ].pack("m").strip! - opts = { :input => io, "HTTP_CONTENT_MD5" => md5 } - r = req.request("PUT", "/dev666/666.fid", opts) - assert_equal 400, r.status - assert_equal "HELLO", IO.read("#@docroot/dev666/666.fid") - - # valid MD5 - io = StringIO.new("VALID") - md5 = [ Digest::MD5.digest("VALID") ].pack("m").strip! - opts = { :input => io, "HTTP_CONTENT_MD5" => md5 } - r = req.request("PUT", "/dev666/666.fid", opts) - assert_equal 201, r.status - assert_equal "VALID", IO.read("#@docroot/dev666/666.fid") - - r = req.request("GET", "/dev666/666.fid") - assert_equal 200, r.status - assert_equal "VALID", r.body - - r = req.request("DELETE", "/dev666/666.fid") - assert_equal 204, r.status - assert ! File.exist?("#@docroot/dev666/666.fid") - end - - def teardown - FileUtils.rmtree(@docroot) - end -end if defined?(Rack) -- cgit v1.2.3-24-ge0c7