diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | GNUmakefile | 7 | ||||
| -rw-r--r-- | Rakefile | 7 | ||||
| -rw-r--r-- | config/.gitignore | 1 | ||||
| -rw-r--r-- | config/isolate.rb | 25 | ||||
| -rw-r--r-- | lib/rainbows.rb | 8 | ||||
| -rw-r--r-- | lib/rainbows/base.rb | 36 | ||||
| -rw-r--r-- | lib/rainbows/const.rb | 1 | ||||
| -rw-r--r-- | lib/rainbows/ev_core.rb | 2 | ||||
| -rw-r--r-- | lib/rainbows/fiber/base.rb | 6 | ||||
| -rw-r--r-- | lib/rainbows/fiber/rev.rb | 2 | ||||
| -rw-r--r-- | lib/rainbows/http_response.rb | 41 | ||||
| -rw-r--r-- | lib/rainbows/http_server.rb | 1 | ||||
| -rw-r--r-- | lib/rainbows/rev/deferred_response.rb | 21 | ||||
| -rw-r--r-- | lib/rainbows/revactor.rb | 2 | ||||
| -rw-r--r-- | local.mk.sample | 35 | ||||
| -rw-r--r-- | t/async_examples/async_app.ru | 16 | ||||
| -rw-r--r-- | t/rack-fiber_pool/app.ru | 5 | ||||
| -rwxr-xr-x | t/t0600-rack-fiber_pool.sh | 49 |
19 files changed, 191 insertions, 75 deletions
@@ -16,3 +16,4 @@ pkg/ /.manifest /GIT-VERSION-FILE /man +/tmp diff --git a/GNUmakefile b/GNUmakefile index b11ecf9..42c2162 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -3,6 +3,7 @@ all:: RUBY = ruby RAKE = rake GIT_URL = git://git.bogomips.org/rainbows.git +ISOLATE_CONFIG = config/isolate.rb GIT-VERSION-FILE: .FORCE-GIT-VERSION-FILE @./GIT-VERSION-GEN @@ -15,6 +16,12 @@ ifeq ($(RUBY_VERSION),) RUBY_VERSION := $(shell $(RUBY) -e 'puts RUBY_VERSION') endif +# rake takes forever to start +isolate: tmp/gems/$(RUBY_VERSION)/.isolate +tmp/gems/$(RUBY_VERSION)/.isolate: $(ISOLATE_CONFIG) + ISOLATE_CONFIG=$(ISOLATE_CONFIG) $(RAKE) isolate + > $@ + base_bins := rainbows bins := $(addprefix bin/, $(base_bins)) man1_bins := $(addsuffix .1, $(base_bins)) @@ -183,3 +183,10 @@ task :fm_update do p http.post(uri.path, req, {'Content-Type'=>'application/json'}) end end + +desc 'isolate gems for development' +task :isolate do + require 'isolate' + Isolate.gems "tmp/gems/#{RUBY_VERSION}", + :file => ENV['ISOLATE_CONFIG'] +end diff --git a/config/.gitignore b/config/.gitignore new file mode 100644 index 0000000..0aaed9f --- /dev/null +++ b/config/.gitignore @@ -0,0 +1 @@ +/isolate_*.rb diff --git a/config/isolate.rb b/config/isolate.rb new file mode 100644 index 0000000..1c6874b --- /dev/null +++ b/config/isolate.rb @@ -0,0 +1,25 @@ +# this the default config file used by John Barnette's isolate gem +# you can create a config/isolate_local.rb file to override this +# See the corresponding tasks in Rakefile and GNUmakefile +# `rake isolate' or (faster in the unmodified case, `make isolate') + +gem 'rack', '1.1.0' +gem 'unicorn', '0.97.0' + +gem 'iobuffer', '0.1.3' +gem 'rev', '0.3.2' + +gem 'eventmachine', '0.12.10' + +gem 'sinatra', '0.9.4' +gem 'async_sinatra', '0.1.5' + +gem 'neverblock', '0.1.6.2' + +if defined?(::Fiber) + gem 'case', '0.5' + gem 'revactor', '0.1.5' + gem 'rack-fiber_pool', '0.9.0' +end + +gem 'cramp', '0.10' diff --git a/lib/rainbows.rb b/lib/rainbows.rb index 5c8ee94..ccf211e 100644 --- a/lib/rainbows.rb +++ b/lib/rainbows.rb @@ -73,6 +73,14 @@ module Rainbows rv rescue Errno::EAGAIN, Errno::ECONNABORTED end + + # returns a string representing the address of the given client +io+ + # For local UNIX domain sockets, this will return a string referred + # to by the (non-frozen) Unicorn::HttpRequest::LOCALHOST constant. + def addr(io) + io.respond_to?(:peeraddr) ? + io.peeraddr.last : Unicorn::HttpRequest::LOCALHOST + end end # configures \Rainbows! with a given concurrency model to +use+ and diff --git a/lib/rainbows/base.rb b/lib/rainbows/base.rb index a29a5bb..27b4c1d 100644 --- a/lib/rainbows/base.rb +++ b/lib/rainbows/base.rb @@ -29,14 +29,35 @@ module Rainbows logger.info "Rainbows! #@use worker_connections=#@worker_connections" end + if IO.respond_to?(:copy_stream) + def write_body(client, body) + if body.respond_to?(:to_path) + io = body.respond_to?(:to_io) ? body.to_io : body.to_path + IO.copy_stream(io, client) + else + body.each { |chunk| client.write(chunk) } + end + ensure + body.respond_to?(:close) and body.close + end + else + def write_body(client, body) + body.each { |chunk| client.write(chunk) } + ensure + body.respond_to?(:close) and body.close + end + end + # once a client is accepted, it is processed in its entirety here # in 3 easy steps: read request, call app, write app response + # this is used by synchronous concurrency models + # Base, ThreadSpawn, ThreadPool def process_client(client) buf = client.readpartial(CHUNK_SIZE) # accept filters protect us here hp = HttpParser.new env = {} alive = true - remote_addr = TCPSocket === client ? client.peeraddr.last : LOCALHOST + remote_addr = Rainbows.addr(client) begin # loop while ! hp.headers(env, buf) @@ -49,17 +70,20 @@ module Rainbows HttpRequest::NULL_IO : Unicorn::TeeInput.new(client, env, hp, buf) env[REMOTE_ADDR] = remote_addr - response = app.call(env.update(RACK_DEFAULTS)) + status, headers, body = app.call(env.update(RACK_DEFAULTS)) - if 100 == response.first.to_i + if 100 == status client.write(EXPECT_100_RESPONSE) env.delete(HTTP_EXPECT) - response = app.call(env) + status, headers, body = app.call(env) end alive = hp.keepalive? && G.alive - out = [ alive ? CONN_ALIVE : CONN_CLOSE ] if hp.headers? - HttpResponse.write(client, response, out) + if hp.headers? + out = [ alive ? CONN_ALIVE : CONN_CLOSE ] + client.write(HttpResponse.header_string(status, headers, out)) + end + write_body(client, body) end while alive and hp.reset.nil? and env.clear # if we get any error, try to write something back to the client # assuming we haven't closed the socket, but don't get hung up diff --git a/lib/rainbows/const.rb b/lib/rainbows/const.rb index 99fb257..08c4821 100644 --- a/lib/rainbows/const.rb +++ b/lib/rainbows/const.rb @@ -17,7 +17,6 @@ module Rainbows CONN_CLOSE = "Connection: close\r\n" CONN_ALIVE = "Connection: keep-alive\r\n" - LOCALHOST = Unicorn::HttpRequest::LOCALHOST # client IO object that supports reading and writing directly # without filtering it through the HTTP chunk parser. diff --git a/lib/rainbows/ev_core.rb b/lib/rainbows/ev_core.rb index 3d02b8a..682bdd6 100644 --- a/lib/rainbows/ev_core.rb +++ b/lib/rainbows/ev_core.rb @@ -14,7 +14,7 @@ module Rainbows ASYNC_CLOSE = "async.close".freeze def post_init - @remote_addr = ::TCPSocket === @_io ? @_io.peeraddr.last : LOCALHOST + @remote_addr = Rainbows.addr(@_io) @env = {} @hp = HttpParser.new @state = :headers # [ :body [ :trailers ] ] :app_call :close diff --git a/lib/rainbows/fiber/base.rb b/lib/rainbows/fiber/base.rb index b731947..a056152 100644 --- a/lib/rainbows/fiber/base.rb +++ b/lib/rainbows/fiber/base.rb @@ -57,15 +57,17 @@ module Rainbows def schedule_sleepers max = nil now = Time.now + fibs = [] ZZ.delete_if { |fib, time| if now >= time - fib.resume + fibs << fib now = Time.now else max = time false end } + fibs.each { |fib| fib.resume } max.nil? || max > (now + 1) ? 1 : max - now end @@ -76,7 +78,7 @@ module Rainbows hp = HttpParser.new env = {} alive = true - remote_addr = TCPSocket === io ? io.peeraddr.last : LOCALHOST + remote_addr = Rainbows.addr(io) begin # loop while ! hp.headers(env, buf) diff --git a/lib/rainbows/fiber/rev.rb b/lib/rainbows/fiber/rev.rb index bd9638f..a733103 100644 --- a/lib/rainbows/fiber/rev.rb +++ b/lib/rainbows/fiber/rev.rb @@ -80,7 +80,7 @@ module Rainbows::Fiber hp = HttpParser.new env = {} alive = true - remote_addr = TCPSocket === io ? io.peeraddr.last : LOCALHOST + remote_addr = Rainbows.addr(io) begin # loop buf << (client.read_timeout or return) until hp.headers(env, buf) diff --git a/lib/rainbows/http_response.rb b/lib/rainbows/http_response.rb index 55c2ad2..1933218 100644 --- a/lib/rainbows/http_response.rb +++ b/lib/rainbows/http_response.rb @@ -1,34 +1,35 @@ # -*- encoding: binary -*- require 'time' -require 'rainbows' module Rainbows class HttpResponse < ::Unicorn::HttpResponse - def self.write(socket, rack_response, out = []) - status, headers, body = rack_response - - if Array === out - status = CODES[status.to_i] || status + def self.header_string(status, headers, out) + status = CODES[status.to_i] || status - headers.each do |key, value| - next if %r{\AX-Rainbows-}i =~ key - next if SKIP.include?(key.downcase) - if value =~ /\n/ - # avoiding blank, key-only cookies with /\n+/ - out.concat(value.split(/\n+/).map! { |v| "#{key}: #{v}\r\n" }) - else - out << "#{key}: #{value}\r\n" - end + headers.each do |key, value| + next if %r{\AX-Rainbows-}i =~ key + next if SKIP.include?(key.downcase) + if value =~ /\n/ + # avoiding blank, key-only cookies with /\n+/ + out.concat(value.split(/\n+/).map! { |v| "#{key}: #{v}\r\n" }) + else + out << "#{key}: #{value}\r\n" end - - socket.write("HTTP/1.1 #{status}\r\n" \ - "Date: #{Time.now.httpdate}\r\n" \ - "Status: #{status}\r\n" \ - "#{out.join('')}\r\n") end + "HTTP/1.1 #{status}\r\n" \ + "Date: #{Time.now.httpdate}\r\n" \ + "Status: #{status}\r\n" \ + "#{out.join('')}\r\n" + end + + def self.write(socket, rack_response, out = []) + status, headers, body = rack_response + out.instance_of?(Array) and + socket.write(header_string(status, headers, out)) + body.each { |chunk| socket.write(chunk) } ensure body.respond_to?(:close) and body.close diff --git a/lib/rainbows/http_server.rb b/lib/rainbows/http_server.rb index c4f804a..ea2e23f 100644 --- a/lib/rainbows/http_server.rb +++ b/lib/rainbows/http_server.rb @@ -1,5 +1,4 @@ # -*- encoding: binary -*- -require 'rainbows' module Rainbows class HttpServer < ::Unicorn::HttpServer diff --git a/lib/rainbows/rev/deferred_response.rb b/lib/rainbows/rev/deferred_response.rb index b69c7be..dd7a229 100644 --- a/lib/rainbows/rev/deferred_response.rb +++ b/lib/rainbows/rev/deferred_response.rb @@ -10,9 +10,13 @@ module Rainbows G = Rainbows::G HH = Rack::Utils::HeaderHash - def self.defer!(client, response, out) - body = response.last - headers = HH.new(response[1]) + def self.write(client, response, out) + status, headers, body = response + + body.respond_to?(:to_path) or + return HttpResponse.write(client, response, out) + + headers = HH.new(headers) # to_io is not part of the Rack spec, but make an exception # here since we can't get here without checking to_path first @@ -39,16 +43,11 @@ module Rainbows headers.delete('Transfer-Encoding') headers['Content-Length'] ||= st.size.to_s else # char/block device, directory, whatever... nobody cares - return response + return HttpResponse.write(client, response, out) end client.defer_body(io, out) - [ response.first, headers.to_hash, [] ] - end - - def self.write(client, response, out) - response.last.respond_to?(:to_path) and - response = defer!(client, response, out) - HttpResponse.write(client, response, out) + out.nil? or + client.write(HttpResponse.header_string(status, headers.to_hash, out)) end def initialize(io, client, do_chunk, body) diff --git a/lib/rainbows/revactor.rb b/lib/rainbows/revactor.rb index ab65184..ed08f2c 100644 --- a/lib/rainbows/revactor.rb +++ b/lib/rainbows/revactor.rb @@ -37,7 +37,7 @@ module Rainbows rd_args << RD_ARGS client.remote_addr else - LOCALHOST + Unicorn::HttpRequest::LOCALHOST end buf = client.read(*rd_args) hp = HttpParser.new diff --git a/local.mk.sample b/local.mk.sample index e6b4daf..1bd8832 100644 --- a/local.mk.sample +++ b/local.mk.sample @@ -6,39 +6,28 @@ RSYNC = rsync DLEXT := so -gems := rack-1.1.0 -# gems += unicorn-0.96.0 # installed via setup.rb -gems += rev-0.3.2 -gems += iobuffer-0.1.3 -gems += eventmachine-0.12.10 -gems += async_sinatra-0.1.5 sinatra-0.9.4 -gems += espace-neverblock-0.1.6.1 - -# Cramp isn't enabled by default since it depends on several prerelease gems -ifdef CRAMP - gems += cramp-0.7 - gems += activesupport-3.0.pre - gems += activemodel-3.0.pre - gems += arel-0.2.pre - gems += usher-0.6.2 - gems += fuzzyhash-0.0.11 - gems += mysqlplus-0.1.1 -endif # Avoid loading rubygems to speed up tests because gmake is # fork+exec heavy with Ruby. prefix = $(HOME) + ifeq ($(r19),) RUBY := $(prefix)/bin/ruby - gem_paths := $(addprefix $(prefix)/lib/ruby/gems/1.8/gems/,$(gems)) else prefix := $(prefix)/ruby-1.9 export PATH := $(prefix)/bin:$(PATH) RUBY := $(prefix)/bin/ruby --disable-gems - gems += case-0.5 revactor-0.1.5 - gem_paths := $(addprefix $(prefix)/lib/ruby/gems/1.9.1/gems/,$(gems)) endif +ifndef NO_ISOLATE + x := $(shell test -d t/ && NO_ISOLATE=T $(MAKE) -s isolate RUBY:="$(RUBY)") +endif + +RUBY_VERSION := $(shell $(RUBY) -e 'puts RUBY_VERSION') + +updir := $(shell git rev-parse --show-cdup) +gem_paths := $(wildcard $(updir)tmp/gems/$(RUBY_VERSION)/gems/*-*) + ifdef gem_paths sp := sp += @@ -55,9 +44,9 @@ TRACER = /usr/bin/time -v -o $(t_pfx).time full-test: test-18 test-19 test-18: - $(MAKE) test 2>&1 | sed -u -e 's!^!1.8 !' + $(MAKE) test 2>&1 | sed -e 's!^!1.8 !' test-19: - $(MAKE) test r19=t 2>&1 | sed -u -e 's!^!1.9 !' + $(MAKE) test r19=T 2>&1 | sed -e 's!^!1.9 !' latest: NEWS @awk 'BEGIN{RS="=== ";ORS=""}NR==2{sub(/\n$$/,"");print RS""$$0 }' < $< diff --git a/t/async_examples/async_app.ru b/t/async_examples/async_app.ru index 328effb..29f10f0 100644 --- a/t/async_examples/async_app.ru +++ b/t/async_examples/async_app.ru @@ -89,16 +89,16 @@ class AsyncApp # Get the headers out there asap, let the client know we're alive... EventMachine::next_tick { env['async.callback'].call [200, {'Content-Type' => 'text/plain'}, body] } - # Semi-emulate a long db request, instead of a timer, in reality we'd be - # waiting for the response data. Whilst this happens, other connections + # Semi-emulate a long db request, instead of a timer, in reality we'd be + # waiting for the response data. Whilst this happens, other connections # can be serviced. # This could be any callback based thing though, a deferrable waiting on - # IO data, a db request, an http request, an smtp send, whatever. + # IO data, a db request, an http request, an smtp send, whatever. EventMachine::add_timer(1) { body.call ["Woah, async!\n"] EventMachine::next_tick { - # This could actually happen any time, you could spawn off to new + # This could actually happen any time, you could spawn off to new # threads, pause as a good looking lady walks by, whatever. # Just shows off how we can defer chunks of data in the body, you can # even call this many times. @@ -116,11 +116,11 @@ end # The additions to env for async.connection and async.callback absolutely # destroy the speed of the request if Lint is doing it's checks on env. -# It is also important to note that an async response will not pass through -# any further middleware, as the async response notification has been passed -# right up to the webserver, and the callback goes directly there too. +# It is also important to note that an async response will not pass through +# any further middleware, as the async response notification has been passed +# right up to the webserver, and the callback goes directly there too. # Middleware could possibly catch :async, and also provide a different -# async.connection and async.callback. +# async.connection and async.callback. # use Rack::Lint run AsyncApp.new diff --git a/t/rack-fiber_pool/app.ru b/t/rack-fiber_pool/app.ru new file mode 100644 index 0000000..a4777ca --- /dev/null +++ b/t/rack-fiber_pool/app.ru @@ -0,0 +1,5 @@ +require 'rack/fiber_pool' +use Rack::FiberPool +use Rack::ContentLength +use Rack::ContentType, 'text/plain' +run lambda { |env| [ 200, {}, [ "#{Fiber.current}\n" ] ] } diff --git a/t/t0600-rack-fiber_pool.sh b/t/t0600-rack-fiber_pool.sh new file mode 100755 index 0000000..01f28b5 --- /dev/null +++ b/t/t0600-rack-fiber_pool.sh @@ -0,0 +1,49 @@ +#!/bin/sh +. ./test-lib.sh +case $model in +EventMachine) ;; +*) + t_info "skipping $T since it's not compatible with $model" + exit 0 + ;; +esac + +require_check rack/fiber_pool Rack::FiberPool + +t_plan 7 "basic test with rack-fiber_pool gem" + +CONFIG_RU=rack-fiber_pool/app.ru + +t_begin "setup and start" && { + rainbows_setup + rtmpfiles curl_err curl_out + + rainbows -D -c $unicorn_config $CONFIG_RU + rainbows_wait_start +} + +t_begin "send requests off in parallel" && { + curl --no-buffer -sSf http://$listen/ >> $curl_out 2>> $curl_err & + curl --no-buffer -sSf http://$listen/ >> $curl_out 2>> $curl_err & + curl --no-buffer -sSf http://$listen/ >> $curl_out 2>> $curl_err & +} + +t_begin "wait for curl terminations" && { + wait +} + +t_begin "termination signal sent" && { + kill $rainbows_pid +} + +t_begin "no errors from curl" && { + test ! -s $curl_err +} + +t_begin "no errors in stderr" && check_stderr + +t_begin "ensure we hit 3 separate fibers" && { + test x3 = x"$(sort < $curl_out | uniq | wc -l)" +} + +t_done |