about summary refs log tree commit homepage
diff options
context:
space:
mode:
authorEric Wong <normalperson@yhbt.net>2011-05-16 19:04:06 +0000
committerEric Wong <normalperson@yhbt.net>2011-05-16 19:34:52 +0000
commit39f625fff05d457b01f088017f463a86d3b6c626 (patch)
treeeae2a3a759b70e0f6d12caefdf177d3eca418efe
parentd520222050ce88388ae5e446c63642d79cc9796e (diff)
This allows using IO::Splice.copy_stream from the "io_splice"
RubyGem on recent Linux systems.  This also allows users to
disable copy_stream usage entirely and use traditional
response_body.each calls which are compatible with all Rack
servers (to workaround bugs in IO.copy_stream under 1.9.2-p180).
-rw-r--r--lib/rainbows/configurator.rb39
-rw-r--r--lib/rainbows/http_server.rb1
-rw-r--r--lib/rainbows/response.rb21
-rw-r--r--lib/rainbows/writer_thread_pool/client.rb2
-rw-r--r--lib/rainbows/writer_thread_spawn/client.rb2
-rw-r--r--t/t0026-splice-copy_stream-byte-range.sh40
-rw-r--r--t/t0027-nil-copy_stream.sh68
-rw-r--r--t/test-lib.sh27
-rw-r--r--t/test_isolate.rb7
9 files changed, 194 insertions, 13 deletions
diff --git a/lib/rainbows/configurator.rb b/lib/rainbows/configurator.rb
index 29d905b..a1d90cb 100644
--- a/lib/rainbows/configurator.rb
+++ b/lib/rainbows/configurator.rb
@@ -27,6 +27,7 @@ module Rainbows::Configurator
     :keepalive_requests => 100,
     :client_max_body_size => 1024 * 1024,
     :client_header_buffer_size => 1024,
+    :copy_stream => IO.respond_to?(:copy_stream) ? IO : false,
   })
 
   # Configures \Rainbows! with a given concurrency model to +use+ and
@@ -161,6 +162,44 @@ module Rainbows::Configurator
     check!
     set_int(:client_header_buffer_size, bytes, 1)
   end
+
+  # Allows overriding the +klass+ where the +copy_stream+ method is
+  # used to do efficient copying of regular files, pipes, and sockets.
+  #
+  # This is only used with multi-threaded concurrency models:
+  #
+  # * ThreadSpawn
+  # * ThreadPool
+  # * WriterThreadSpawn
+  # * WriterThreadPool
+  # * XEpollThreadSpawn
+  # * XEpollThreadPool
+  #
+  # Due to existing {bugs}[http://redmine.ruby-lang.org/search?q=copy_stream]
+  # in the Ruby IO.copy_stream implementation, \Rainbows! uses the
+  # "sendfile" RubyGem that instead of copy_stream to transfer regular files
+  # to clients.  The "sendfile" RubyGem also supports more operating systems,
+  # and works with more concurrency models.
+  #
+  # Recent Linux 2.6 users may override this with "IO::Splice" from the
+  # "io_splice" RubyGem:
+  #
+  #   require "io/splice"
+  #   Rainbows! do
+  #     copy_stream IO::Splice
+  #   end
+  #
+  # Keep in mind that splice(2) itself is a relatively new system call
+  # and has been buggy in many older Linux kernels.
+  #
+  # Default: IO on Ruby 1.9+, false otherwise
+  def copy_stream(klass)
+    check!
+    if klass && ! klass.respond_to?(:copy_stream)
+      abort "#{klass} must respond to `copy_stream' or be `false'"
+    end
+    set[:copy_stream] = klass
+  end
 end
 
 # :enddoc:
diff --git a/lib/rainbows/http_server.rb b/lib/rainbows/http_server.rb
index 62a5927..0fbc38f 100644
--- a/lib/rainbows/http_server.rb
+++ b/lib/rainbows/http_server.rb
@@ -2,6 +2,7 @@
 # :enddoc:
 
 class Rainbows::HttpServer < Unicorn::HttpServer
+  attr_accessor :copy_stream
   attr_accessor :worker_connections
   attr_accessor :keepalive_timeout
   attr_accessor :client_header_buffer_size
diff --git a/lib/rainbows/response.rb b/lib/rainbows/response.rb
index 65599e9..fac2c0e 100644
--- a/lib/rainbows/response.rb
+++ b/lib/rainbows/response.rb
@@ -6,6 +6,7 @@ module Rainbows::Response
   KeepAlive = "keep-alive"
   Content_Length = "Content-Length".freeze
   Transfer_Encoding = "Transfer-Encoding".freeze
+  Rainbows.config!(self, :copy_stream)
 
   # private file class for IO objects opened by Rainbows! itself (and not
   # the app or middleware)
@@ -67,7 +68,7 @@ module Rainbows::Response
     end
 
     # generic response writer, used for most dynamically-generated responses
-    # and also when IO.copy_stream and/or IO#trysendfile is unavailable
+    # and also when copy_stream and/or IO#trysendfile is unavailable
     def write_response(status, headers, body, alive)
       write_headers(status, headers, alive)
       write_body_each(body)
@@ -89,29 +90,29 @@ module Rainbows::Response
     include Sendfile
   end
 
-  if IO.respond_to?(:copy_stream)
+  if COPY_STREAM
     unless IO.method_defined?(:trysendfile)
       module CopyStream
         def write_body_file(body, range)
-          range ? IO.copy_stream(body, self, range[1], range[0]) :
-                  IO.copy_stream(body, self, nil, 0)
+          range ? COPY_STREAM.copy_stream(body, self, range[1], range[0]) :
+                  COPY_STREAM.copy_stream(body, self, nil, 0)
         end
       end
       include CopyStream
     end
 
-    # write_body_stream is an alias for write_body_each if IO.copy_stream
+    # write_body_stream is an alias for write_body_each if copy_stream
     # isn't used or available.
     def write_body_stream(body)
-      IO.copy_stream(io = body_to_io(body), self)
+      COPY_STREAM.copy_stream(io = body_to_io(body), self)
       ensure
         close_if_private(io)
     end
-  else # ! IO.respond_to?(:copy_stream)
+  else # ! COPY_STREAM
     alias write_body_stream write_body_each
-  end  # ! IO.respond_to?(:copy_stream)
+  end  # ! COPY_STREAM
 
-  if IO.method_defined?(:trysendfile) || IO.respond_to?(:copy_stream)
+  if IO.method_defined?(:trysendfile) || COPY_STREAM
     HTTP_RANGE = 'HTTP_RANGE'
     Content_Range = 'Content-Range'.freeze
 
@@ -181,5 +182,5 @@ module Rainbows::Response
       end
     end
     include ToPath
-  end # IO.respond_to?(:copy_stream) || IO.method_defined?(:trysendfile)
+  end # COPY_STREAM || IO.method_defined?(:trysendfile)
 end
diff --git a/lib/rainbows/writer_thread_pool/client.rb b/lib/rainbows/writer_thread_pool/client.rb
index f02826e..fd537ed 100644
--- a/lib/rainbows/writer_thread_pool/client.rb
+++ b/lib/rainbows/writer_thread_pool/client.rb
@@ -18,7 +18,7 @@ class Rainbows::WriterThreadPool::Client < Struct.new(:to_io, :q)
       }
     end
 
-    if IO.respond_to?(:copy_stream) || IO.method_defined?(:trysendfile)
+    if Rainbows::Response::COPY_STREAM
       def write_response(status, headers, body, alive)
         if body.respond_to?(:close)
           write_response_close(status, headers, body, alive)
diff --git a/lib/rainbows/writer_thread_spawn/client.rb b/lib/rainbows/writer_thread_spawn/client.rb
index e5c8854..de51c17 100644
--- a/lib/rainbows/writer_thread_spawn/client.rb
+++ b/lib/rainbows/writer_thread_spawn/client.rb
@@ -21,7 +21,7 @@ class Rainbows::WriterThreadSpawn::Client < Struct.new(:to_io, :q, :thr)
       }
     end
 
-    if IO.respond_to?(:copy_stream) || IO.method_defined?(:trysendfile)
+    if Rainbows::Response::COPY_STREAM
       def write_response(status, headers, body, alive)
         self.q ||= queue_writer
         if body.respond_to?(:close)
diff --git a/t/t0026-splice-copy_stream-byte-range.sh b/t/t0026-splice-copy_stream-byte-range.sh
new file mode 100644
index 0000000..70546b6
--- /dev/null
+++ b/t/t0026-splice-copy_stream-byte-range.sh
@@ -0,0 +1,40 @@
+#!/bin/sh
+. ./test-lib.sh
+test -r random_blob || die "random_blob required, run with 'make $0'"
+case $RUBY_VERSION in
+1.9.*) ;;
+*)
+        t_info "skipping $T since it can't IO::Splice.copy_stream"
+        exit 0
+        ;;
+esac
+check_splice
+
+case $model in
+ThreadSpawn|WriterThreadSpawn|ThreadPool|WriterThreadPool|Base) ;;
+XEpollThreadSpawn) ;;
+*)
+        t_info "skipping $T since it doesn't use copy_stream"
+        exit 0
+        ;;
+esac
+
+t_plan 13 "IO::Splice.copy_stream byte range response for $model"
+
+t_begin "setup and startup" && {
+        rtmpfiles out err
+        rainbows_setup $model
+        cat >> $unicorn_config <<EOF
+require "io/splice"
+Rainbows! do
+  copy_stream IO::Splice
+end
+def (::IO).copy_stream(*x); abort "NO"; end
+EOF
+
+        # can't load Rack::Lint here since it clobbers body#to_path
+        rainbows -E none -D large-file-response.ru -c $unicorn_config
+        rainbows_wait_start
+}
+
+. ./byte-range-common.sh
diff --git a/t/t0027-nil-copy_stream.sh b/t/t0027-nil-copy_stream.sh
new file mode 100644
index 0000000..03c7f0a
--- /dev/null
+++ b/t/t0027-nil-copy_stream.sh
@@ -0,0 +1,68 @@
+#!/bin/sh
+. ./test-lib.sh
+test -r random_blob || die "random_blob required, run with 'make $0'"
+
+case $model in
+ThreadSpawn|WriterThreadSpawn|ThreadPool|WriterThreadPool|Base) ;;
+XEpollThreadSpawn) ;;
+*)
+        t_info "skipping $T since it doesn't use copy_stream"
+        exit 0
+        ;;
+esac
+
+t_plan 7 "large file 'copy_stream nil' test for $model"
+
+t_begin "setup and startup" && {
+        rtmpfiles curl_out
+        rainbows_setup $model
+        cat >> $unicorn_config <<EOF
+Rainbows! do
+  copy_stream nil
+end
+EOF
+        rainbows -E none -D large-file-response.ru -c $unicorn_config
+        rainbows_wait_start
+}
+
+t_begin "read random blob sha1 and size" && {
+        random_blob_sha1=$(rsha1 < random_blob)
+        random_blob_size=$(wc -c < random_blob)
+}
+
+t_begin "send a series HTTP/1.1 requests sequentially" && {
+        for i in a b c
+        do
+                sha1=$( (curl -sSfv http://$listen/random_blob &&
+                         echo ok >$ok) | rsha1)
+                test $sha1 = $random_blob_sha1
+                test xok = x$(cat $ok)
+        done
+}
+
+# this was a problem during development
+t_begin "HTTP/1.0 test" && {
+        sha1=$( (curl -0 -sSfv http://$listen/random_blob &&
+                 echo ok >$ok) | rsha1)
+        test $sha1 = $random_blob_sha1
+        test xok = x$(cat $ok)
+}
+
+t_begin "HTTP/0.9 test" && {
+        (
+                printf 'GET /random_blob\r\n'
+                rsha1 < $fifo > $tmp &
+                wait
+                echo ok > $ok
+        ) | socat - TCP:$listen > $fifo
+        test $(cat $tmp) = $random_blob_sha1
+        test xok = x$(cat $ok)
+}
+
+t_begin "shutdown server" && {
+        kill -QUIT $rainbows_pid
+}
+
+t_begin "check stderr" && check_stderr
+
+t_done
diff --git a/t/test-lib.sh b/t/test-lib.sh
index 655f36b..be654d6 100644
--- a/t/test-lib.sh
+++ b/t/test-lib.sh
@@ -198,6 +198,33 @@ req_curl_chunked_upload_err_check () {
         fi
 }
 
+check_splice () {
+        case $(uname -s) in
+        Linux) ;;
+        *)
+                t_info "skipping $T since it's not Linux"
+                exit 0
+                ;;
+        esac
+
+        # we only allow splice on 2.6.32+
+        min=32 uname_r=$(uname -r)
+        case $uname_r in
+        2.6.*)
+                sub=$(expr "$uname_r" : '2\.6\.\(.*\)$')
+                if test $sub -lt $min
+                then
+                        t_info "skipping $T (Linux $(uname_r < 2.6.$min)"
+                        exit 0
+                fi
+                ;;
+        *)
+                t_info "skipping $T (Linux $uname_r < 2.6.$min)"
+                exit 0
+                ;;
+        esac
+}
+
 case $model in
 Rev) require_check rev Rev::VERSION ;;
 Coolio) require_check coolio Coolio::VERSION ;;
diff --git a/t/test_isolate.rb b/t/test_isolate.rb
index 3cc646f..fe2aebc 100644
--- a/t/test_isolate.rb
+++ b/t/test_isolate.rb
@@ -37,7 +37,12 @@ Isolate.now!(opts) do
     gem 'rack-fiber_pool', '0.9.1'
   end
 
-  gem 'sleepy_penguin', '2.0.0' if RUBY_PLATFORM =~ /linux/
+  if RUBY_PLATFORM =~ /linux/
+    gem 'sleepy_penguin', '2.0.0'
+
+    # is 2.6.32 new enough?
+    gem 'io_splice', '4.1.0' if `uname -r`.strip > '2.6.32'
+  end
 end
 
 $stdout.reopen(old_out)