From 81026ea66279695206ea53287427c05281662572 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Sun, 9 Aug 2009 03:02:54 -0700 Subject: Switch to Ragel/C-based chunk/trailer parser This should be more robust, faster and easier to deal with than the ugly proof-of-concept regexp-based ones. --- test/unit/test_chunked_reader.rb | 123 --------------------------------------- test/unit/test_http_parser.rb | 105 ++++++++++++++++++--------------- test/unit/test_server.rb | 9 ++- test/unit/test_tee_input.rb | 21 ++++++- test/unit/test_trailer_parser.rb | 52 ----------------- 5 files changed, 85 insertions(+), 225 deletions(-) delete mode 100644 test/unit/test_chunked_reader.rb delete mode 100644 test/unit/test_trailer_parser.rb (limited to 'test') diff --git a/test/unit/test_chunked_reader.rb b/test/unit/test_chunked_reader.rb deleted file mode 100644 index a1323e9..0000000 --- a/test/unit/test_chunked_reader.rb +++ /dev/null @@ -1,123 +0,0 @@ -require 'test/unit' -require 'unicorn' -require 'unicorn_http' -require 'tempfile' -require 'io/nonblock' -require 'digest/sha1' - -class TestChunkedReader < Test::Unit::TestCase - - def setup - @env = {} - @rd, @wr = IO.pipe - @rd.binmode - @wr.binmode - @rd.sync = @wr.sync = true - @start_pid = $$ - end - - def teardown - return if $$ != @start_pid - @rd.close rescue nil - @wr.close rescue nil - begin - Process.wait - rescue Errno::ECHILD - break - end while true - end - - def test_error - cr = bin_reader("8\r\nasdfasdf\r\n8\r\nasdfasdfa#{'a' * 1024}") - a = nil - assert_nothing_raised { a = cr.readpartial(8192) } - assert_equal 'asdfasdf', a - assert_nothing_raised { a = cr.readpartial(8192) } - assert_equal 'asdfasdf', a - assert_raises(Unicorn::HttpParserError) { cr.readpartial(8192) } - end - - def test_eof1 - cr = bin_reader("0\r\n") - assert_raises(EOFError) { cr.readpartial(8192) } - end - - def test_eof2 - cr = bin_reader("0\r\n\r\n") - assert_raises(EOFError) { cr.readpartial(8192) } - end - - def test_readpartial1 - cr = bin_reader("4\r\nasdf\r\n0\r\n") - assert_equal 'asdf', cr.readpartial(8192) - assert_raises(EOFError) { cr.readpartial(8192) } - end - - def test_dd - cr = bin_reader("6\r\nhello\n\r\n") - tmp = Tempfile.new('test_dd') - tmp.sync = true - - pid = fork { - crd, cwr = IO.pipe - crd.binmode - cwr.binmode - crd.sync = cwr.sync = true - - pid = fork { - STDOUT.reopen(cwr) - crd.close - cwr.close - exec('dd', 'if=/dev/urandom', 'bs=93390', 'count=16') - } - cwr.close - begin - buf = crd.readpartial(16384) - tmp.write(buf) - @wr.write("#{'%x' % buf.size}\r\n#{buf}\r\n") - rescue EOFError - @wr.write("0\r\n\r\n") - Process.waitpid(pid) - exit 0 - end while true - } - assert_equal "hello\n", cr.readpartial(6) - sha1 = Digest::SHA1.new - buf = Unicorn::Z.dup - begin - cr.readpartial(16384, buf) - sha1.update(buf) - rescue EOFError - break - end while true - - assert_nothing_raised { Process.waitpid(pid) } - sha1_file = Digest::SHA1.new - File.open(tmp.path, 'rb') { |fp| - while fp.read(16384, buf) - sha1_file.update(buf) - end - } - assert_equal sha1_file.hexdigest, sha1.hexdigest - end - - def test_trailer - @env['HTTP_TRAILER'] = 'Content-MD5' - pid = fork { @wr.syswrite("Content-MD5: asdf\r\n") } - cr = bin_reader("8\r\nasdfasdf\r\n8\r\nasdfasdf\r\n0\r\n") - assert_equal 'asdfasdf', cr.readpartial(4096) - assert_equal 'asdfasdf', cr.readpartial(4096) - assert_raises(EOFError) { cr.readpartial(4096) } - pid, status = Process.waitpid2(pid) - assert status.success? - assert_equal 'asdf', @env['HTTP_CONTENT_MD5'] - end - -private - - def bin_reader(buf) - buf.force_encoding(Encoding::BINARY) if buf.respond_to?(:force_encoding) - Unicorn::ChunkedReader.new(@env, @rd, buf) - end - -end diff --git a/test/unit/test_http_parser.rb b/test/unit/test_http_parser.rb index 560f8d4..7072571 100644 --- a/test/unit/test_http_parser.rb +++ b/test/unit/test_http_parser.rb @@ -14,7 +14,8 @@ class HttpParserTest < Test::Unit::TestCase parser = HttpParser.new req = {} http = "GET / HTTP/1.1\r\n\r\n" - assert parser.execute(req, http) + assert_equal req, parser.headers(req, http) + assert_equal '', http assert_equal 'HTTP/1.1', req['SERVER_PROTOCOL'] assert_equal '/', req['REQUEST_PATH'] @@ -23,17 +24,18 @@ class HttpParserTest < Test::Unit::TestCase assert_equal 'GET', req['REQUEST_METHOD'] assert_nil req['FRAGMENT'] assert_equal '', req['QUERY_STRING'] - assert_nil req[:http_body] parser.reset req.clear - assert ! parser.execute(req, "G") + http = "G" + assert_nil parser.headers(req, http) + assert_equal "G", http assert req.empty? # try parsing again to ensure we were reset correctly http = "GET /hello-world HTTP/1.1\r\n\r\n" - assert parser.execute(req, http) + assert parser.headers(req, http) assert_equal 'HTTP/1.1', req['SERVER_PROTOCOL'] assert_equal '/hello-world', req['REQUEST_PATH'] @@ -42,52 +44,56 @@ class HttpParserTest < Test::Unit::TestCase assert_equal 'GET', req['REQUEST_METHOD'] assert_nil req['FRAGMENT'] assert_equal '', req['QUERY_STRING'] - assert_nil req[:http_body] + assert_equal '', http end def test_parse_server_host_default_port parser = HttpParser.new req = {} - assert parser.execute(req, "GET / HTTP/1.1\r\nHost: foo\r\n\r\n") + tmp = "GET / HTTP/1.1\r\nHost: foo\r\n\r\n" + assert_equal req, parser.headers(req, tmp) assert_equal 'foo', req['SERVER_NAME'] assert_equal '80', req['SERVER_PORT'] - assert_nil req[:http_body] + assert_equal '', tmp end def test_parse_server_host_alt_port parser = HttpParser.new req = {} - assert parser.execute(req, "GET / HTTP/1.1\r\nHost: foo:999\r\n\r\n") + tmp = "GET / HTTP/1.1\r\nHost: foo:999\r\n\r\n" + assert_equal req, parser.headers(req, tmp) assert_equal 'foo', req['SERVER_NAME'] assert_equal '999', req['SERVER_PORT'] - assert_nil req[:http_body] + assert_equal '', tmp end def test_parse_server_host_empty_port parser = HttpParser.new req = {} - assert parser.execute(req, "GET / HTTP/1.1\r\nHost: foo:\r\n\r\n") + tmp = "GET / HTTP/1.1\r\nHost: foo:\r\n\r\n" + assert_equal req, parser.headers(req, tmp) assert_equal 'foo', req['SERVER_NAME'] assert_equal '80', req['SERVER_PORT'] - assert_nil req[:http_body] + assert_equal '', tmp end def test_parse_server_host_xfp_https parser = HttpParser.new req = {} - assert parser.execute(req, "GET / HTTP/1.1\r\nHost: foo:\r\n" \ - "X-Forwarded-Proto: https\r\n\r\n") + tmp = "GET / HTTP/1.1\r\nHost: foo:\r\n" \ + "X-Forwarded-Proto: https\r\n\r\n" + assert_equal req, parser.headers(req, tmp) assert_equal 'foo', req['SERVER_NAME'] assert_equal '443', req['SERVER_PORT'] - assert_nil req[:http_body] + assert_equal '', tmp end def test_parse_strange_headers parser = HttpParser.new req = {} should_be_good = "GET / HTTP/1.1\r\naaaaaaaaaaaaa:++++++++++\r\n\r\n" - assert parser.execute(req, should_be_good) - assert_nil req[:http_body] + assert_equal req, parser.headers(req, should_be_good) + assert_equal '', should_be_good # ref: http://thread.gmane.org/gmane.comp.lang.ruby.mongrel.devel/37/focus=45 # (note we got 'pen' mixed up with 'pound' in that thread, @@ -110,8 +116,9 @@ class HttpParserTest < Test::Unit::TestCase parser = HttpParser.new req = {} sorta_safe = %(GET #{path} HTTP/1.1\r\n\r\n) - assert parser.execute(req, sorta_safe) - assert_nil req[:http_body] + assert_equal req, parser.headers(req, sorta_safe) + assert_equal path, req['REQUEST_URI'] + assert_equal '', sorta_safe end end @@ -120,30 +127,32 @@ class HttpParserTest < Test::Unit::TestCase req = {} bad_http = "GET / SsUTF/1.1" - assert_raises(HttpParserError) { parser.execute(req, bad_http) } + assert_raises(HttpParserError) { parser.headers(req, bad_http) } + + # make sure we can recover parser.reset - assert(parser.execute({}, "GET / HTTP/1.0\r\n\r\n")) - assert_nil req[:http_body] + req.clear + assert_equal req, parser.headers(req, "GET / HTTP/1.0\r\n\r\n") end def test_piecemeal parser = HttpParser.new req = {} http = "GET" - assert ! parser.execute(req, http) - assert_raises(HttpParserError) { parser.execute(req, http) } - assert ! parser.execute(req, http << " / HTTP/1.0") + assert_nil parser.headers(req, http) + assert_nil parser.headers(req, http) + assert_nil parser.headers(req, http << " / HTTP/1.0") assert_equal '/', req['REQUEST_PATH'] assert_equal '/', req['REQUEST_URI'] assert_equal 'GET', req['REQUEST_METHOD'] - assert ! parser.execute(req, http << "\r\n") + assert_nil parser.headers(req, http << "\r\n") assert_equal 'HTTP/1.0', req['HTTP_VERSION'] - assert ! parser.execute(req, http << "\r") - assert parser.execute(req, http << "\n") + assert_nil parser.headers(req, http << "\r") + assert_equal req, parser.headers(req, http << "\n") assert_equal 'HTTP/1.1', req['SERVER_PROTOCOL'] assert_nil req['FRAGMENT'] assert_equal '', req['QUERY_STRING'] - assert_nil req[:http_body] + assert_equal "", http end # not common, but underscores do appear in practice @@ -151,7 +160,7 @@ class HttpParserTest < Test::Unit::TestCase parser = HttpParser.new req = {} http = "GET http://under_score.example.com/foo?q=bar HTTP/1.0\r\n\r\n" - assert parser.execute(req, http) + assert_equal req, parser.headers(req, http) assert_equal 'http', req['rack.url_scheme'] assert_equal '/foo?q=bar', req['REQUEST_URI'] assert_equal '/foo', req['REQUEST_PATH'] @@ -160,14 +169,14 @@ class HttpParserTest < Test::Unit::TestCase assert_equal 'under_score.example.com', req['HTTP_HOST'] assert_equal 'under_score.example.com', req['SERVER_NAME'] assert_equal '80', req['SERVER_PORT'] - assert_nil req[:http_body] + assert_equal "", http end def test_absolute_uri parser = HttpParser.new req = {} http = "GET http://example.com/foo?q=bar HTTP/1.0\r\n\r\n" - assert parser.execute(req, http) + assert_equal req, parser.headers(req, http) assert_equal 'http', req['rack.url_scheme'] assert_equal '/foo?q=bar', req['REQUEST_URI'] assert_equal '/foo', req['REQUEST_PATH'] @@ -176,6 +185,7 @@ class HttpParserTest < Test::Unit::TestCase assert_equal 'example.com', req['HTTP_HOST'] assert_equal 'example.com', req['SERVER_NAME'] assert_equal '80', req['SERVER_PORT'] + assert_equal "", http end # X-Forwarded-Proto is not in rfc2616, absolute URIs are, however... @@ -184,7 +194,7 @@ class HttpParserTest < Test::Unit::TestCase req = {} http = "GET https://example.com/foo?q=bar HTTP/1.1\r\n" \ "X-Forwarded-Proto: http\r\n\r\n" - assert parser.execute(req, http) + assert_equal req, parser.headers(req, http) assert_equal 'https', req['rack.url_scheme'] assert_equal '/foo?q=bar', req['REQUEST_URI'] assert_equal '/foo', req['REQUEST_PATH'] @@ -193,6 +203,7 @@ class HttpParserTest < Test::Unit::TestCase assert_equal 'example.com', req['HTTP_HOST'] assert_equal 'example.com', req['SERVER_NAME'] assert_equal '443', req['SERVER_PORT'] + assert_equal "", http end # Host: header should be ignored for absolute URIs @@ -201,7 +212,7 @@ class HttpParserTest < Test::Unit::TestCase req = {} http = "GET http://example.com:8080/foo?q=bar HTTP/1.2\r\n" \ "Host: bad.example.com\r\n\r\n" - assert parser.execute(req, http) + assert_equal req, parser.headers(req, http) assert_equal 'http', req['rack.url_scheme'] assert_equal '/foo?q=bar', req['REQUEST_URI'] assert_equal '/foo', req['REQUEST_PATH'] @@ -210,6 +221,7 @@ class HttpParserTest < Test::Unit::TestCase assert_equal 'example.com:8080', req['HTTP_HOST'] assert_equal 'example.com', req['SERVER_NAME'] assert_equal '8080', req['SERVER_PORT'] + assert_equal "", http end def test_absolute_uri_with_empty_port @@ -217,7 +229,7 @@ class HttpParserTest < Test::Unit::TestCase req = {} http = "GET https://example.com:/foo?q=bar HTTP/1.1\r\n" \ "Host: bad.example.com\r\n\r\n" - assert parser.execute(req, http) + assert_equal req, parser.headers(req, http) assert_equal 'https', req['rack.url_scheme'] assert_equal '/foo?q=bar', req['REQUEST_URI'] assert_equal '/foo', req['REQUEST_PATH'] @@ -226,32 +238,33 @@ class HttpParserTest < Test::Unit::TestCase assert_equal 'example.com:', req['HTTP_HOST'] assert_equal 'example.com', req['SERVER_NAME'] assert_equal '443', req['SERVER_PORT'] + assert_equal "", http end def test_put_body_oneshot parser = HttpParser.new req = {} http = "PUT / HTTP/1.0\r\nContent-Length: 5\r\n\r\nabcde" - assert parser.execute(req, http) + assert_equal req, parser.headers(req, http) assert_equal '/', req['REQUEST_PATH'] assert_equal '/', req['REQUEST_URI'] assert_equal 'PUT', req['REQUEST_METHOD'] assert_equal 'HTTP/1.0', req['HTTP_VERSION'] assert_equal 'HTTP/1.1', req['SERVER_PROTOCOL'] - assert_equal "abcde", req[:http_body] + assert_equal "abcde", http end def test_put_body_later parser = HttpParser.new req = {} http = "PUT /l HTTP/1.0\r\nContent-Length: 5\r\n\r\n" - assert parser.execute(req, http) + assert_equal req, parser.headers(req, http) assert_equal '/l', req['REQUEST_PATH'] assert_equal '/l', req['REQUEST_URI'] assert_equal 'PUT', req['REQUEST_METHOD'] assert_equal 'HTTP/1.0', req['HTTP_VERSION'] assert_equal 'HTTP/1.1', req['SERVER_PROTOCOL'] - assert_equal "", req[:http_body] + assert_equal "", http end def test_unknown_methods @@ -261,13 +274,13 @@ class HttpParserTest < Test::Unit::TestCase s = "#{m} /forums/1/topics/2375?page=1#posts-17408 HTTP/1.1\r\n\r\n" ok = false assert_nothing_raised do - ok = parser.execute(req, s) + ok = parser.headers(req, s) end assert ok assert_equal '/forums/1/topics/2375?page=1', req['REQUEST_URI'] assert_equal 'posts-17408', req['FRAGMENT'] assert_equal 'page=1', req['QUERY_STRING'] - assert_equal "", req[:http_body] + assert_equal "", s assert_equal m, req['REQUEST_METHOD'] } end @@ -278,13 +291,13 @@ class HttpParserTest < Test::Unit::TestCase get = "GET /forums/1/topics/2375?page=1#posts-17408 HTTP/1.1\r\n\r\n" ok = false assert_nothing_raised do - ok = parser.execute(req, get) + ok = parser.headers(req, get) end assert ok assert_equal '/forums/1/topics/2375?page=1', req['REQUEST_URI'] assert_equal 'posts-17408', req['FRAGMENT'] assert_equal 'page=1', req['QUERY_STRING'] - assert_nil req[:http_body] + assert_equal '', get end # lame random garbage maker @@ -309,7 +322,7 @@ class HttpParserTest < Test::Unit::TestCase 10.times do |c| get = "GET /#{rand_data(10,120)} HTTP/1.1\r\nX-#{rand_data(1024, 1024+(c*1024))}: Test\r\n\r\n" assert_raises Unicorn::HttpParserError do - parser.execute({}, get) + parser.headers({}, get) parser.reset end end @@ -318,7 +331,7 @@ class HttpParserTest < Test::Unit::TestCase 10.times do |c| get = "GET /#{rand_data(10,120)} HTTP/1.1\r\nX-Test: #{rand_data(1024, 1024+(c*1024), false)}\r\n\r\n" assert_raises Unicorn::HttpParserError do - parser.execute({}, get) + parser.headers({}, get) parser.reset end end @@ -327,7 +340,7 @@ class HttpParserTest < Test::Unit::TestCase get = "GET /#{rand_data(10,120)} HTTP/1.1\r\n" get << "X-Test: test\r\n" * (80 * 1024) assert_raises Unicorn::HttpParserError do - parser.execute({}, get) + parser.headers({}, get) parser.reset end @@ -335,7 +348,7 @@ class HttpParserTest < Test::Unit::TestCase 10.times do |c| get = "GET #{rand_data(1024, 1024+(c*1024), false)} #{rand_data(1024, 1024+(c*1024), false)}\r\n\r\n" assert_raises Unicorn::HttpParserError do - parser.execute({}, get) + parser.headers({}, get) parser.reset end end diff --git a/test/unit/test_server.rb b/test/unit/test_server.rb index 22b9934..0b409ba 100644 --- a/test/unit/test_server.rb +++ b/test/unit/test_server.rb @@ -33,6 +33,8 @@ class WebServerTest < Test::Unit::TestCase def teardown redirect_test_io do + wait_workers_ready("test_stderr.#$$.log", 1) + File.truncate("test_stderr.#$$.log", 0) @server.stop(true) end end @@ -53,8 +55,10 @@ class WebServerTest < Test::Unit::TestCase end results = hit(["http://localhost:#@port/"]) worker_pid = results[0].to_i + assert worker_pid != 0 tmp.sysseek(0) loader_pid = tmp.sysread(4096).to_i + assert loader_pid != 0 assert_equal worker_pid, loader_pid teardown @@ -65,6 +69,7 @@ class WebServerTest < Test::Unit::TestCase end results = hit(["http://localhost:#@port/"]) worker_pid = results[0].to_i + assert worker_pid != 0 tmp.sysseek(0) loader_pid = tmp.sysread(4096).to_i assert_equal $$, loader_pid @@ -158,9 +163,9 @@ class WebServerTest < Test::Unit::TestCase do_test(long, Unicorn::Const::CHUNK_SIZE * 2 -400) end - def test_file_streamed_request_bad_method + def test_file_streamed_request_bad_body body = "a" * (Unicorn::Const::MAX_BODY * 2) - long = "GET /test HTTP/1.1\r\nContent-length: #{body.length}\r\n\r\n" + body + long = "GET /test HTTP/1.1\r\nContent-ength: #{body.length}\r\n\r\n" + body assert_raises(EOFError,Errno::ECONNRESET,Errno::EPIPE,Errno::EINVAL, Errno::EBADF) { do_test(long, Unicorn::Const::CHUNK_SIZE * 2 -400) diff --git a/test/unit/test_tee_input.rb b/test/unit/test_tee_input.rb index a6c61e6..3a6c998 100644 --- a/test/unit/test_tee_input.rb +++ b/test/unit/test_tee_input.rb @@ -26,7 +26,8 @@ class TestTeeInput < Test::Unit::TestCase end def test_gets_long - ti = Unicorn::TeeInput.new(@rd, nil, "hello") + init_parser("hello", 5 + (4096 * 4 * 3) + "#$/foo#$/".size) + ti = Unicorn::TeeInput.new(@rd) status = line = nil pid = fork { @rd.close @@ -46,7 +47,8 @@ class TestTeeInput < Test::Unit::TestCase end def test_gets_short - ti = Unicorn::TeeInput.new(@rd, nil, "hello") + init_parser("hello", 5 + "#$/foo".size) + ti = Unicorn::TeeInput.new(@rd) status = line = nil pid = fork { @rd.close @@ -63,4 +65,19 @@ class TestTeeInput < Test::Unit::TestCase assert status.success? end +private + + def init_parser(body, size = nil) + @parser = Unicorn::TeeInput::PARSER + @parser.reset + body = body.to_s.freeze + buf = "POST / HTTP/1.1\r\n" \ + "Host: localhost\r\n" \ + "Content-Length: #{size || body.size}\r\n" \ + "\r\n#{body}" + buf = Unicorn::TeeInput::RAW.replace(buf) + assert_equal @env, @parser.headers(@env, buf) + assert_equal body, buf + end + end diff --git a/test/unit/test_trailer_parser.rb b/test/unit/test_trailer_parser.rb deleted file mode 100644 index 5f3b16d..0000000 --- a/test/unit/test_trailer_parser.rb +++ /dev/null @@ -1,52 +0,0 @@ -require 'test/unit' -require 'unicorn' -require 'unicorn_http' -require 'unicorn/trailer_parser' - -class TestTrailerParser < Test::Unit::TestCase - - def test_basic - tp = Unicorn::TrailerParser.new('Content-MD5') - env = {} - assert ! tp.execute!(env, "Content-MD5: asdf") - assert env.empty? - assert tp.execute!(env, "Content-MD5: asdf\r\n") - assert_equal 'asdf', env['HTTP_CONTENT_MD5'] - assert_equal 1, env.size - end - - def test_invalid_trailer - tp = Unicorn::TrailerParser.new('Content-MD5') - env = {} - assert_raises(Unicorn::HttpParserError) { - tp.execute!(env, "Content-MD: asdf\r\n") - } - assert env.empty? - end - - def test_multiple_trailer - tp = Unicorn::TrailerParser.new('Foo,Bar') - env = {} - buf = "Bar: a\r\nFoo: b\r\n" - assert tp.execute!(env, buf) - assert_equal 'a', env['HTTP_BAR'] - assert_equal 'b', env['HTTP_FOO'] - end - - def test_too_big_key - tp = Unicorn::TrailerParser.new('Foo,Bar') - env = {} - buf = "Bar#{'a' * 1024}: a\r\nFoo: b\r\n" - assert_raises(Unicorn::HttpParserError) { tp.execute!(env, buf) } - assert env.empty? - end - - def test_too_big_value - tp = Unicorn::TrailerParser.new('Foo,Bar') - env = {} - buf = "Bar: #{'a' * (1024 * 1024)}: a\r\nFoo: b\r\n" - assert_raises(Unicorn::HttpParserError) { tp.execute!(env, buf) } - assert env.empty? - end - -end -- cgit v1.2.3-24-ge0c7