From 181e4b5b6339fc5e9c3ad7d3690b736f6bd038aa Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Mon, 5 Jun 2023 10:12:50 +0000 Subject: [PATCH 21/23] tests: move test_upload.rb tests to t/integration.t The overread tests are ported over, and checksumming alone is enough to guard against data corruption. Randomizing the size of `read' calls on the client side will shake out any boundary bugs on the server side. --- t/integration.t | 32 ++++- test/unit/test_upload.rb | 301 --------------------------------------- 2 files changed, 27 insertions(+), 306 deletions(-) delete mode 100644 test/unit/test_upload.rb diff --git a/t/integration.t b/t/integration.t index 38a9675..a568758 100644 --- a/t/integration.t +++ b/t/integration.t @@ -26,7 +26,6 @@ POSIX::mkfifo($fifo, 0600) or die "mkfifo: $!"; my %PUT = ( chunked_md5 => sub { my ($in, $out, $path, %opt) = @_; - my $bs = $opt{bs} // 16384; my $dig = Digest::MD5->new; print $out < sub { my ($in, $out, $path, %opt) = @_; - my $bs = $opt{bs} // 16384; my $clen = $opt{-s} // -s $in; print $out < $bs ? $bs : $clen; $r = read($in, $buf, $len); die 'premature EOF' if $r == 0; @@ -192,8 +191,10 @@ SKIP: { my ($sub, $path, %opt) = @_; seek($rh, 0, SEEK_SET); $c = tcp_start($srv); - $c->autoflush(0); + $c->autoflush($opt{sync} // 0); $PUT{$sub}->($rh, $c, $path, %opt); + defined($opt{overwrite}) and + print { $c } ('x' x $opt{overwrite}); $c->flush or die $!; ($status, $hdr) = slurp_hdr($c); is(readline($c), $blob_hash, "$sub $path"); @@ -205,6 +206,27 @@ SKIP: { $ck_hash->('chunked_md5', '/rack_input/size_first'); $ck_hash->('chunked_md5', '/rack_input/rewind_first'); + $ck_hash->('identity', '/rack_input', -s => $blob_size, sync => 1); + $ck_hash->('chunked_md5', '/rack_input', sync => 1); + + # ensure small overwrites don't get checksummed + $ck_hash->('identity', '/rack_input', -s => $blob_size, + overwrite => 1); # one extra byte + + # excessive overwrite truncated + $c = tcp_start($srv); + $c->autoflush(0); + print $c "PUT /rack_input HTTP/1.0\r\nContent-Length: 1\r\n\r\n"; + if (1) { + local $SIG{PIPE} = 'IGNORE'; + my $buf = "\0" x 8192; + my $n = 0; + my $end = time + 5; + $! = 0; + while (print $c $buf and time < $end) { ++$n } + ok($!, 'overwrite truncated') or diag "n=$n err=$! ".time; + } + undef $c; $curl // skip 'no curl found in PATH', 1; diff --git a/test/unit/test_upload.rb b/test/unit/test_upload.rb deleted file mode 100644 index 76e6c1c..0000000 --- a/test/unit/test_upload.rb +++ /dev/null @@ -1,301 +0,0 @@ -# -*- encoding: binary -*- - -# Copyright (c) 2009 Eric Wong -require './test/test_helper' -require 'digest/md5' - -include Unicorn - -class UploadTest < Test::Unit::TestCase - - def setup - @addr = ENV['UNICORN_TEST_ADDR'] || '127.0.0.1' - @port = unused_port - @hdr = {'Content-Type' => 'text/plain', 'Content-Length' => '0'} - @bs = 4096 - @count = 256 - @server = nil - - # we want random binary data to test 1.9 encoding-aware IO craziness - @random = File.open('/dev/urandom','rb') - @sha1 = Digest::SHA1.new - @sha1_app = lambda do |env| - input = env['rack.input'] - resp = {} - - @sha1.reset - while buf = input.read(@bs) - @sha1.update(buf) - end - resp[:sha1] = @sha1.hexdigest - - # rewind and read again - input.rewind - @sha1.reset - while buf = input.read(@bs) - @sha1.update(buf) - end - - if resp[:sha1] == @sha1.hexdigest - resp[:sysread_read_byte_match] = true - end - - if expect_size = env['HTTP_X_EXPECT_SIZE'] - if expect_size.to_i == input.size - resp[:expect_size_match] = true - end - end - resp[:size] = input.size - resp[:content_md5] = env['HTTP_CONTENT_MD5'] - - [ 200, @hdr.merge({'X-Resp' => resp.inspect}), [] ] - end - end - - def teardown - redirect_test_io { @server.stop(false) } if @server - @random.close - reset_sig_handlers - end - - def test_put - start_server(@sha1_app) - sock = tcp_socket(@addr, @port) - sock.syswrite("PUT / HTTP/1.0\r\nContent-Length: #{length}\r\n\r\n") - @count.times do |i| - buf = @random.sysread(@bs) - @sha1.update(buf) - sock.syswrite(buf) - end - read = sock.read.split(/\r\n/) - assert_equal "HTTP/1.1 200 OK", read[0] - resp = eval(read.grep(/^X-Resp: /).first.sub!(/X-Resp: /, '')) - assert_equal length, resp[:size] - assert_equal @sha1.hexdigest, resp[:sha1] - end - - def test_put_content_md5 - md5 = Digest::MD5.new - start_server(@sha1_app) - sock = tcp_socket(@addr, @port) - sock.syswrite("PUT / HTTP/1.0\r\nTransfer-Encoding: chunked\r\n" \ - "Trailer: Content-MD5\r\n\r\n") - @count.times do |i| - buf = @random.sysread(@bs) - @sha1.update(buf) - md5.update(buf) - sock.syswrite("#{'%x' % buf.size}\r\n") - sock.syswrite(buf << "\r\n") - end - sock.syswrite("0\r\n") - - content_md5 = [ md5.digest! ].pack('m').strip.freeze - sock.syswrite("Content-MD5: #{content_md5}\r\n\r\n") - read = sock.read.split(/\r\n/) - assert_equal "HTTP/1.1 200 OK", read[0] - resp = eval(read.grep(/^X-Resp: /).first.sub!(/X-Resp: /, '')) - assert_equal length, resp[:size] - assert_equal @sha1.hexdigest, resp[:sha1] - assert_equal content_md5, resp[:content_md5] - end - - def test_put_trickle_small - @count, @bs = 2, 128 - start_server(@sha1_app) - assert_equal 256, length - sock = tcp_socket(@addr, @port) - hdr = "PUT / HTTP/1.0\r\nContent-Length: #{length}\r\n\r\n" - @count.times do - buf = @random.sysread(@bs) - @sha1.update(buf) - hdr << buf - sock.syswrite(hdr) - hdr = '' - sleep 0.6 - end - read = sock.read.split(/\r\n/) - assert_equal "HTTP/1.1 200 OK", read[0] - resp = eval(read.grep(/^X-Resp: /).first.sub!(/X-Resp: /, '')) - assert_equal length, resp[:size] - assert_equal @sha1.hexdigest, resp[:sha1] - end - - def test_put_keepalive_truncates_small_overwrite - start_server(@sha1_app) - sock = tcp_socket(@addr, @port) - to_upload = length + 1 - sock.syswrite("PUT / HTTP/1.0\r\nContent-Length: #{to_upload}\r\n\r\n") - @count.times do - buf = @random.sysread(@bs) - @sha1.update(buf) - sock.syswrite(buf) - end - sock.syswrite('12345') # write 4 bytes more than we expected - @sha1.update('1') - - buf = sock.readpartial(4096) - while buf !~ /\r\n\r\n/ - buf << sock.readpartial(4096) - end - read = buf.split(/\r\n/) - assert_equal "HTTP/1.1 200 OK", read[0] - resp = eval(read.grep(/^X-Resp: /).first.sub!(/X-Resp: /, '')) - assert_equal to_upload, resp[:size] - assert_equal @sha1.hexdigest, resp[:sha1] - end - - def test_put_excessive_overwrite_closed - tmp = Tempfile.new('overwrite_check') - tmp.sync = true - start_server(lambda { |env| - nr = 0 - while buf = env['rack.input'].read(65536) - nr += buf.size - end - tmp.write(nr.to_s) - [ 200, @hdr, [] ] - }) - sock = tcp_socket(@addr, @port) - buf = ' ' * @bs - sock.syswrite("PUT / HTTP/1.0\r\nContent-Length: #{length}\r\n\r\n") - - @count.times { sock.syswrite(buf) } - assert_raise(Errno::ECONNRESET, Errno::EPIPE) do - ::Unicorn::Const::CHUNK_SIZE.times { sock.syswrite(buf) } - end - sock.gets - tmp.rewind - assert_equal length, tmp.read.to_i - end - - # Despite reading numerous articles and inspecting the 1.9.1-p0 C - # source, Eric Wong will never trust that we're always handling - # encoding-aware IO objects correctly. Thus this test uses shell - # utilities that should always operate on files/sockets on a - # byte-level. - def test_uncomfortable_with_onenine_encodings - # POSIX doesn't require all of these to be present on a system - which('curl') or return - which('sha1sum') or return - which('dd') or return - - start_server(@sha1_app) - - tmp = Tempfile.new('dd_dest') - assert(system("dd", "if=#{@random.path}", "of=#{tmp.path}", - "bs=#{@bs}", "count=#{@count}"), - "dd #@random to #{tmp}") - sha1_re = %r!\b([a-f0-9]{40})\b! - sha1_out = `sha1sum #{tmp.path}` - assert $?.success?, 'sha1sum ran OK' - - assert_match(sha1_re, sha1_out) - sha1 = sha1_re.match(sha1_out)[1] - resp = `curl -isSfN -T#{tmp.path} http://#@addr:#@port/` - assert $?.success?, 'curl ran OK' - assert_match(%r!\b#{sha1}\b!, resp) - assert_match(/sysread_read_byte_match/, resp) - - # small StringIO path - assert(system("dd", "if=#{@random.path}", "of=#{tmp.path}", - "bs=1024", "count=1"), - "dd #@random to #{tmp}") - sha1_re = %r!\b([a-f0-9]{40})\b! - sha1_out = `sha1sum #{tmp.path}` - assert $?.success?, 'sha1sum ran OK' - - assert_match(sha1_re, sha1_out) - sha1 = sha1_re.match(sha1_out)[1] - resp = `curl -isSfN -T#{tmp.path} http://#@addr:#@port/` - assert $?.success?, 'curl ran OK' - assert_match(%r!\b#{sha1}\b!, resp) - assert_match(/sysread_read_byte_match/, resp) - end - - def test_chunked_upload_via_curl - # POSIX doesn't require all of these to be present on a system - which('curl') or return - which('sha1sum') or return - which('dd') or return - - start_server(@sha1_app) - - tmp = Tempfile.new('dd_dest') - assert(system("dd", "if=#{@random.path}", "of=#{tmp.path}", - "bs=#{@bs}", "count=#{@count}"), - "dd #@random to #{tmp}") - sha1_re = %r!\b([a-f0-9]{40})\b! - sha1_out = `sha1sum #{tmp.path}` - assert $?.success?, 'sha1sum ran OK' - - assert_match(sha1_re, sha1_out) - sha1 = sha1_re.match(sha1_out)[1] - cmd = "curl -H 'X-Expect-Size: #{tmp.size}' --tcp-nodelay \ - -isSf --no-buffer -T- " \ - "http://#@addr:#@port/" - resp = Tempfile.new('resp') - resp.sync = true - - rd, wr = IO.pipe.each do |io| - io.sync = io.close_on_exec = true - end - pid = spawn(*cmd, { 0 => rd, 1 => resp }) - rd.close - - tmp.rewind - @count.times { |i| - wr.write(tmp.read(@bs)) - sleep(rand / 10) if 0 == i % 8 - } - wr.close - pid, status = Process.waitpid2(pid) - - resp.rewind - resp = resp.read - assert status.success?, 'curl ran OK' - assert_match(%r!\b#{sha1}\b!, resp) - assert_match(/sysread_read_byte_match/, resp) - assert_match(/expect_size_match/, resp) - end - - def test_curl_chunked_small - # POSIX doesn't require all of these to be present on a system - which('curl') or return - which('sha1sum') or return - which('dd') or return - - start_server(@sha1_app) - - tmp = Tempfile.new('dd_dest') - # small StringIO path - assert(system("dd", "if=#{@random.path}", "of=#{tmp.path}", - "bs=1024", "count=1"), - "dd #@random to #{tmp}") - sha1_re = %r!\b([a-f0-9]{40})\b! - sha1_out = `sha1sum #{tmp.path}` - assert $?.success?, 'sha1sum ran OK' - - assert_match(sha1_re, sha1_out) - sha1 = sha1_re.match(sha1_out)[1] - resp = `curl -H 'X-Expect-Size: #{tmp.size}' --tcp-nodelay \ - -isSf --no-buffer -T- http://#@addr:#@port/ < #{tmp.path}` - assert $?.success?, 'curl ran OK' - assert_match(%r!\b#{sha1}\b!, resp) - assert_match(/sysread_read_byte_match/, resp) - assert_match(/expect_size_match/, resp) - end - - private - - def length - @bs * @count - end - - def start_server(app) - redirect_test_io do - @server = HttpServer.new(app, :listeners => [ "#{@addr}:#{@port}" ] ) - @server.start - end - end - -end