From bd6b2263869c271113577b88d526c7c2a6f1455d Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Sat, 10 Jul 2010 02:41:46 -0700 Subject: updates for Rainbows! 0.95.0 Rainbows! 0.95.0 made some incompatible changes, so update everything. Unfortunately we have to avoid subclassing here. Tests use isolate now. --- GNUmakefile | 48 +++++++++++++++++++++++++----- Rakefile | 33 +++++++++++++++++++-- lib/zbatery.rb | 66 ++++++++++++++++++++++-------------------- t/.gitignore | 4 ++- t/GNUmakefile | 60 +++++++++++++++++++++++--------------- t/my-tap-lib.sh | 5 ++-- t/t0005-large-file-response.sh | 27 ++++++++++------- t/test-lib.sh | 49 +++++++++++++++++++++++-------- t/test_isolate.rb | 43 +++++++++++++++++++++++++++ zbatery.gemspec | 3 +- 10 files changed, 246 insertions(+), 92 deletions(-) create mode 100644 t/test_isolate.rb diff --git a/GNUmakefile b/GNUmakefile index 201d6f3..20ad53d 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -1,7 +1,9 @@ # use GNU Make to run tests in parallel, and without depending on RubyGems all:: +MRI = ruby RUBY = ruby RAKE = rake +RSYNC = rsync GIT_URL = git://git.bogomips.org/zbatery.git GIT-VERSION-FILE: .FORCE-GIT-VERSION-FILE @@ -14,9 +16,11 @@ endif ifeq ($(RUBY_VERSION),) RUBY_VERSION := $(shell $(RUBY) -e 'puts RUBY_VERSION') endif +RUBY_ENGINE := $(shell $(RUBY) -e 'puts((RUBY_ENGINE rescue "ruby"))') base_bins := zbatery bins := $(addprefix bin/, $(base_bins)) +man1_rdoc := $(addsuffix _1, $(base_bins)) man1_bins := $(addsuffix .1, $(base_bins)) man1_paths := $(addprefix man/man1/, $(man1_bins)) @@ -76,26 +80,52 @@ cgit_atom := http://git.bogomips.org/cgit/zbatery.git/atom/?h=master atom = -# using rdoc 2.4.1+ +# using rdoc 2.5.x+ doc: .document NEWS ChangeLog - for i in $(man1_bins); do > $$i; done - rdoc -Na -t "$(shell sed -ne '1s/^= //p' README)" + for i in $(man1_rdoc); do echo > $$i; done + find bin lib -type f -name '*.rbc' -exec rm -f '{}' ';' + rdoc -t "$(shell sed -ne '1s/^= //p' README)" install -m644 COPYING doc/COPYING install -m644 $(shell grep '^[A-Z]' .document) doc/ $(MAKE) -C Documentation install-html install-man install -m644 $(man1_paths) doc/ cd doc && for i in $(base_bins); do \ + $(RM) 1.html $${i}.1.html; \ sed -e '/"documentation">/r man1/'$$i'.1.html' \ - < $${i}_1.html > tmp && mv tmp $${i}_1.html; done - $(RUBY) -i -p -e \ + < $${i}_1.html > tmp && mv tmp $${i}_1.html; \ + ln $${i}_1.html $${i}.1.html; \ + done + $(MRI) -i -p -e \ '$$_.gsub!("",%q{\&$(call atom,$(cgit_atom))})' \ doc/ChangeLog.html - $(RUBY) -i -p -e \ + $(MRI) -i -p -e \ '$$_.gsub!("",%q{\&$(call atom,$(news_atom))})' \ doc/NEWS.html doc/README.html $(RAKE) -s news_atom > doc/NEWS.atom.xml cd doc && ln README.html tmp && mv tmp index.html - $(RM) $(man1_bins) + $(RM) $(man1_rdoc) + +# publishes docs to http://zbatery.bogomip.org/ +publish_doc: + -git set-file-times + $(RM) -r doc ChangeLog NEWS + $(MAKE) doc LOG_VERSION=$(shell git tag -l | tail -1) + awk 'BEGIN{RS="=== ";ORS=""}NR==2{sub(/\n$$/,"");print RS""$$0 }' \ + < NEWS > doc/LATEST + find doc/images doc/js -type f | \ + TZ=UTC xargs touch -d '1970-01-01 00:00:01' doc/rdoc.css + $(MAKE) doc_gz + chmod 644 $$(find doc -type f) + $(RSYNC) -av doc/ zbatery.bogomip.org:/srv/zbatery/ + git ls-files | xargs touch + +# Create gzip variants of the same timestamp as the original so nginx +# "gzip_static on" can serve the gzipped versions directly. +doc_gz: docs = $(shell find doc -type f ! -regex '^.*\.\(gif\|jpg\|png\|gz\)$$') +doc_gz: + touch doc/NEWS.atom.xml -d "$$(awk 'NR==1{print $$4,$$5,$$6}' NEWS)" + for i in $(docs); do \ + gzip --rsyncable -9 < $$i > $$i.gz; touch -r $$i $$i.gz; done ifneq ($(VERSION),) rfproject := rainbows @@ -152,11 +182,13 @@ release: verify package $(release_notes) $(release_changes) # make tgz release on RubyForge rubyforge add_release -f -n $(release_notes) -a $(release_changes) \ $(rfproject) $(rfpackage) $(VERSION) $(pkgtgz) - # push gem to Gemcutter + # push gem to RubyGems.org gem push $(pkggem) # in case of gem downloads from RubyForge releases page -rubyforge add_file \ $(rfproject) $(rfpackage) $(VERSION) $(pkggem) + $(RAKE) raa_update VERSION=$(VERSION) + $(RAKE) fm_update VERSION=$(VERSION) else gem install-gem: GIT-VERSION-FILE $(MAKE) $@ VERSION=$(GIT_VERSION) diff --git a/Rakefile b/Rakefile index 340ce38..a57f249 100644 --- a/Rakefile +++ b/Rakefile @@ -1,4 +1,5 @@ # -*- encoding: binary -*- +autoload :Gem, 'rubygems' # most tasks are in the GNUmakefile which offers better parallelism @@ -89,8 +90,6 @@ end desc "print release notes for Rubyforge" task :release_notes do - require 'rubygems' - spec = Gem::Specification.load('zbatery.gemspec') puts spec.description.strip puts "" @@ -121,7 +120,6 @@ end desc "post to RAA" task :raa_update do - require 'rubygems' require 'net/http' require 'net/netrc' rc = Net::Netrc.locate('zbatery-raa') or abort "~/.netrc not found" @@ -156,3 +154,32 @@ task :raa_update do p res puts res.body end + +desc "post to FM" +task :fm_update do + require 'tempfile' + require 'net/http' + require 'net/netrc' + require 'json' + version = ENV['VERSION'] or abort "VERSION= needed" + uri = URI.parse('http://freshmeat.net/projects/zbatery/releases.json') + rc = Net::Netrc.locate('zbatery-fm') or abort "~/.netrc not found" + api_token = rc.password + changelog = tags.find { |t| t[:tag] == "v#{version}" }[:body] + tmp = Tempfile.new('fm-changelog') + tmp.syswrite(changelog) + system(ENV["VISUAL"], tmp.path) or abort "#{ENV["VISUAL"]} failed: #$?" + changelog = File.read(tmp.path).strip + + req = { + "auth_code" => api_token, + "release" => { + "tag_list" => "Stable", + "version" => version, + "changelog" => changelog, + }, + }.to_json + Net::HTTP.start(uri.host, uri.port) do |http| + p http.post(uri.path, req, {'Content-Type'=>'application/json'}) + end +end diff --git a/lib/zbatery.rb b/lib/zbatery.rb index 75b03ba..224c9f1 100644 --- a/lib/zbatery.rb +++ b/lib/zbatery.rb @@ -1,4 +1,5 @@ # -*- encoding: binary -*- +# :enddoc: require 'rainbows' module Zbatery @@ -11,7 +12,7 @@ module Zbatery # runs the Zbatery HttpServer with +app+ and +options+ and does # not return until the server has exited. def run(app, options = {}) - HttpServer.new(app, options).start.join + Rainbows::HttpServer.new(app, options).start.join end end @@ -32,7 +33,37 @@ module Zbatery # config files... FORK_HOOK = lambda { |_,_| } - class HttpServer < Rainbows::HttpServer +end + +# :stopdoc: +# override stuff we don't need or can't use portably +module Rainbows + + module Base + # master == worker in our case + def init_worker_process(worker) + after_fork.call(self, worker) + worker.user(*user) if user.kind_of?(Array) && ! worker.switched + build_app! unless preload_app + Rainbows::Response.setup(self.class) + Rainbows::MaxBody.setup + + # avoid spurious wakeups and blocking-accept() with 1.8 green threads + if RUBY_VERSION.to_f < 1.9 + require "io/nonblock" + HttpServer::LISTENERS.each { |l| l.nonblock = true } + end + + logger.info "Zbatery #@use worker_connections=#@worker_connections" + end + end + + # we can't/don't need to do the fchmod heartbeat Unicorn/Rainbows! does + def G.tick + alive + end + + class HttpServer # this class is only used to avoid breaking Unicorn user switching class DeadIO @@ -115,47 +146,20 @@ module Zbatery def before_fork hook = super - hook == FORK_HOOK or + hook == Zbatery::FORK_HOOK or logger.warn "calling before_fork without forking" hook end def after_fork hook = super - hook == FORK_HOOK or + hook == Zbatery::FORK_HOOK or logger.warn "calling after_fork without having forked" hook end end end -# :stopdoc: -# override stuff we don't need or can't use portably -module Rainbows - - module Base - # master == worker in our case - def init_worker_process(worker) - after_fork.call(self, worker) - worker.user(*user) if user.kind_of?(Array) && ! worker.switched - build_app! unless preload_app - - # avoid spurious wakeups and blocking-accept() with 1.8 green threads - if RUBY_VERSION.to_f < 1.9 - require "io/nonblock" - HttpServer::LISTENERS.each { |l| l.nonblock = true } - end - - logger.info "Zbatery #@use worker_connections=#@worker_connections" - end - end - - # we can't/don't need to do the fchmod heartbeat Unicorn/Rainbows! does - def G.tick - alive - end -end - module Unicorn class Configurator diff --git a/t/.gitignore b/t/.gitignore index bee30c6..f810a71 100644 --- a/t/.gitignore +++ b/t/.gitignore @@ -1,4 +1,6 @@ /test-results-* -/test-bin-* +/bin-* /random_blob /.dep+* +/trash +/tmp diff --git a/t/GNUmakefile b/t/GNUmakefile index dbd78c4..74d3fbc 100644 --- a/t/GNUmakefile +++ b/t/GNUmakefile @@ -4,6 +4,7 @@ all:: pid := $(shell echo $$PPID) +MRI = ruby RUBY = ruby zbatery_lib := $(shell cd ../lib && pwd) -include ../local.mk @@ -15,13 +16,11 @@ ifeq ($(RUBY_VERSION),) $(error unable to detect RUBY_VERSION) endif -ifeq ($(RUBYLIB),) - RUBYLIB := $(zbatery_lib) -else - RUBYLIB := $(zbatery_lib):$(RUBYLIB) -endif -export RUBYLIB RUBY_VERSION +RUBY_ENGINE := $(shell $(RUBY) -e 'puts((RUBY_ENGINE rescue "ruby"))') +export RUBY_VERSION RUBY_ENGINE +models += WriterThreadPool +models += WriterThreadSpawn models += ThreadPool models += ThreadSpawn models += Rev @@ -30,13 +29,19 @@ models += NeverBlock models += RevThreadSpawn models += RevThreadPool -rp := ) -ONENINE := $(shell case $(RUBY_VERSION) in 1.9.*$(rp) echo true;;esac) -ifeq ($(ONENINE),true) - models += Revactor - models += FiberSpawn - models += RevFiberSpawn - models += FiberPool +ifeq ($(RUBY_ENGINE),ruby) + rp := ) + ONENINE := $(shell case $(RUBY_VERSION) in 1.9.*$(rp) echo true;;esac) + ifeq ($(ONENINE),true) + models += Revactor + models += FiberSpawn + models += RevFiberSpawn + models += FiberPool + endif +endif + +ifeq ($(RUBY_ENGINE),rbx) + models += ActorSpawn endif all_models := $(models) Base @@ -63,7 +68,7 @@ $(all_models): all:: $(T) # can't rely on "set -o pipefail" since we don't require bash or ksh93 :< -t_pfx = trash/$@-$(RUBY_VERSION) +t_pfx = trash/$@-$(RUBY_ENGINE)-$(RUBY_VERSION) TEST_OPTS = # TRACER = strace -f -o $(t_pfx).strace -s 100000 # TRACER = /usr/bin/time -o $(t_pfx).time @@ -76,11 +81,13 @@ ifdef V endif endif -test-bin-$(RUBY_VERSION)/zbatery: ruby_bin = $(shell which $(RUBY)) -test-bin-$(RUBY_VERSION)/zbatery: ../bin/zbatery +bindir := $(CURDIR)/bin-$(RUBY_ENGINE)-$(RUBY_VERSION) +bin_zbatery := $(bindir)/zbatery +$(bin_zbatery): ruby_bin = $(shell which $(RUBY)) +$(bin_zbatery): ../bin/zbatery mkdir -p $(@D) install -m 755 $^ $@.$(pid) - $(RUBY) -i -p -e '$$_.gsub!(%r{^#!.*$$},"#!$(ruby_bin)")' $@.$(pid) + $(MRI) -i -p -e '$$_.gsub!(%r{^#!.*$$},"#!$(ruby_bin)")' $@.$(pid) mv $@.$(pid) $@ random_blob: @@ -97,21 +104,28 @@ $(deps): @test -s $@.$(pid) || \ { echo >&2 "E '$(dep_bin)' not found in PATH=$(PATH)"; exit 1; } @mv $@.$(pid) $@ -dep: $(deps) + +libs := tmp/isolate/$(RUBY_ENGINE)-$(RUBY_VERSION)/.libs +$(libs): test_isolate.rb + mkdir -p $(@D) + $(RUBY) $< > $@+ + mv $@+ $@ +t_deps := $(libs) $(deps) $(bin_zbatery) trash/.gitignore +$(T): $(t_deps) $(MODEL_T): export model = $(firstword $(subst ., ,$@)) $(MODEL_T): script = $(subst $(model).,,$@) -$(MODEL_T): trash/.gitignore $(MODEL_T): export RUBY := $(RUBY) -$(MODEL_T): export PATH := $(CURDIR)/test-bin-$(RUBY_VERSION):$(PATH) -$(MODEL_T): test-bin-$(RUBY_VERSION)/zbatery dep - $(TRACER) $(SHELL) $(SH_TEST_OPTS) $(script) $(TEST_OPTS) +$(MODEL_T): export PATH := $(bindir):$(PATH) +$(MODEL_T): $(t_deps) + RUBYLIB=$(zbatery_lib):$$(cat $(libs)) \ + $(TRACER) $(SHELL) $(SH_TEST_OPTS) $(script) $(TEST_OPTS) trash/.gitignore: mkdir -p $(@D) echo '*' > $@ clean: - $(RM) -r trash/*.log trash/*.code test-bin-$(RUBY_VERSION) + $(RM) -r trash/*.log trash/*.code $(bindir) .PHONY: $(T) clean diff --git a/t/my-tap-lib.sh b/t/my-tap-lib.sh index ada77ac..3270095 100644 --- a/t/my-tap-lib.sh +++ b/t/my-tap-lib.sh @@ -188,9 +188,10 @@ then ( # use a subshell so seds are not waitable - $SED -e 's/^/#: /' $t_stdout & - $SED -e 's/^/#! /' $t_stderr & + $SED -e 's/^/#: /' < $t_stdout & + $SED -e 's/^/#! /' < $t_stderr & ) & + wait exec > $t_stdout 2> $t_stderr else exec > /dev/null 2> /dev/null diff --git a/t/t0005-large-file-response.sh b/t/t0005-large-file-response.sh index 2deba61..a869a38 100755 --- a/t/t0005-large-file-response.sh +++ b/t/t0005-large-file-response.sh @@ -12,13 +12,14 @@ t_plan 10 "large file response slurp avoidance for $model" t_begin "setup and startup" && { rtmpfiles curl_out - zbatery_setup $model + zbatery_setup $model 1 # can't load Rack::Lint here since it'll cause Rev to slurp zbatery -E none -D large-file-response.ru -c $unicorn_config zbatery_wait_start } -t_begin "read random blob size" && { +t_begin "read random blob sha1 and size" && { + random_blob_sha1=$(rsha1 < random_blob) random_blob_size=$(wc -c < random_blob) } @@ -32,29 +33,29 @@ t_begin "read current RSS" && { t_begin "send a series HTTP/1.1 requests sequentially" && { for i in a b c do - size=$( (curl -sSfv http://$listen/random_blob && - echo ok >$ok) |wc -c) - test $size -eq $random_blob_size + sha1=$( (curl -sSfv http://$listen/random_blob && + echo ok >$ok) | rsha1) + test $sha1 = $random_blob_sha1 test xok = x$(cat $ok) done } # this was a problem during development t_begin "HTTP/1.0 test" && { - size=$( (curl -0 -sSfv http://$listen/random_blob && - echo ok >$ok) |wc -c) - test $size -eq $random_blob_size + sha1=$( (curl -0 -sSfv http://$listen/random_blob && + echo ok >$ok) | rsha1) + test $sha1 = $random_blob_sha1 test xok = x$(cat $ok) } t_begin "HTTP/0.9 test" && { ( printf 'GET /random_blob\r\n' - cat $fifo > $tmp & + rsha1 < $fifo > $tmp & wait echo ok > $ok ) | socat - TCP:$listen > $fifo - cmp $tmp random_blob + test $(cat $tmp) = $random_blob_sha1 test xok = x$(cat $ok) } @@ -72,8 +73,12 @@ t_begin "shutdown server" && { t_begin "compare RSS before and after" && { diff=$(( $rss_after - $rss_before )) + + # default GC malloc limit in MRI: + fudge=$(( 8 * 1024 * 1024 )) + t_info "test diff=$diff < orig=$random_blob_size" - test $diff -le $random_blob_size + test $diff -le $(( $random_blob_size + $fudge )) } dbgcat r_err diff --git a/t/test-lib.sh b/t/test-lib.sh index 7ea07e7..0738b16 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -3,6 +3,12 @@ . ./my-tap-lib.sh set +u + +# sometimes we rely on http_proxy to avoid wasting bandwidth with Isolate +# and multiple Ruby versions +NO_PROXY=${UNICORN_TEST_ADDR-127.0.0.1} +export NO_PROXY + if test -z "$model" then # defaulting to Base would unfortunately fail some concurrency tests @@ -13,7 +19,7 @@ fi set -e RUBY="${RUBY-ruby}" RUBY_VERSION=${RUBY_VERSION-$($RUBY -e 'puts RUBY_VERSION')} -t_pfx=$PWD/trash/$model.$T-$RUBY_VERSION +t_pfx=$PWD/trash/$model.$T-$RUBY_ENGINE-$RUBY_VERSION set -u PATH=$PWD/bin:$PATH @@ -21,16 +27,6 @@ export PATH test -x $PWD/bin/unused_listen || die "must be run in 't' directory" -wait_for_pid () { - path="$1" - nr=30 - while ! test -s "$path" && test $nr -gt 0 - do - nr=$(($nr - 1)) - sleep 1 - done -} - # requires $1 and prints out the value of $2 require_check () { lib=$1 @@ -42,6 +38,19 @@ require_check () { fi } +skip_models () { + for i in "$@" + do + if test x"$model" != x"$i" + then + continue + fi + t_info "skipping $T since it is not compatible with $model" + exit 0 + done +} + + # given a list of variable names, create temporary files and assign # the pathnames to those variables rtmpfiles () { @@ -113,6 +122,7 @@ EOF # boxes and sometimes sleep 1s in tests kato=5 echo 'Rainbows! do' + echo " client_max_body_size nil" if test $# -ge 1 then echo " use :$1" @@ -145,7 +155,22 @@ rsha1 () { # last resort, see comments in sha1sum.rb for reasoning test -n "$_cmd" || _cmd=sha1sum.rb - expr "$($_cmd < random_blob)" : '\([a-f0-9]\{40\}\)' + expr "$($_cmd)" : '\([a-f0-9]\{40\}\)' +} + +req_curl_chunked_upload_err_check () { + set +e + curl --version 2>/dev/null | awk '$1 == "curl" { + split($2, v, /\./) + if ((v[1] < 7) || (v[1] == 7 && v[2] < 18)) + code = 1 + } + END { exit(code) }' + if test $? -ne 0 + then + t_info "curl >= 7.18.0 required for $T" + exit 0 + fi } case $model in diff --git a/t/test_isolate.rb b/t/test_isolate.rb new file mode 100644 index 0000000..a383bbc --- /dev/null +++ b/t/test_isolate.rb @@ -0,0 +1,43 @@ +require 'rubygems' +require 'isolate' +engine = defined?(RUBY_ENGINE) ? RUBY_ENGINE : 'ruby' + +path = "tmp/isolate/#{engine}-#{RUBY_VERSION}" +opts = { + :system => false, + # we want "ruby-1.8.7" and not "ruby-1.8", so disable multiruby + :multiruby => false, + :path => path, +} + +old_out = $stdout.dup +$stdout.reopen($stderr) + +Isolate.now!(opts) do + gem 'rack', '1.1.0' + gem 'unicorn', '1.1.0' + gem 'rainbows', '0.95.0' + + if engine == "ruby" + gem 'sendfile', '1.0.0' # next Rubinius should support this + + gem 'iobuffer', '0.1.3' + gem 'rev', '0.3.2' + + gem 'eventmachine', '0.12.10' + gem 'sinatra', '1.0.0' + gem 'async_sinatra', '0.2.1' + + gem 'neverblock', '0.1.6.2' + gem 'cramp', '0.11' + end + + if defined?(::Fiber) && engine == "ruby" + gem 'case', '0.5' + gem 'revactor', '0.1.5' + gem 'rack-fiber_pool', '0.9.0' + end +end + +$stdout.reopen(old_out) +puts Dir["#{path}/gems/*-*/lib"].map { |x| File.expand_path(x) }.join(':') diff --git a/zbatery.gemspec b/zbatery.gemspec index 67c595b..9b14c88 100644 --- a/zbatery.gemspec +++ b/zbatery.gemspec @@ -54,7 +54,8 @@ Gem::Specification.new do |s| # Unicorn were vulnerable to a remote DoS when exposed directly to # untrusted clients (a configuration only supported by Zbatery and Rainbows!, # Unicorn has never and will never be supported without trusted LAN clients. - s.add_dependency(%q, [">= 0.91.1", "<= 1.0.0"]) + s.add_dependency(%q, [">= 0.95.0", "<= 1.0.0"]) + s.add_development_dependency(%q, "~> 2.1.0") # s.licenses = %w(GPLv2 Ruby) # accessor not compatible with older RubyGems end -- cgit v1.2.3-24-ge0c7