diff options
110 files changed, 2844 insertions, 2834 deletions
diff --git a/.olddoc.yml b/.olddoc.yml index d2d340f..9780a83 100644 --- a/.olddoc.yml +++ b/.olddoc.yml @@ -1,8 +1,9 @@ --- -cgit_url: https://bogomips.org/unicorn.git -git_url: https://bogomips.org/unicorn.git -rdoc_url: https://bogomips.org/unicorn/ -ml_url: https://bogomips.org/unicorn-public/ +cgit_url: https://yhbt.net/unicorn.git +rdoc_url: https://yhbt.net/unicorn/ +ml_url: +- https://yhbt.net/unicorn-public/ +- http://7fh6tueqddpjyxjmgtdiueylzoqt6pt7hec3pukyptlmohoowvhde4yd.onion/unicorn-public/ merge_html: unicorn_1: Documentation/unicorn.1.html unicorn_rails_1: Documentation/unicorn_rails.1.html @@ -11,7 +12,14 @@ noindex: - LATEST - TODO - unicorn_rails_1 -public_email: unicorn-public@bogomips.org +public_email: unicorn-public@yhbt.net +imap_url: +- imaps://;AUTH=ANONYMOUS@yhbt.net/inbox.comp.lang.ruby.unicorn.0 +- imap://;AUTH=ANONYMOUS@7fh6tueqddpjyxjmgtdiueylzoqt6pt7hec3pukyptlmohoowvhde4yd.onion/inbox.comp.lang.ruby.unicorn.0 nntp_url: - - nntp://news.public-inbox.org/inbox.comp.lang.ruby.unicorn - - nntp://news.gmane.org/gmane.comp.lang.ruby.unicorn.general +- nntps://news.public-inbox.org/inbox.comp.lang.ruby.unicorn +- nntp://7fh6tueqddpjyxjmgtdiueylzoqt6pt7hec3pukyptlmohoowvhde4yd.onion/inbox.comp.lang.ruby.unicorn +- nntp://news.gmane.io/gmane.comp.lang.ruby.unicorn.general +source_code: +- git clone https://yhbt.net/unicorn.git +- torsocks git clone http://7fh6tueqddpjyxjmgtdiueylzoqt6pt7hec3pukyptlmohoowvhde4yd.onion/unicorn.git diff --git a/CONTRIBUTORS b/CONTRIBUTORS index bda399b..9991fdc 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -1,5 +1,9 @@ -Unicorn developers (let us know if we forgot you): -* Eric Wong (BDFL, BOFH) +Unicorn developers (let us know if we forgot you, ...or if you no longer wish +to be associated with the doofus running this disaster :P): +* Eric Wong (Bozo Doofus For Life, Bastard Operator From Hell) + +There's numerous contributors over email the years, all of our mail +is archived @ https://yhbt.net/unicorn-public/ * Suraj N. Kurapati * Andrey Stikheev * Wayne Larsen @@ -1,5 +1,9 @@ == Design +Unicorn was designed to support poorly-written codebases back in 2008. +Its unfortunate popularity has only proliferated the existence of +poorly-written code ever since... + * Simplicity: Unicorn is a traditional UNIX prefork web server. No threads are used at all, this makes applications easier to debug and fix. When your application goes awry, a BOFH can just diff --git a/Documentation/.gitignore b/Documentation/.gitignore index 46679d6..0a3b033 100644 --- a/Documentation/.gitignore +++ b/Documentation/.gitignore @@ -1,5 +1,3 @@ -*.1 -*.5 -*.7 *.gz *.html +*.txt diff --git a/Documentation/GNUmakefile b/Documentation/GNUmakefile deleted file mode 100644 index 2c04bdb..0000000 --- a/Documentation/GNUmakefile +++ /dev/null @@ -1,30 +0,0 @@ -all:: - -PANDOC = pandoc -PANDOC_OPTS = -f markdown --email-obfuscation=none -pandoc = $(PANDOC) $(PANDOC_OPTS) -pandoc_html = $(pandoc) --toc -t html --no-wrap - -man1 := $(addsuffix .1,unicorn unicorn_rails) -html1 := $(addsuffix .html,$(man1)) - -all:: html man - -html: $(html1) -man: $(man1) - -install-html: html - mkdir -p ../doc/man1 - install -m 644 $(html1) ../doc/man1 - -install-man: man - mkdir -p ../man/man1 - install -m 644 $(man1) ../man/man1 - -%.1: %.1.txt - $(pandoc) -s -t man < $< > $@+ && mv $@+ $@ -%.1.html: %.1.txt - $(pandoc_html) < $< > $@+ && mv $@+ $@ - -clean:: - $(RM) $(man1) $(html1) diff --git a/Documentation/unicorn.1 b/Documentation/unicorn.1 new file mode 100644 index 0000000..b2c5e70 --- /dev/null +++ b/Documentation/unicorn.1 @@ -0,0 +1,222 @@ +.TH "UNICORN" "1" "September 15, 2009" "Unicorn User Manual" "" +.hy +.SH NAME +.PP +unicorn \- a rackup\-like command to launch the Unicorn HTTP server +.SH SYNOPSIS +.PP +unicorn [\-c CONFIG_FILE] [\-E RACK_ENV] [\-D] [RACKUP_FILE] +.SH DESCRIPTION +.PP +A rackup(1)\-like command to launch Rack applications using Unicorn. +It is expected to be started in your application root (APP_ROOT), +but the "working_directory" directive may be used in the CONFIG_FILE. +.PP +While unicorn takes a myriad of command\-line options for +compatibility with ruby(1) and rackup(1), it is recommended to stick +to the few command\-line options specified in the SYNOPSIS and use +the CONFIG_FILE as much as possible. +.SH RACKUP FILE +.PP +This defaults to "config.ru" in APP_ROOT. It should be the same +file used by rackup(1) and other Rack launchers, it uses the +\f[I]Rack::Builder\f[] DSL. +.PP +Embedded command\-line options are mostly parsed for compatibility +with rackup(1) but strongly discouraged. +.SH UNICORN OPTIONS +.TP +.B \-c, \-\-config\-file CONFIG_FILE +Path to the Unicorn\-specific config file. The config file is +implemented as a Ruby DSL, so Ruby code may executed. +See the RDoc/ri for the \f[I]Unicorn::Configurator\f[] class for the full +list of directives available from the DSL. +Using an absolute path for for CONFIG_FILE is recommended as it +makes multiple instances of Unicorn easily distinguishable when +viewing ps(1) output. +.RS +.RE +.TP +.B \-D, \-\-daemonize +Run daemonized in the background. The process is detached from +the controlling terminal and stdin is redirected to "/dev/null". +Unlike many common UNIX daemons, we do not chdir to "/" +upon daemonization to allow more control over the startup/upgrade +process. +Unless specified in the CONFIG_FILE, stderr and stdout will +also be redirected to "/dev/null". +.RS +.RE +.TP +.B \-E, \-\-env RACK_ENV +Run under the given RACK_ENV. See the RACK ENVIRONMENT section +for more details. +.RS +.RE +.TP +.B \-l, \-\-listen ADDRESS +Listens on a given ADDRESS. ADDRESS may be in the form of +HOST:PORT or PATH, HOST:PORT is taken to mean a TCP socket +and PATH is meant to be a path to a UNIX domain socket. +Defaults to "0.0.0.0:8080" (all addresses on TCP port 8080) +For production deployments, specifying the "listen" directive in +CONFIG_FILE is recommended as it allows fine\-tuning of socket +options. +.RS +.RE +.TP +.B \-N, \-\-no\-default\-middleware +Disables loading middleware implied by RACK_ENV. This bypasses the +configuration documented in the RACK ENVIRONMENT section, but still +allows RACK_ENV to be used for application/framework\-specific purposes. +.RS +.RE +.SH RACKUP COMPATIBILITY OPTIONS +.TP +.B \-o, \-\-host HOST +Listen on a TCP socket belonging to HOST, default is +"0.0.0.0" (all addresses). +If specified multiple times on the command\-line, only the +last\-specified value takes effect. +This option only exists for compatibility with the rackup(1) command, +use of "\-l"/"\-\-listen" switch is recommended instead. +.RS +.RE +.TP +.B \-p, \-\-port PORT +Listen on the specified TCP PORT, default is 8080. +If specified multiple times on the command\-line, only the last\-specified +value takes effect. +This option only exists for compatibility with the rackup(1) command, +use of "\-l"/"\-\-listen" switch is recommended instead. +.RS +.RE +.TP +.B \-s, \-\-server SERVER +No\-op, this exists only for compatibility with rackup(1). +.RS +.RE +.SH RUBY OPTIONS +.TP +.B \-e, \-\-eval LINE +Evaluate a LINE of Ruby code. This evaluation happens +immediately as the command\-line is being parsed. +.RS +.RE +.TP +.B \-d, \-\-debug +Turn on debug mode, the $DEBUG variable is set to true. +.RS +.RE +.TP +.B \-w, \-\-warn +Turn on verbose warnings, the $VERBOSE variable is set to true. +.RS +.RE +.TP +.B \-I, \-\-include PATH +specify $LOAD_PATH. PATH will be prepended to $LOAD_PATH. +The \[aq]:\[aq] character may be used to delimit multiple directories. +This directive may be used more than once. Modifications to +$LOAD_PATH take place immediately and in the order they were +specified on the command\-line. +.RS +.RE +.TP +.B \-r, \-\-require LIBRARY +require a specified LIBRARY before executing the application. The +"require" statement will be executed immediately and in the order +they were specified on the command\-line. +.RS +.RE +.SH SIGNALS +.PP +The following UNIX signals may be sent to the master process: +.IP \[bu] 2 +HUP \- reload config file, app, and gracefully restart all workers +.IP \[bu] 2 +INT/TERM \- quick shutdown, kills all workers immediately +.IP \[bu] 2 +QUIT \- graceful shutdown, waits for workers to finish their +current request before finishing. +.IP \[bu] 2 +USR1 \- reopen all logs owned by the master and all workers +See Unicorn::Util.reopen_logs for what is considered a log. +.IP \[bu] 2 +USR2 \- reexecute the running binary. A separate QUIT +should be sent to the original process once the child is verified to +be up and running. +.IP \[bu] 2 +WINCH \- gracefully stops workers but keep the master running. +This will only work for daemonized processes. +.IP \[bu] 2 +TTIN \- increment the number of worker processes by one +.IP \[bu] 2 +TTOU \- decrement the number of worker processes by one +.PP +See the SIGNALS (https://yhbt.net/unicorn/SIGNALS.html) document for +full description of all signals used by Unicorn. +.SH RACK ENVIRONMENT +.PP +Accepted values of RACK_ENV and the middleware they automatically load +(outside of RACKUP_FILE) are exactly as those in rackup(1): +.IP \[bu] 2 +development \- loads Rack::CommonLogger, Rack::ShowExceptions, and + Rack::Lint middleware +.IP \[bu] 2 +deployment \- loads Rack::CommonLogger middleware +.IP \[bu] 2 +none \- loads no middleware at all, relying entirely on RACKUP_FILE +.PP +All unrecognized values for RACK_ENV are assumed to be +"none". Production deployments are strongly encouraged to use +"deployment" or "none" for maximum performance. +.PP +As of Unicorn 0.94.0, RACK_ENV is exported as a process\-wide environment +variable as well. While not current a part of the Rack specification as +of Rack 1.0.1, this has become a de facto standard in the Rack world. +.PP +Note the Rack::ContentLength middleware is also +loaded by "deployment" and "development", but no other values of +RACK_ENV. If needed, they must be individually specified in the +RACKUP_FILE, some frameworks do not require them. +.SH ENVIRONMENT VARIABLES +.PP +The RACK_ENV variable is set by the aforementioned \-E switch. +All application or library\-specific environment variables (e.g. TMPDIR) +may always be set in the Unicorn CONFIG_FILE in addition to the spawning +shell. When transparently upgrading Unicorn, all environment variables +set in the old master process are inherited by the new master process. +Unicorn only uses (and will overwrite) the UNICORN_FD environment +variable internally when doing transparent upgrades. +.PP +UNICORN_FD is a comma\-delimited list of one or more file descriptors +used to implement USR2 upgrades. Init systems may bind listen sockets +itself and spawn unicorn with UNICORN_FD set to the file descriptor +numbers of the listen socket(s). +.PP +As of unicorn 5.0, LISTEN_PID and LISTEN_FDS are used for socket +activation as documented in the sd_listen_fds(3) manpage. Users +relying on this feature do not need to specify a listen socket in +the unicorn config file. +.SH SEE ALSO +.IP \[bu] 2 +\f[I]Rack::Builder\f[] ri/RDoc +.IP \[bu] 2 +\f[I]Unicorn::Configurator\f[] ri/RDoc +.UR https://yhbt.net/unicorn/Unicorn/Configurator.html +.UE +.IP \[bu] 2 +unicorn RDoc +.UR https://yhbt.net/unicorn/ +.UE +.IP \[bu] 2 +Rack RDoc +.UR https://www.rubydoc.info/github/rack/rack/ +.UE +.IP \[bu] 2 +Rackup HowTo +.UR https://github.com/rack/rack/wiki/(tutorial)-rackup-howto +.UE +.SH AUTHORS +The Unicorn Community <unicorn-public@yhbt.net>. diff --git a/Documentation/unicorn.1.txt b/Documentation/unicorn.1.txt deleted file mode 100644 index da7281d..0000000 --- a/Documentation/unicorn.1.txt +++ /dev/null @@ -1,187 +0,0 @@ -% UNICORN(1) Unicorn User Manual -% The Unicorn Community <unicorn-public@bogomips.org> -% September 15, 2009 - -# NAME - -unicorn - a rackup-like command to launch the Unicorn HTTP server - -# SYNOPSIS - -unicorn [-c CONFIG_FILE] [-E RACK_ENV] [-D] [RACKUP_FILE] - -# DESCRIPTION - -A rackup(1)-like command to launch Rack applications using Unicorn. -It is expected to be started in your application root (APP_ROOT), -but the "working_directory" directive may be used in the CONFIG_FILE. - -While unicorn takes a myriad of command-line options for -compatibility with ruby(1) and rackup(1), it is recommended to stick -to the few command-line options specified in the SYNOPSIS and use -the CONFIG_FILE as much as possible. - -# RACKUP FILE - -This defaults to \"config.ru\" in APP_ROOT. It should be the same -file used by rackup(1) and other Rack launchers, it uses the -*Rack::Builder* DSL. - -Embedded command-line options are mostly parsed for compatibility -with rackup(1) but strongly discouraged. - -# UNICORN OPTIONS --c, \--config-file CONFIG_FILE -: Path to the Unicorn-specific config file. The config file is - implemented as a Ruby DSL, so Ruby code may executed. - See the RDoc/ri for the *Unicorn::Configurator* class for the full - list of directives available from the DSL. - Using an absolute path for for CONFIG_FILE is recommended as it - makes multiple instances of Unicorn easily distinguishable when - viewing ps(1) output. - --D, \--daemonize -: Run daemonized in the background. The process is detached from - the controlling terminal and stdin is redirected to "/dev/null". - Unlike many common UNIX daemons, we do not chdir to \"/\" - upon daemonization to allow more control over the startup/upgrade - process. - Unless specified in the CONFIG_FILE, stderr and stdout will - also be redirected to "/dev/null". - --E, \--env RACK_ENV -: Run under the given RACK_ENV. See the RACK ENVIRONMENT section - for more details. - --l, \--listen ADDRESS -: Listens on a given ADDRESS. ADDRESS may be in the form of - HOST:PORT or PATH, HOST:PORT is taken to mean a TCP socket - and PATH is meant to be a path to a UNIX domain socket. - Defaults to "0.0.0.0:8080" (all addresses on TCP port 8080) - For production deployments, specifying the "listen" directive in - CONFIG_FILE is recommended as it allows fine-tuning of socket - options. --N, \--no-default-middleware -: Disables loading middleware implied by RACK_ENV. This bypasses the - configuration documented in the RACK ENVIRONMENT section, but still - allows RACK_ENV to be used for application/framework-specific purposes. - -# RACKUP COMPATIBILITY OPTIONS --o, \--host HOST -: Listen on a TCP socket belonging to HOST, default is - "0.0.0.0" (all addresses). - If specified multiple times on the command-line, only the - last-specified value takes effect. - This option only exists for compatibility with the rackup(1) command, - use of "-l"/"\--listen" switch is recommended instead. - --p, \--port PORT -: Listen on the specified TCP PORT, default is 8080. - If specified multiple times on the command-line, only the last-specified - value takes effect. - This option only exists for compatibility with the rackup(1) command, - use of "-l"/"\--listen" switch is recommended instead. - --s, \--server SERVER -: No-op, this exists only for compatibility with rackup(1). - -# RUBY OPTIONS --e, \--eval LINE -: Evaluate a LINE of Ruby code. This evaluation happens - immediately as the command-line is being parsed. - --d, \--debug -: Turn on debug mode, the $DEBUG variable is set to true. - --w, \--warn -: Turn on verbose warnings, the $VERBOSE variable is set to true. - --I, \--include PATH -: specify $LOAD_PATH. PATH will be prepended to $LOAD_PATH. - The \':\' character may be used to delimit multiple directories. - This directive may be used more than once. Modifications to - $LOAD_PATH take place immediately and in the order they were - specified on the command-line. - --r, \--require LIBRARY -: require a specified LIBRARY before executing the application. The - \"require\" statement will be executed immediately and in the order - they were specified on the command-line. - -# SIGNALS - -The following UNIX signals may be sent to the master process: - -* HUP - reload config file, app, and gracefully restart all workers -* INT/TERM - quick shutdown, kills all workers immediately -* QUIT - graceful shutdown, waits for workers to finish their - current request before finishing. -* USR1 - reopen all logs owned by the master and all workers - See Unicorn::Util.reopen_logs for what is considered a log. -* USR2 - reexecute the running binary. A separate QUIT - should be sent to the original process once the child is verified to - be up and running. -* WINCH - gracefully stops workers but keep the master running. - This will only work for daemonized processes. -* TTIN - increment the number of worker processes by one -* TTOU - decrement the number of worker processes by one - -See the [SIGNALS][4] document for full description of all signals -used by Unicorn. - -# RACK ENVIRONMENT - -Accepted values of RACK_ENV and the middleware they automatically load -(outside of RACKUP_FILE) are exactly as those in rackup(1): - -* development - loads Rack::CommonLogger, Rack::ShowExceptions, and - Rack::Lint middleware -* deployment - loads Rack::CommonLogger middleware -* none - loads no middleware at all, relying - entirely on RACKUP_FILE - -All unrecognized values for RACK_ENV are assumed to be -"none". Production deployments are strongly encouraged to use -"deployment" or "none" for maximum performance. - -As of Unicorn 0.94.0, RACK_ENV is exported as a process-wide environment -variable as well. While not current a part of the Rack specification as -of Rack 1.0.1, this has become a de facto standard in the Rack world. - -Note the Rack::ContentLength and Rack::Chunked middlewares are also -loaded by "deployment" and "development", but no other values of -RACK_ENV. If needed, they must be individually specified in the -RACKUP_FILE, some frameworks do not require them. - -# ENVIRONMENT VARIABLES - -The RACK_ENV variable is set by the aforementioned \-E switch. -All application or library-specific environment variables (e.g. TMPDIR) -may always be set in the Unicorn CONFIG_FILE in addition to the spawning -shell. When transparently upgrading Unicorn, all environment variables -set in the old master process are inherited by the new master process. -Unicorn only uses (and will overwrite) the UNICORN_FD environment -variable internally when doing transparent upgrades. - -UNICORN_FD is a comma-delimited list of one or more file descriptors -used to implement USR2 upgrades. Init systems may bind listen sockets -itself and spawn unicorn with UNICORN_FD set to the file descriptor -numbers of the listen socket(s). - -As of unicorn 5.0, LISTEN_PID and LISTEN_FDS are used for socket -activation as documented in the sd_listen_fds(3) manpage. Users -relying on this feature do not need to specify a listen socket in -the unicorn config file. - -# SEE ALSO - -* *Rack::Builder* ri/RDoc -* *Unicorn::Configurator* ri/RDoc -* [Unicorn RDoc][1] -* [Rack RDoc][2] -* [Rackup HowTo][3] - -[1]: https://bogomips.org/unicorn/ -[2]: https://www.rubydoc.info/github/rack/rack/ -[3]: https://github.com/rack/rack/wiki/tutorial-rackup-howto -[4]: https://bogomips.org/unicorn/SIGNALS.html diff --git a/Documentation/unicorn_rails.1 b/Documentation/unicorn_rails.1 new file mode 100644 index 0000000..fec0a2a --- /dev/null +++ b/Documentation/unicorn_rails.1 @@ -0,0 +1,207 @@ +.TH "UNICORN_RAILS" "1" "September 17, 2009" "Unicorn User Manual" "" +.hy +.SH NAME +.PP +unicorn_rails \- unicorn launcher for Rails 1.x and 2.x users +.SH SYNOPSIS +.PP +unicorn_rails [\-c CONFIG_FILE] [\-E RAILS_ENV] [\-D] [RACKUP_FILE] +.SH DESCRIPTION +.PP +A rackup(1)\-like command to launch ancient Rails (2.x and earlier) +applications using Unicorn. Rails 3 (and later) support Rack natively, +so users are encouraged to use unicorn(1) instead of unicorn_rails(1). +.PP +It is expected to be started in your Rails application root (RAILS_ROOT), +but the "working_directory" directive may be used in the CONFIG_FILE. +.PP +The outward interface resembles rackup(1), the internals and default +middleware loading is designed like the \f[C]script/server\f[] command +distributed with Rails. +.PP +While Unicorn takes a myriad of command\-line options for compatibility +with ruby(1) and rackup(1), it is recommended to stick to the few +command\-line options specified in the SYNOPSIS and use the CONFIG_FILE +as much as possible. +.SH UNICORN OPTIONS +.TP +.B \-c, \-\-config\-file CONFIG_FILE +Path to the Unicorn\-specific config file. The config file is +implemented as a Ruby DSL, so Ruby code may executed. +See the RDoc/ri for the \f[I]Unicorn::Configurator\f[] class for the full +list of directives available from the DSL. +Using an absolute path for for CONFIG_FILE is recommended as it +makes multiple instances of Unicorn easily distinguishable when +viewing ps(1) output. +.RS +.RE +.TP +.B \-D, \-\-daemonize +Run daemonized in the background. The process is detached from +the controlling terminal and stdin is redirected to "/dev/null". +Unlike many common UNIX daemons, we do not chdir to "/" upon +daemonization to allow more control over the startup/upgrade +process. +Unless specified in the CONFIG_FILE, stderr and stdout will +also be redirected to "/dev/null". +Daemonization will \f[I]skip\f[] loading of the +\f[I]Rails::Rack::LogTailer\f[] +middleware under Rails >= 2.3.x. +By default, unicorn_rails(1) will create a PID file in +\f[I]"RAILS_ROOT/tmp/pids/unicorn.pid"\f[]. You may override this +by specifying the "pid" directive to override this Unicorn config file. +.RS +.RE +.TP +.B \-E, \-\-env RAILS_ENV +Run under the given RAILS_ENV. This sets the RAILS_ENV environment +variable. Acceptable values are exactly those you expect in your Rails +application, typically "development" or "production". +.RS +.RE +.TP +.B \-l, \-\-listen ADDRESS +Listens on a given ADDRESS. ADDRESS may be in the form of +HOST:PORT or PATH, HOST:PORT is taken to mean a TCP socket +and PATH is meant to be a path to a UNIX domain socket. +Defaults to "0.0.0.0:8080" (all addresses on TCP port 8080). +For production deployments, specifying the "listen" directive in +CONFIG_FILE is recommended as it allows fine\-tuning of socket +options. +.RS +.RE +.SH RACKUP COMPATIBILITY OPTIONS +.TP +.B \-o, \-\-host HOST +Listen on a TCP socket belonging to HOST, default is +"0.0.0.0" (all addresses). +If specified multiple times on the command\-line, only the +last\-specified value takes effect. +This option only exists for compatibility with the rackup(1) command, +use of "\-l"/"\-\-listen" switch is recommended instead. +.RS +.RE +.TP +.B \-p, \-\-port PORT +Listen on the specified TCP PORT, default is 8080. +If specified multiple times on the command\-line, only the last\-specified +value takes effect. +This option only exists for compatibility with the rackup(1) command, +use of "\-l"/"\-\-listen" switch is recommended instead. +.RS +.RE +.TP +.B \-\-path PATH +Mounts the Rails application at the given PATH (instead of "/"). +This is equivalent to setting the RAILS_RELATIVE_URL_ROOT +environment variable. This is only supported under Rails 2.3 +or later at the moment. +.RS +.RE +.SH RUBY OPTIONS +.TP +.B \-e, \-\-eval LINE +Evaluate a LINE of Ruby code. This evaluation happens +immediately as the command\-line is being parsed. +.RS +.RE +.TP +.B \-d, \-\-debug +Turn on debug mode, the $DEBUG variable is set to true. +For Rails >= 2.3.x, this loads the \f[I]Rails::Rack::Debugger\f[] +middleware. +.RS +.RE +.TP +.B \-w, \-\-warn +Turn on verbose warnings, the $VERBOSE variable is set to true. +.RS +.RE +.TP +.B \-I, \-\-include PATH +specify $LOAD_PATH. PATH will be prepended to $LOAD_PATH. +The \[aq]:\[aq] character may be used to delimit multiple directories. +This directive may be used more than once. Modifications to +$LOAD_PATH take place immediately and in the order they were +specified on the command\-line. +.RS +.RE +.TP +.B \-r, \-\-require LIBRARY +require a specified LIBRARY before executing the application. The +"require" statement will be executed immediately and in the order +they were specified on the command\-line. +.RS +.RE +.SH RACKUP FILE +.PP +This defaults to "config.ru" in RAILS_ROOT. It should be the same +file used by rackup(1) and other Rack launchers, it uses the +\f[I]Rack::Builder\f[] DSL. Unlike many other Rack applications, RACKUP_FILE +is completely \f[I]optional\f[] for Rails, but may be used to disable +some of the default middleware for performance. +.PP +Embedded command\-line options are mostly parsed for compatibility +with rackup(1) but strongly discouraged. +.SH ENVIRONMENT VARIABLES +.PP +The RAILS_ENV variable is set by the aforementioned \-E switch. The +RAILS_RELATIVE_URL_ROOT is set by the aforementioned \-\-path switch. +Either of these variables may also be set in the shell or the Unicorn +CONFIG_FILE. All application or library\-specific environment variables +(e.g. TMPDIR, RAILS_ASSET_ID) may always be set in the Unicorn +CONFIG_FILE in addition to the spawning shell. When transparently +upgrading Unicorn, all environment variables set in the old master +process are inherited by the new master process. Unicorn only uses (and +will overwrite) the UNICORN_FD environment variable internally when +doing transparent upgrades. +.SH SIGNALS +.PP +The following UNIX signals may be sent to the master process: +.IP \[bu] 2 +HUP \- reload config file, app, and gracefully restart all workers +.IP \[bu] 2 +INT/TERM \- quick shutdown, kills all workers immediately +.IP \[bu] 2 +QUIT \- graceful shutdown, waits for workers to finish their current +request before finishing. +.IP \[bu] 2 +USR1 \- reopen all logs owned by the master and all workers +See Unicorn::Util.reopen_logs for what is considered a log. +.IP \[bu] 2 +USR2 \- reexecute the running binary. A separate QUIT +should be sent to the original process once the child is verified to +be up and running. +.IP \[bu] 2 +WINCH \- gracefully stops workers but keep the master running. +This will only work for daemonized processes. +.IP \[bu] 2 +TTIN \- increment the number of worker processes by one +.IP \[bu] 2 +TTOU \- decrement the number of worker processes by one +.PP +See the SIGNALS (https://yhbt.net/unicorn/SIGNALS.html) document for +full description of all signals used by Unicorn. +.SH SEE ALSO +.IP \[bu] 2 +unicorn(1) +.IP \[bu] 2 +\f[I]Rack::Builder\f[] ri/RDoc +.IP \[bu] 2 +\f[I]Unicorn::Configurator\f[] ri/RDoc +.UR https://yhbt.net/unicorn/Unicorn/Configurator.html +.UE +.IP \[bu] 2 +unicorn RDoc +.UR https://yhbt.net/unicorn/ +.UE +.IP \[bu] 2 +Rack RDoc +.UR https://www.rubydoc.info/github/rack/rack/ +.UE +.IP \[bu] 2 +Rackup HowTo +.UR https://github.com/rack/rack/wiki/(tutorial)-rackup-howto +.UE +.SH AUTHORS +The Unicorn Community <unicorn-public@yhbt.net>. diff --git a/Documentation/unicorn_rails.1.txt b/Documentation/unicorn_rails.1.txt deleted file mode 100644 index 0ce9bcf..0000000 --- a/Documentation/unicorn_rails.1.txt +++ /dev/null @@ -1,173 +0,0 @@ -% UNICORN_RAILS(1) Unicorn User Manual -% The Unicorn Community <unicorn-public@bogomips.org> -% September 17, 2009 - -# NAME - -unicorn_rails - unicorn launcher for Rails 1.x and 2.x users - -# SYNOPSIS - -unicorn_rails [-c CONFIG_FILE] [-E RAILS_ENV] [-D] [RACKUP_FILE] - -# DESCRIPTION - -A rackup(1)-like command to launch ancient Rails (2.x and earlier) -applications using Unicorn. Rails 3 (and later) support Rack natively, -so users are encouraged to use unicorn(1) instead of unicorn_rails(1). - -It is expected to be started in your Rails application root (RAILS_ROOT), -but the "working_directory" directive may be used in the CONFIG_FILE. - -The outward interface resembles rackup(1), the internals and default -middleware loading is designed like the `script/server` command -distributed with Rails. - -While Unicorn takes a myriad of command-line options for compatibility -with ruby(1) and rackup(1), it is recommended to stick to the few -command-line options specified in the SYNOPSIS and use the CONFIG_FILE -as much as possible. - -# UNICORN OPTIONS --c, \--config-file CONFIG_FILE -: Path to the Unicorn-specific config file. The config file is - implemented as a Ruby DSL, so Ruby code may executed. - See the RDoc/ri for the *Unicorn::Configurator* class for the full - list of directives available from the DSL. - Using an absolute path for for CONFIG_FILE is recommended as it - makes multiple instances of Unicorn easily distinguishable when - viewing ps(1) output. - --D, \--daemonize -: Run daemonized in the background. The process is detached from - the controlling terminal and stdin is redirected to "/dev/null". - Unlike many common UNIX daemons, we do not chdir to \"/\" - upon daemonization to allow more control over the startup/upgrade - process. - Unless specified in the CONFIG_FILE, stderr and stdout will - also be redirected to "/dev/null". - Daemonization will _skip_ loading of the *Rails::Rack::LogTailer* - middleware under Rails \>\= 2.3.x. - By default, unicorn\_rails(1) will create a PID file in - _\"RAILS\_ROOT/tmp/pids/unicorn.pid\"_. You may override this - by specifying the "pid" directive to override this Unicorn config file. - --E, \--env RAILS_ENV -: Run under the given RAILS_ENV. This sets the RAILS_ENV environment - variable. Acceptable values are exactly those you expect in your Rails - application, typically "development" or "production". - --l, \--listen ADDRESS -: Listens on a given ADDRESS. ADDRESS may be in the form of - HOST:PORT or PATH, HOST:PORT is taken to mean a TCP socket - and PATH is meant to be a path to a UNIX domain socket. - Defaults to "0.0.0.0:8080" (all addresses on TCP port 8080). - For production deployments, specifying the "listen" directive in - CONFIG_FILE is recommended as it allows fine-tuning of socket - options. - -# RACKUP COMPATIBILITY OPTIONS --o, \--host HOST -: Listen on a TCP socket belonging to HOST, default is - "0.0.0.0" (all addresses). - If specified multiple times on the command-line, only the - last-specified value takes effect. - This option only exists for compatibility with the rackup(1) command, - use of "-l"/"\--listen" switch is recommended instead. - --p, \--port PORT -: Listen on the specified TCP PORT, default is 8080. - If specified multiple times on the command-line, only the last-specified - value takes effect. - This option only exists for compatibility with the rackup(1) command, - use of "-l"/"\--listen" switch is recommended instead. - -\--path PATH -: Mounts the Rails application at the given PATH (instead of "/"). - This is equivalent to setting the RAILS_RELATIVE_URL_ROOT - environment variable. This is only supported under Rails 2.3 - or later at the moment. - -# RUBY OPTIONS --e, \--eval LINE -: Evaluate a LINE of Ruby code. This evaluation happens - immediately as the command-line is being parsed. - --d, \--debug -: Turn on debug mode, the $DEBUG variable is set to true. - For Rails \>\= 2.3.x, this loads the *Rails::Rack::Debugger* - middleware. - --w, \--warn -: Turn on verbose warnings, the $VERBOSE variable is set to true. - --I, \--include PATH -: specify $LOAD_PATH. PATH will be prepended to $LOAD_PATH. - The \':\' character may be used to delimit multiple directories. - This directive may be used more than once. Modifications to - $LOAD_PATH take place immediately and in the order they were - specified on the command-line. - --r, \--require LIBRARY -: require a specified LIBRARY before executing the application. The - \"require\" statement will be executed immediately and in the order - they were specified on the command-line. - -# RACKUP FILE - -This defaults to \"config.ru\" in RAILS_ROOT. It should be the same -file used by rackup(1) and other Rack launchers, it uses the -*Rack::Builder* DSL. Unlike many other Rack applications, RACKUP_FILE -is completely _optional_ for Rails, but may be used to disable some -of the default middleware for performance. - -Embedded command-line options are mostly parsed for compatibility -with rackup(1) but strongly discouraged. - -# ENVIRONMENT VARIABLES - -The RAILS_ENV variable is set by the aforementioned \-E switch. The -RAILS_RELATIVE_URL_ROOT is set by the aforementioned \--path switch. -Either of these variables may also be set in the shell or the Unicorn -CONFIG_FILE. All application or library-specific environment variables -(e.g. TMPDIR, RAILS_ASSET_ID) may always be set in the Unicorn -CONFIG_FILE in addition to the spawning shell. When transparently -upgrading Unicorn, all environment variables set in the old master -process are inherited by the new master process. Unicorn only uses (and -will overwrite) the UNICORN_FD environment variable internally when -doing transparent upgrades. - -# SIGNALS - -The following UNIX signals may be sent to the master process: - -* HUP - reload config file, app, and gracefully restart all workers -* INT/TERM - quick shutdown, kills all workers immediately -* QUIT - graceful shutdown, waits for workers to finish their - current request before finishing. -* USR1 - reopen all logs owned by the master and all workers - See Unicorn::Util.reopen_logs for what is considered a log. -* USR2 - reexecute the running binary. A separate QUIT - should be sent to the original process once the child is verified to - be up and running. -* WINCH - gracefully stops workers but keep the master running. - This will only work for daemonized processes. -* TTIN - increment the number of worker processes by one -* TTOU - decrement the number of worker processes by one - -See the [SIGNALS][4] document for full description of all signals -used by Unicorn. - -# SEE ALSO - -* unicorn(1) -* *Rack::Builder* ri/RDoc -* *Unicorn::Configurator* ri/RDoc -* [Unicorn RDoc][1] -* [Rack RDoc][2] -* [Rackup HowTo][3] - -[1]: https://bogomips.org/unicorn/ -[2]: https://www.rubydoc.info/github/rack/rack/ -[3]: https://github.com/rack/rack/wiki/tutorial-rackup-howto -[4]: https://bogomips.org/unicorn/SIGNALS.html @@ -7,7 +7,7 @@ drained entirely by the application. This may happen when request bodies are gzipped, as unicorn reads request body data lazily to avoid overhead from bad requests. -Ref: https://bogomips.org/unicorn-public/FC91211E-FD32-432C-92FC-0318714C2170@zendesk.com/ +Ref: https://yhbt.net/unicorn-public/FC91211E-FD32-432C-92FC-0318714C2170@zendesk.com/ === Why aren't my Rails log files rotated when I use SIGUSR1? diff --git a/GIT-VERSION-GEN b/GIT-VERSION-GEN index 9a534d3..d11b1e5 100755 --- a/GIT-VERSION-GEN +++ b/GIT-VERSION-GEN @@ -1,5 +1,5 @@ #!/usr/bin/env ruby -DEF_VER = "v5.5.0" +DEF_VER = "v6.1.0" CONSTANT = "Unicorn::Const::UNICORN_VERSION" RVF = "lib/unicorn/version.rb" GVF = "GIT-VERSION-FILE" diff --git a/GNUmakefile b/GNUmakefile index a7e4102..70e7e10 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -10,6 +10,8 @@ RAGEL = ragel RSYNC = rsync OLDDOC = olddoc RDOC = rdoc +INSTALL = install +PROVE = prove GIT-VERSION-FILE: .FORCE-GIT-VERSION-FILE @./GIT-VERSION-GEN @@ -25,7 +27,38 @@ endif RUBY_ENGINE := $(shell $(RUBY) -e 'puts((RUBY_ENGINE rescue "ruby"))') -MYLIBS = $(RUBYLIB) +# we should never package more than one ext to avoid DSO proliferation: +# https://udrepper.livejournal.com/8790.html +ext := $(firstword $(wildcard ext/*)) + +ragel: $(ext)/unicorn_http.c + +rl_files := $(wildcard $(ext)/*.rl) +ragel: $(ext)/unicorn_http.c +$(ext)/unicorn_http.c: $(rl_files) + cd $(@D) && $(RAGEL) unicorn_http.rl -C $(RLFLAGS) -o $(@F) +ext_pfx := test/$(RUBY_ENGINE)-$(RUBY_VERSION) +tmp_bin := $(ext_pfx)/bin +ext_h := $(wildcard $(ext)/*/*.h $(ext)/*.h) +ext_src := $(sort $(wildcard $(ext)/*.c) $(ext_h) $(ext)/unicorn_http.c) +ext_pfx_src := $(addprefix $(ext_pfx)/,$(ext_src)) +ext_dir := $(ext_pfx)/$(ext) +$(ext)/extconf.rb: + @>>$@ +$(ext_dir) $(tmp_bin) man/man1 doc/man1 pkg t/trash: + @mkdir -p $@ +$(ext_pfx)/$(ext)/%: $(ext)/% | $(ext_dir) + $(INSTALL) -m 644 $< $@ +$(ext_pfx)/$(ext)/Makefile: $(ext)/extconf.rb | $(ext_dir) + $(RM) -f $(@D)/*.o + cd $(@D) && $(RUBY) $(CURDIR)/$(ext)/extconf.rb $(EXTCONF_ARGS) +ext_sfx := _ext.$(DLEXT) +ext_dl := $(ext_pfx)/$(ext)/$(notdir $(ext)_ext.$(DLEXT)) +$(ext_dl): $(ext_src) $(ext_pfx_src) $(ext_pfx)/$(ext)/Makefile + $(MAKE) -C $(@D) +lib := $(CURDIR)/lib:$(CURDIR)/$(ext_pfx)/$(ext) +http build: $(ext_dl) +$(ext_pfx)/$(ext)/unicorn_http.c: ext/unicorn_http/unicorn_http.c # dunno how to implement this as concisely in Ruby, and hell, I love awk awk_slow := awk '/def test_/{print FILENAME"--"$$2".n"}' 2>/dev/null @@ -37,61 +70,80 @@ T := $(filter-out $(slow_tests), $(wildcard test/*/test*.rb)) T_n := $(shell $(awk_slow) $(slow_tests)) T_log := $(subst .rb,$(log_suffix),$(T)) T_n_log := $(subst .n,$(log_suffix),$(T_n)) -test_prefix = $(CURDIR)/test/$(RUBY_ENGINE)-$(RUBY_VERSION) -ext := ext/unicorn_http -c_files := $(ext)/unicorn_http.c $(ext)/httpdate.c $(wildcard $(ext)/*.h) -rl_files := $(wildcard $(ext)/*.rl) base_bins := unicorn unicorn_rails bins := $(addprefix bin/, $(base_bins)) man1_rdoc := $(addsuffix _1, $(base_bins)) man1_bins := $(addsuffix .1, $(base_bins)) man1_paths := $(addprefix man/man1/, $(man1_bins)) -rb_files := $(bins) $(shell find lib ext -type f -name '*.rb') -inst_deps := $(c_files) $(rb_files) GNUmakefile test/test_helper.rb +tmp_bins = $(addprefix $(tmp_bin)/, unicorn unicorn_rails) +pid := $(shell echo $$PPID) -ragel: $(ext)/unicorn_http.c -$(ext)/unicorn_http.c: $(rl_files) - cd $(@D) && $(RAGEL) unicorn_http.rl -C $(RLFLAGS) -o $(@F) -$(ext)/Makefile: $(ext)/extconf.rb $(c_files) - cd $(@D) && $(RUBY) extconf.rb -$(ext)/unicorn_http.$(DLEXT): $(ext)/Makefile - $(MAKE) -C $(@D) -http: $(ext)/unicorn_http.$(DLEXT) - -# only used for tests -http-install: $(ext)/unicorn_http.$(DLEXT) - install -m644 $< lib/ +$(tmp_bin)/%: bin/% | $(tmp_bin) + $(INSTALL) -m 755 $< $@.$(pid) + $(MRI) -i -p -e '$$_.gsub!(%r{^#!.*$$},"#!$(ruby_bin)")' $@.$(pid) + mv $@.$(pid) $@ -test-install: $(test_prefix)/.stamp -$(test_prefix)/.stamp: $(inst_deps) - mkdir -p $(test_prefix)/.ccache - tar cf - $(inst_deps) GIT-VERSION-GEN | \ - (cd $(test_prefix) && tar xf -) - $(MAKE) -C $(test_prefix) clean - $(MAKE) -C $(test_prefix) http-install shebang RUBY="$(RUBY)" - > $@ - -# this is only intended to be run within $(test_prefix) -shebang: $(bins) - $(MRI) -i -p -e '$$_.gsub!(%r{^#!.*$$},"#!$(ruby_bin)")' $^ +bins: $(tmp_bins) t_log := $(T_log) $(T_n_log) -test: $(T) $(T_n) +test: $(T) $(T_n) test-prove @cat $(t_log) | $(MRI) test/aggregate.rb @$(RM) $(t_log) test-exec: $(wildcard test/exec/test_*.rb) test-unit: $(wildcard test/unit/test_*.rb) -$(slow_tests): $(test_prefix)/.stamp +$(slow_tests): $(ext_dl) @$(MAKE) $(shell $(awk_slow) $@) # ensure we can require just the HTTP parser without the rest of unicorn -test-require: $(ext)/unicorn_http.$(DLEXT) - $(RUBY) --disable-gems -I$(ext) -runicorn_http -e Unicorn +test-require: $(ext_dl) + $(RUBY) --disable-gems -I$(ext_pfx)/$(ext) -runicorn_http -e Unicorn + +test_prereq := $(tmp_bins) $(ext_dl) + +SH_TEST_OPTS = +ifdef V + ifeq ($(V),2) + SH_TEST_OPTS += --trace + else + SH_TEST_OPTS += --verbose + endif +endif -test-integration: $(test_prefix)/.stamp - $(MAKE) -C t +# do we trust Ruby behavior to be stable? some tests are +# (mostly) POSIX sh (not bash or ksh93, so no "set -o pipefail" +# TRACER = strace -f -o $(t_pfx).strace -s 100000 +# TRACER = /usr/bin/time -o $(t_pfx).time +t_pfx = trash/$@-$(RUBY_ENGINE)-$(RUBY_VERSION) +T_sh = $(wildcard t/t[0-9][0-9][0-9][0-9]-*.sh) +$(T_sh): export RUBY := $(RUBY) +$(T_sh): export PATH := $(CURDIR)/$(tmp_bin):$(PATH) +$(T_sh): export RUBYLIB := $(lib):$(RUBYLIB) +$(T_sh): dep $(test_prereq) t/random_blob t/trash/.gitignore + cd t && $(TRACER) $(SHELL) $(SH_TEST_OPTS) $(@F) $(TEST_OPTS) + +t/trash/.gitignore : | t/trash + echo '*' >$@ + +dependencies := curl +deps := $(addprefix t/.dep+,$(dependencies)) +$(deps): dep_bin = $(lastword $(subst +, ,$@)) +$(deps): + @which $(dep_bin) > $@.$(pid) 2>/dev/null || : + @test -s $@.$(pid) || \ + { echo >&2 "E '$(dep_bin)' not found in PATH=$(PATH)"; exit 1; } + @mv $@.$(pid) $@ +dep: $(deps) + +t/random_blob: + dd if=/dev/urandom bs=1M count=30 of=$@.$(pid) + mv $@.$(pid) $@ + +test-integration: $(T_sh) + +test-prove: t/random_blob + $(PROVE) -vw check: test-require test test-integration test-all: check @@ -122,16 +174,16 @@ run_test = $(quiet_pre) \ %.n: arg = $(subst .n,,$(subst --, -n ,$@)) %.n: t = $(subst .n,$(log_suffix),$@) -%.n: export PATH := $(test_prefix)/bin:$(PATH) -%.n: export RUBYLIB := $(test_prefix)/lib:$(MYLIBS) -%.n: $(test_prefix)/.stamp +%.n: export PATH := $(CURDIR)/$(tmp_bin):$(PATH) +%.n: export RUBYLIB := $(lib):$(RUBYLIB) +%.n: $(test_prereq) $(run_test) $(T): arg = $@ $(T): t = $(subst .rb,$(log_suffix),$@) -$(T): export PATH := $(test_prefix)/bin:$(PATH) -$(T): export RUBYLIB := $(test_prefix)/lib:$(MYLIBS) -$(T): $(test_prefix)/.stamp +$(T): export PATH := $(CURDIR)/$(tmp_bin):$(PATH) +$(T): export RUBYLIB := $(lib):$(RUBYLIB) +$(T): $(test_prereq) $(run_test) install: $(bins) $(ext)/unicorn_http.c @@ -150,13 +202,21 @@ prep_setup_rb := @-$(RM) $(setup_rb_files);$(MAKE) -C $(ext) clean clean: -$(MAKE) -C $(ext) clean - -$(MAKE) -C Documentation clean $(RM) $(ext)/Makefile $(RM) $(setup_rb_files) $(t_log) - $(RM) -r $(test_prefix) man + $(RM) -r $(ext_pfx) man t/trash + $(RM) $(html1) + +man1 := $(addprefix Documentation/, unicorn.1 unicorn_rails.1) +html1 := $(addsuffix .html, $(man1)) +man : $(man1) | man/man1 + $(INSTALL) -m 644 $(man1) man/man1 + +html : $(html1) | doc/man1 + $(INSTALL) -m 644 $(html1) doc/man1 -man html: - $(MAKE) -C Documentation install-$@ +%.1.html: %.1 + $(OLDDOC) man2html -o $@ ./$< pkg_extra := GIT-VERSION-FILE lib/unicorn/version.rb LATEST NEWS \ $(ext)/unicorn_http.c $(man1_paths) @@ -177,19 +237,20 @@ doc: .document $(ext)/unicorn_http.c man html .olddoc.yml $(PLACEHOLDERS) $(OLDDOC) prepare $(RDOC) -f dark216 $(OLDDOC) merge - install -m644 COPYING doc/COPYING - install -m644 NEWS.atom.xml doc/NEWS.atom.xml - install -m644 $(shell LC_ALL=C grep '^[A-Z]' .document) doc/ - install -m644 $(man1_paths) doc/ + $(INSTALL) -m 644 COPYING doc/COPYING + $(INSTALL) -m 644 NEWS.atom.xml doc/NEWS.atom.xml + $(INSTALL) -m 644 $(shell LC_ALL=C grep '^[A-Z]' .document) doc/ + $(INSTALL) -m 644 $(man1_paths) doc/ tar cf - $$(git ls-files examples/) | (cd doc && tar xf -) -# publishes docs to https://bogomips.org/unicorn/ +# publishes docs to https://yhbt.net/unicorn/ publish_doc: -git set-file-times $(MAKE) doc $(MAKE) doc_gz chmod 644 $$(find doc -type f) - $(RSYNC) -av doc/ bogomips.org:/srv/bogomips/unicorn/ + $(RSYNC) -av doc/ yhbt.net:/srv/yhbt/unicorn/ \ + --exclude index.html* --exclude created.rid* git ls-files | xargs touch # Create gzip variants of the same timestamp as the original so nginx @@ -221,9 +282,8 @@ gem: $(pkggem) install-gem: $(pkggem) gem install --local $(CURDIR)/$< -$(pkggem): .manifest fix-perms +$(pkggem): .manifest fix-perms | pkg gem build $(rfpackage).gemspec - mkdir -p pkg mv $(@F) $@ $(pkgtgz): distdir = $(basename $@) @@ -254,5 +314,4 @@ check-warnings: do $(RUBY) --disable-gems -d -W2 -c \ $$i; done) | grep -v '^Syntax OK$$' || : -.PHONY: .FORCE-GIT-VERSION-FILE doc $(T) $(slow_tests) man -.PHONY: test-install +.PHONY: .FORCE-GIT-VERSION-FILE doc $(T) $(slow_tests) man $(T_sh) clean @@ -50,20 +50,17 @@ programming experience will come in handy (or be learned) here. === Documentation -Due to the lack of RDoc-to-manpage converters we know about, we're -writing manpages in Markdown and converting to troff/HTML with Pandoc. - Please wrap documentation at 72 characters-per-line or less (long URLs are exempt) so it is comfortably readable from terminals. When referencing mailing list posts, use -<tt>https://bogomips.org/unicorn-public/$MESSAGE_ID/</tt> if possible +<tt>https://yhbt.net/unicorn-public/$MESSAGE_ID/</tt> if possible since the Message-ID remains searchable even if a particular site becomes unavailable. === Ruby/C Compatibility -We target mainline Ruby 1.9.3 and later. We need the Ruby +We target C Ruby 2.5 and later. We need the Ruby implementation to support fork, exec, pipe, UNIX signals, access to integer file descriptors and ability to use unlinked files. @@ -102,10 +99,6 @@ don't email the git mailing list or maintainer with Unicorn patches :) == Building a Gem -In order to build the gem, you must install the following components: - - * pandoc - You can build the Unicorn gem with the following command: gmake gem @@ -1,9 +1,9 @@ = Issues -mailto:unicorn-public@bogomips.org is the best place to report bugs, +mailto:unicorn-public@yhbt.net is the best place to report bugs, submit patches and/or obtain support after you have searched the -{email archives}[https://bogomips.org/unicorn-public/] and -{documentation}[https://bogomips.org/unicorn/]. +{email archives}[https://yhbt.net/unicorn-public/] and +{documentation}[https://yhbt.net/unicorn/]. * No subscription will ever be required to email us * Cc: all participants in a thread or commit, as subscription is optional @@ -12,14 +12,18 @@ submit patches and/or obtain support after you have searched the * Do not send HTML mail or images, they hurt reader privacy and will be flagged as spam * Anonymous and pseudonymous messages will ALWAYS be welcome -* The email submission port (587) is enabled on the bogomips.org MX: - https://bogomips.org/unicorn-public/20141004232241.GA23908@dcvr.yhbt.net/t/ +* The email submission port (587) is enabled on the yhbt.net MX: + https://yhbt.net/unicorn-public/20141004232241.GA23908@dcvr.yhbt.net/t/ We will never have a centralized or formal bug tracker. Instead we can interoperate with any bug tracker which can Cc: us plain-text to -mailto:unicorn-public@bogomips.org This includes the Debian BTS +mailto:unicorn-public@yhbt.net This includes the Debian BTS at https://bugs.debian.org/unicorn and possibly others. +unicorn is a server; it does not depend on graphics/audio. Nobody +communicating with us will ever be expected to go through the trouble +of setting up graphics nor audio support. + If your issue is of a sensitive nature or you're just shy in public, use anonymity tools such as Tor or Mixmaster; and rely on the public mail archives for responses. Be sure to scrub sensitive log messages @@ -28,6 +32,10 @@ and such. If you don't get a response within a few days, we may have forgotten about it so feel free to ask again. +The project does not and will never endorse nor promote commercial +services (including support). The author of unicorn must never be +allowed to profit off the damage it's done to the entire Ruby world. + == Bugs in related projects unicorn is sometimes affected by bugs in its dependencies. Bugs @@ -61,7 +69,7 @@ There is a kernel.org Bugzilla instance, but it is ignored by most. Likewise for any rare glibc bugs we might encounter, we should Cc: mailto:libc-alpha@sourceware.org -Unofficial archives are available at: https://public-inbox.org/libc-alpha/ +Archives are available at: https://inbox.sourceware.org/libc-alpha/ Keep in mind glibc upstream does use Bugzilla for tracking bugs: https://sourceware.org/bugzilla/ @@ -73,24 +81,22 @@ document distributed with git) on guidelines for patch submission. == Contact Info -* public: mailto:unicorn-public@bogomips.org -* nntp://news.gmane.org/gmane.comp.lang.ruby.unicorn.general -* nntp://news.public-inbox.org/inbox.comp.lang.ruby.unicorn -* https://bogomips.org/unicorn-public/ -* http://ou63pmih66umazou.onion/unicorn-public/ - -Mailing list subscription is optional, so Cc: all participants. - -You can follow along via NNTP (read-only): +Mail is publicly-archived, SMTP subscription is discouraged to avoid +servers being a single-point-of-failure, so Cc: all participants. - nntp://news.public-inbox.org/inbox.comp.lang.ruby.unicorn - nntp://news.gmane.org/gmane.comp.lang.ruby.unicorn.general +The HTTP(S) archives have links to per-thread Atom feeds and downloadable +mboxes. Read-only IMAP(S) folders, POP3, and NNTP(S) newsgroups are available. -Or Atom feeds: +* https://yhbt.net/unicorn-public/ +* http://7fh6tueqddpjyxjmgtdiueylzoqt6pt7hec3pukyptlmohoowvhde4yd.onion/unicorn-public/ +* imaps://;AUTH=ANONYMOUS@yhbt.net/inbox.comp.lang.ruby.unicorn.0 +* imap://;AUTH=ANONYMOUS@7fh6tueqddpjyxjmgtdiueylzoqt6pt7hec3pukyptlmohoowvhde4yd.onion/inbox.comp.lang.ruby.unicorn.0 +* nntps://news.public-inbox.org/inbox.comp.lang.ruby.unicorn +* nntp://news.gmane.io/gmane.comp.lang.ruby.unicorn.general +* https://yhbt.net/unicorn-public/_/text/help/#pop3 - https://bogomips.org/unicorn-public/new.atom - http://ou63pmih66umazou.onion/unicorn-public/new.atom +Full Atom feeds: +* https://yhbt.net/unicorn-public/new.atom +* http://7fh6tueqddpjyxjmgtdiueylzoqt6pt7hec3pukyptlmohoowvhde4yd.onion/unicorn-public/new.atom - The HTML archives at https://bogomips.org/unicorn-public/ - also has links to per-thread Atom feeds and downloadable - mboxes. +We only accept plain-text mail: mailto:unicorn-public@yhbt.net diff --git a/KNOWN_ISSUES b/KNOWN_ISSUES index ebd4822..0017f20 100644 --- a/KNOWN_ISSUES +++ b/KNOWN_ISSUES @@ -9,7 +9,7 @@ acceptable solution. Those issues are documented here. handlers. * Issues with FreeBSD jails can be worked around as documented by Tatsuya Ono: - https://bogomips.org/unicorn-public/CAHBuKRj09FdxAgzsefJWotexw-7JYZGJMtgUp_dhjPz9VbKD6Q@mail.gmail.com/ + https://yhbt.net/unicorn-public/CAHBuKRj09FdxAgzsefJWotexw-7JYZGJMtgUp_dhjPz9VbKD6Q@mail.gmail.com/ * PRNGs (pseudo-random number generators) loaded before forking (e.g. "preload_app true") may need to have their internal state @@ -60,7 +60,7 @@ acceptable solution. Those issues are documented here. application to use Rails 2.3.2 and you have no other choice, then you may edit your unicorn gemspec and remove the Rack dependency. - ref: https://bogomips.org/unicorn-public/20091014221552.GA30624@dcvr.yhbt.net/ + ref: https://yhbt.net/unicorn-public/20091014221552.GA30624@dcvr.yhbt.net/ Note: the workaround described in the article above only made the issue more subtle and we didn't notice them immediately. @@ -2,7 +2,7 @@ If you're interested in unicorn, you may be interested in some of the projects listed below. If you have any links to add/change/remove, please tell us at -mailto:unicorn-public@bogomips.org! +mailto:unicorn-public@yhbt.net! == Disclaimer @@ -23,10 +23,10 @@ or services behind them. * {golden_brindle}[https://github.com/simonoff/golden_brindle] - tool to manage multiple unicorn instances/applications on a single server -* {raindrops}[https://bogomips.org/raindrops/] - real-time stats for +* {raindrops}[https://yhbt.net/raindrops/] - real-time stats for preforking Rack servers -* {UnXF}[https://bogomips.org/unxf/] Un-X-Forward* the Rack environment, +* {UnXF}[https://yhbt.net/unxf/] Un-X-Forward* the Rack environment, useful since unicorn is designed to be deployed behind a reverse proxy. === unicorn is written to work with @@ -52,7 +52,7 @@ or services behind them. * {Mongrel}[https://rubygems.org/gems/mongrel] - the awesome webserver unicorn is based on. A historical archive of the mongrel dev list featuring early discussions of unicorn is available at: - https://bogomips.org/mongrel-devel/ + https://yhbt.net/mongrel-devel/ -* {david}[https://bogomips.org/david.git] - a tool to explain why you need +* {david}[https://yhbt.net/david.git] - a tool to explain why you need nginx in front of unicorn @@ -1,10 +1,13 @@ = unicorn: Rack HTTP server for fast clients and Unix -unicorn is an HTTP server for Rack applications designed to only serve -fast clients on low-latency, high-bandwidth connections and take -advantage of features in Unix/Unix-like kernels. Slow clients should -only be served by placing a reverse proxy capable of fully buffering -both the the request and response in between unicorn and slow clients. +unicorn is an HTTP server for Rack applications that has done +decades of damage to the entire Ruby ecosystem due to its ability +to tolerate (and thus encourage) bad code. It is only designed +to handle fast clients on low-latency, high-bandwidth connections +and take advantage of features in Unix/Unix-like kernels. +Slow clients must only be served by placing a reverse proxy capable of +fully buffering both the the request and response in between unicorn +and slow clients. == Features @@ -12,11 +15,10 @@ both the the request and response in between unicorn and slow clients. cut out everything that is better supported by the operating system, {nginx}[https://nginx.org/] or {Rack}[https://rack.github.io/]. -* Compatible with Ruby 1.9.3 and later. - unicorn 4.x remains supported for Ruby 1.8 users. +* Compatible with Ruby 2.5 and later. -* Process management: unicorn will reap and restart workers that - die from broken apps. There is no need to manage multiple processes +* Process management: unicorn reaps and restarts workers that die + from broken code. There is no need to manage multiple processes or ports yourself. unicorn can spawn and manage any number of worker processes you choose to scale to your backend. @@ -58,7 +60,7 @@ both the the request and response in between unicorn and slow clients. == License -unicorn is copyright 2009-2018 by all contributors (see logs in git). +unicorn is copyright all contributors (see logs in git). It is based on Mongrel 1.1.5. Mongrel is copyright 2007 Zed A. Shaw and contributors. @@ -80,12 +82,12 @@ You may install it via RubyGems on RubyGems.org: You can get the latest source via git from the following locations (these versions may not be stable): - https://bogomips.org/unicorn.git - https://repo.or.cz/unicorn.git (mirror) + git clone https://yhbt.net/unicorn.git + git clone https://repo.or.cz/unicorn.git # mirror You may browse the code from the web: -* https://bogomips.org/unicorn.git +* https://yhbt.net/unicorn.git * https://repo.or.cz/w/unicorn.git (gitweb) See the HACKING guide on how to contribute and build prerelease gems @@ -119,27 +121,36 @@ supported. Run `unicorn -h` to see command-line options. == Disclaimer There is NO WARRANTY whatsoever if anything goes wrong, but -{let us know}[link:ISSUES.html] and we'll try our best to fix it. +{let us know}[link:ISSUES.html] and maybe someone can fix it. unicorn is designed to only serve fast clients either on the local host or a fast LAN. See the PHILOSOPHY and DESIGN documents for more details regarding this. -Due to its ability to tolerate crashes and isolate clients, unicorn -is unfortunately known to prolong the existence of bugs in applications -and libraries which run on top of it. +The use of unicorn in new deployments is STRONGLY DISCOURAGED due to the +damage done to the entire Ruby ecosystem. Its unintentional popularity +set Ruby back decades in parallelism, concurrency and robustness since +it prolongs and proliferates the existence of poorly-written code. == Contact All feedback (bug reports, user/development dicussion, patches, pull -requests) go to the mailing list/newsgroup. See the ISSUES document for -information on the {mailing list}[mailto:unicorn-public@bogomips.org]. +requests) go to the public mailbox. See the ISSUES document for +information on posting to mailto:unicorn-public@yhbt.net + +Mirror-able mail archives are at https://yhbt.net/unicorn-public/ -The mailing list is archived at https://bogomips.org/unicorn-public/ Read-only NNTP access is available at: -nntp://news.public-inbox.org/inbox.comp.lang.ruby.unicorn and -nntp://news.gmane.org/gmane.comp.lang.ruby.unicorn.general +nntps://news.public-inbox.org/inbox.comp.lang.ruby.unicorn and +nntp://news.gmane.io/gmane.comp.lang.ruby.unicorn.general + +Read-only IMAP access is also available at: +imaps://;AUTH=ANONYMOUS@yhbt.net/inbox.comp.lang.ruby.unicorn.0 and +imap://;AUTH=ANONYMOUS@7fh6tueqddpjyxjmgtdiueylzoqt6pt7hec3pukyptlmohoowvhde4yd.onion/inbox.comp.lang.ruby.unicorn.0 + +Archives are also available over POP3, instructions at: +https://yhbt.net/unicorn-public/_/text/help/#pop3 For the latest on unicorn releases, you may also finger us at -unicorn@bogomips.org or check our NEWS page (and subscribe to our Atom +unicorn@yhbt.net or check our NEWS page (and subscribe to our Atom feed). @@ -8,7 +8,7 @@ should be possible to easily share process management scripts between Unicorn and nginx. One example init script is distributed with unicorn: -https://bogomips.org/unicorn/examples/init.sh +https://yhbt.net/unicorn/examples/init.sh === Master Process @@ -34,7 +34,7 @@ is the primary issue with sandboxing tools such as Bundler and Isolate. If you're bundling unicorn, use "bundle exec unicorn" (or "bundle exec unicorn_rails") to start unicorn with the correct environment variables -ref: https://bogomips.org/unicorn-public/9ECF07C4-5216-47BE-961D-AFC0F0C82060@internetfamo.us/ +ref: https://yhbt.net/unicorn-public/9ECF07C4-5216-47BE-961D-AFC0F0C82060@internetfamo.us/ Otherwise (if you choose to not sandbox your unicorn installation), we expect the tips for Isolate (below) apply, too. @@ -44,7 +44,7 @@ expect the tips for Isolate (below) apply, too. This is no longer be an issue as of bundler 0.9.17 ref: -https://bogomips.org/unicorn-public/8FC34B23-5994-41CC-B5AF-7198EF06909E@tramchase.com/ +https://yhbt.net/unicorn-public/8FC34B23-5994-41CC-B5AF-7198EF06909E@tramchase.com/ === BUNDLE_GEMFILE for Capistrano users @@ -87,7 +87,7 @@ For now workarounds include doing one of the following: 3. Explicitly setting RUBYLIB or $LOAD_PATH to include any gem path where the unicorn gem is installed - (e.g. /usr/lib/ruby/gems/1.9.3/gems/unicorn-VERSION/lib) + (e.g. /usr/lib/ruby/gems/3.0.0/gems/unicorn-VERSION/lib) === RUBYOPT pollution from SIGUSR2 upgrades diff --git a/archive/slrnpull.conf b/archive/slrnpull.conf index fcfcafe..fd04f97 100644 --- a/archive/slrnpull.conf +++ b/archive/slrnpull.conf @@ -1,4 +1,4 @@ # group_name max expire headers_only gmane.comp.lang.ruby.unicorn.general 1000000000 1000000000 0 -# usage: slrnpull -d $PWD -h news.gmane.org --no-post +# usage: slrnpull -d $PWD -h news.gmane.io --no-post diff --git a/examples/big_app_gc.rb b/examples/big_app_gc.rb index 9d05719..c1bae10 100644 --- a/examples/big_app_gc.rb +++ b/examples/big_app_gc.rb @@ -1,2 +1,2 @@ -# see {Unicorn::OobGC}[https://bogomips.org/unicorn/Unicorn/OobGC.html] +# see {Unicorn::OobGC}[https://yhbt.net/unicorn/Unicorn/OobGC.html] # Unicorn::OobGC was broken in Unicorn v3.3.1 - v3.6.1 and fixed in v3.6.2 diff --git a/examples/echo.ru b/examples/echo.ru index 14908c5..e982180 100644 --- a/examples/echo.ru +++ b/examples/echo.ru @@ -19,7 +19,6 @@ class EchoBody < Struct.new(:input) end -use Rack::Chunked run lambda { |env| /\A100-continue\z/i =~ env['HTTP_EXPECT'] and return [100, {}, []] [ 200, { 'Content-Type' => 'application/octet-stream' }, diff --git a/examples/logrotate.conf b/examples/logrotate.conf index 77a01b5..c3aa40d 100644 --- a/examples/logrotate.conf +++ b/examples/logrotate.conf @@ -5,7 +5,7 @@ # https://linux.die.net/man/8/logrotate # # public logrotate-related discussion in our archives: -# https://bogomips.org/unicorn-public/?q=logrotate +# https://yhbt.net/unicorn-public/?q=logrotate # Modify the following glob to match the logfiles your app writes to: /var/log/unicorn_app/*.log { @@ -33,7 +33,7 @@ systemctl kill -s SIGUSR1 unicorn@2.service # Examples for other process management systems appreciated - # Mail us at unicorn-public@bogomips.org + # Mail us at unicorn-public@yhbt.net # (see above for archives) # If you use a pid file and assuming your pid file diff --git a/examples/nginx.conf b/examples/nginx.conf index b6b69c1..c5026f9 100644 --- a/examples/nginx.conf +++ b/examples/nginx.conf @@ -113,7 +113,7 @@ http { # try_files directive appeared in in nginx 0.7.27 and has stabilized # over time. Older versions of nginx (e.g. 0.6.x) requires # "if (!-f $request_filename)" which was less efficient: - # https://bogomips.org/unicorn.git/tree/examples/nginx.conf?id=v3.3.1#n127 + # https://yhbt.net/unicorn.git/tree/examples/nginx.conf?id=v3.3.1#n127 try_files $uri/index.html $uri.html $uri @app; location @app { diff --git a/examples/unicorn.conf.minimal.rb b/examples/unicorn.conf.minimal.rb index 2d1bf0a..46fd634 100644 --- a/examples/unicorn.conf.minimal.rb +++ b/examples/unicorn.conf.minimal.rb @@ -1,9 +1,9 @@ # Minimal sample configuration file for Unicorn (not Rack) when used # with daemonization (unicorn -D) started in your working directory. # -# See https://bogomips.org/unicorn/Unicorn/Configurator.html for complete +# See https://yhbt.net/unicorn/Unicorn/Configurator.html for complete # documentation. -# See also https://bogomips.org/unicorn/examples/unicorn.conf.rb for +# See also https://yhbt.net/unicorn/examples/unicorn.conf.rb for # a more verbose configuration using more features. listen 2007 # by default Unicorn listens on port 8080 diff --git a/examples/unicorn.conf.rb b/examples/unicorn.conf.rb index d2897ef..d90bdc4 100644 --- a/examples/unicorn.conf.rb +++ b/examples/unicorn.conf.rb @@ -2,10 +2,10 @@ # # This configuration file documents many features of Unicorn # that may not be needed for some applications. See -# https://bogomips.org/unicorn/examples/unicorn.conf.minimal.rb +# https://yhbt.net/unicorn/examples/unicorn.conf.minimal.rb # for a much simpler configuration file. # -# See https://bogomips.org/unicorn/Unicorn/Configurator.html for complete +# See https://yhbt.net/unicorn/Unicorn/Configurator.html for complete # documentation. # Use at least one worker per core if you're on a dedicated server, diff --git a/examples/unicorn@.service b/examples/unicorn@.service index d95eb83..946de44 100644 --- a/examples/unicorn@.service +++ b/examples/unicorn@.service @@ -14,7 +14,14 @@ After = unicorn.socket # bundler users must use the "--keep-file-descriptors" switch, here: # ExecStart = bundle exec --keep-file-descriptors unicorn -c ... ExecStart = /usr/bin/unicorn -c /path/to/unicorn.conf.rb /path/to/config.ru + +# NonBlocking MUST be true if using socket activation with unicorn. +# Otherwise, there's a small window in-between when the non-blocking +# flag is set by us and our accept4 call where systemd can momentarily +# make the socket blocking, causing us to block on accept4: +NonBlocking = true Sockets = unicorn.socket + KillSignal = SIGQUIT User = nobody Group = nogroup diff --git a/ext/unicorn_http/c_util.h b/ext/unicorn_http/c_util.h index ab1fc0e..5774615 100644 --- a/ext/unicorn_http/c_util.h +++ b/ext/unicorn_http/c_util.h @@ -8,23 +8,15 @@ #include <unistd.h> #include <assert.h> +#include <limits.h> #define MIN(a,b) (a < b ? a : b) #define ARRAY_SIZE(x) (sizeof(x)/sizeof(x[0])) -#ifndef SIZEOF_OFF_T -# define SIZEOF_OFF_T 4 -# warning SIZEOF_OFF_T not defined, guessing 4. Did you run extconf.rb? -#endif - -#if SIZEOF_OFF_T == 4 -# define UH_OFF_T_MAX 0x7fffffff -#elif SIZEOF_OFF_T == 8 -# if SIZEOF_LONG == 4 -# define UH_OFF_T_MAX 0x7fffffffffffffffLL -# else -# define UH_OFF_T_MAX 0x7fffffffffffffff -# endif +#if SIZEOF_OFF_T == SIZEOF_INT +# define UH_OFF_T_MAX INT_MAX +#elif SIZEOF_OFF_T == SIZEOF_LONG_LONG +# define UH_OFF_T_MAX LLONG_MAX #else # error off_t size unknown for this platform! #endif /* SIZEOF_OFF_T check */ diff --git a/ext/unicorn_http/common_field_optimization.h b/ext/unicorn_http/common_field_optimization.h index 0659fc7..250e43e 100644 --- a/ext/unicorn_http/common_field_optimization.h +++ b/ext/unicorn_http/common_field_optimization.h @@ -83,7 +83,6 @@ static void init_common_fields(void) struct common_field *cf = common_http_fields; char tmp[64]; - id_uminus = rb_intern("-@"); memcpy(tmp, HTTP_PREFIX, HTTP_PREFIX_LEN); for(i = ARRAY_SIZE(common_http_fields); --i >= 0; cf++) { diff --git a/ext/unicorn_http/epollexclusive.h b/ext/unicorn_http/epollexclusive.h new file mode 100644 index 0000000..c74a779 --- /dev/null +++ b/ext/unicorn_http/epollexclusive.h @@ -0,0 +1,128 @@ +/* + * This is only intended for use inside a unicorn worker, nowhere else. + * EPOLLEXCLUSIVE somewhat mitigates the thundering herd problem for + * mostly idle processes since we can't use blocking accept4. + * This is NOT intended for use with multi-threaded servers, nor + * single-threaded multi-client ("C10K") servers or anything advanced + * like that. This use of epoll is only appropriate for a primitive, + * single-client, single-threaded servers like unicorn that need to + * support SIGKILL timeouts and parent death detection. + */ +#if defined(HAVE_EPOLL_CREATE1) +# include <sys/epoll.h> +# include <errno.h> +# include <ruby/io.h> +# include <ruby/thread.h> +#endif /* __linux__ */ + +#if defined(EPOLLEXCLUSIVE) && defined(HAVE_EPOLL_CREATE1) +# define USE_EPOLL (1) +#else +# define USE_EPOLL (0) +#endif + +#if USE_EPOLL +#if defined(HAVE_RB_IO_DESCRIPTOR) /* Ruby 3.1+ */ +# define my_fileno(io) rb_io_descriptor(io) +#else /* Ruby <3.1 */ +static int my_fileno(VALUE io) +{ + rb_io_t *fptr; + GetOpenFile(io, fptr); + rb_io_check_closed(fptr); + return fptr->fd; +} +#endif /* Ruby <3.1 */ + +/* + * :nodoc: + * returns IO object if EPOLLEXCLUSIVE works and arms readers + */ +static VALUE prep_readers(VALUE cls, VALUE readers) +{ + long i; + int epfd = epoll_create1(EPOLL_CLOEXEC); + VALUE epio; + + if (epfd < 0) rb_sys_fail("epoll_create1"); + + epio = rb_funcall(cls, rb_intern("for_fd"), 1, INT2NUM(epfd)); + + Check_Type(readers, T_ARRAY); + for (i = 0; i < RARRAY_LEN(readers); i++) { + int rc, fd; + struct epoll_event e; + VALUE io = rb_ary_entry(readers, i); + + e.data.u64 = i; /* the reason readers shouldn't change */ + + /* + * I wanted to use EPOLLET here, but maintaining our own + * equivalent of ep->rdllist in Ruby-space doesn't fit + * our design at all (and the kernel already has it's own + * code path for doing it). So let the kernel spend + * cycles on maintaining level-triggering. + */ + e.events = EPOLLEXCLUSIVE | EPOLLIN; + fd = my_fileno(rb_io_get_io(io)); + rc = epoll_ctl(epfd, EPOLL_CTL_ADD, fd, &e); + if (rc < 0) rb_sys_fail("epoll_ctl"); + } + return epio; +} +#endif /* USE_EPOLL */ + +#if USE_EPOLL +struct ep_wait { + struct epoll_event event; + int epfd; + int timeout_msec; +}; + +static void *do_wait(void *ptr) /* runs w/o GVL */ +{ + struct ep_wait *epw = ptr; + /* + * Linux delivers epoll events in the order received, and using + * maxevents=1 ensures we pluck one item off ep->rdllist + * at-a-time (c.f. fs/eventpoll.c in linux.git, it's quite + * easy-to-understand for anybody familiar with Ruby C). + */ + return (void *)(long)epoll_wait(epw->epfd, &epw->event, 1, + epw->timeout_msec); +} + +/* :nodoc: */ +/* readers must not change between prepare_readers and get_readers */ +static VALUE +get_readers(VALUE epio, VALUE ready, VALUE readers, VALUE timeout_msec) +{ + struct ep_wait epw; + long n; + + Check_Type(ready, T_ARRAY); + Check_Type(readers, T_ARRAY); + + epw.epfd = my_fileno(epio); + epw.timeout_msec = NUM2INT(timeout_msec); + n = (long)rb_thread_call_without_gvl(do_wait, &epw, RUBY_UBF_IO, NULL); + if (n < 0) { + if (errno != EINTR) rb_sys_fail("epoll_wait"); + } else if (n > 0) { /* maxevents is hardcoded to 1 */ + VALUE obj = rb_ary_entry(readers, epw.event.data.u64); + + if (RTEST(obj)) + rb_ary_push(ready, obj); + } /* n == 0 : timeout */ + return Qfalse; +} +#endif /* USE_EPOLL */ + +static void init_epollexclusive(VALUE mUnicorn) +{ +#if USE_EPOLL + VALUE cWaiter = rb_define_class_under(mUnicorn, "Waiter", rb_cIO); + rb_define_singleton_method(cWaiter, "prep_readers", prep_readers, 1); + rb_define_method(cWaiter, "get_readers", get_readers, 3); +#endif +} diff --git a/ext/unicorn_http/ext_help.h b/ext/unicorn_http/ext_help.h index 747c36c..86a187e 100644 --- a/ext/unicorn_http/ext_help.h +++ b/ext/unicorn_http/ext_help.h @@ -8,30 +8,6 @@ # define assert_frozen(f) do {} while (0) #endif /* !defined(OBJ_FROZEN) */ -#if !defined(OFFT2NUM) -# if SIZEOF_OFF_T == SIZEOF_LONG -# define OFFT2NUM(n) LONG2NUM(n) -# else -# define OFFT2NUM(n) LL2NUM(n) -# endif -#endif /* ! defined(OFFT2NUM) */ - -#if !defined(SIZET2NUM) -# if SIZEOF_SIZE_T == SIZEOF_LONG -# define SIZET2NUM(n) ULONG2NUM(n) -# else -# define SIZET2NUM(n) ULL2NUM(n) -# endif -#endif /* ! defined(SIZET2NUM) */ - -#if !defined(NUM2SIZET) -# if SIZEOF_SIZE_T == SIZEOF_LONG -# define NUM2SIZET(n) ((size_t)NUM2ULONG(n)) -# else -# define NUM2SIZET(n) ((size_t)NUM2ULL(n)) -# endif -#endif /* ! defined(NUM2SIZET) */ - static inline int str_cstr_eq(VALUE val, const char *ptr, long len) { return (RSTRING_LEN(val) == len && !memcmp(ptr, RSTRING_PTR(val), len)); diff --git a/ext/unicorn_http/extconf.rb b/ext/unicorn_http/extconf.rb index d5f81fb..11099cd 100644 --- a/ext/unicorn_http/extconf.rb +++ b/ext/unicorn_http/extconf.rb @@ -1,12 +1,7 @@ # -*- encoding: binary -*- require 'mkmf' -have_macro("SIZEOF_OFF_T", "ruby.h") or check_sizeof("off_t", "sys/types.h") -have_macro("SIZEOF_SIZE_T", "ruby.h") or check_sizeof("size_t", "sys/types.h") -have_macro("SIZEOF_LONG", "ruby.h") or check_sizeof("long", "sys/types.h") -have_func("rb_str_set_len", "ruby.h") or abort 'Ruby 1.9.3+ required' -have_func("rb_hash_clear", "ruby.h") # Ruby 2.0+ -have_func("gmtime_r", "time.h") +have_func("rb_hash_clear", "ruby.h") or abort 'Ruby 2.0+ required' message('checking if String#-@ (str_uminus) dedupes... ') begin @@ -38,4 +33,7 @@ else message("no, needs Ruby 2.6+\n") end +if have_func('epoll_create1', %w(sys/epoll.h)) + have_func('rb_io_descriptor') # Ruby 3.1+ +end create_makefile("unicorn_http") diff --git a/ext/unicorn_http/global_variables.h b/ext/unicorn_http/global_variables.h index f8e694c..c9ceebd 100644 --- a/ext/unicorn_http/global_variables.h +++ b/ext/unicorn_http/global_variables.h @@ -55,7 +55,7 @@ NORETURN(static void parser_raise(VALUE klass, const char *)); /** Defines global strings in the init method. */ #define DEF_GLOBAL(N, val) do { \ - g_##N = rb_obj_freeze(rb_str_new(val, sizeof(val) - 1)); \ + g_##N = str_new_dd_freeze(val, (long)sizeof(val) - 1); \ rb_gc_register_mark_object(g_##N); \ } while (0) diff --git a/ext/unicorn_http/httpdate.c b/ext/unicorn_http/httpdate.c index b59d038..0faf5da 100644 --- a/ext/unicorn_http/httpdate.c +++ b/ext/unicorn_http/httpdate.c @@ -1,5 +1,6 @@ #include <ruby.h> #include <time.h> +#include <sys/time.h> #include <stdio.h> static const size_t buf_capa = sizeof("Thu, 01 Jan 1970 00:00:00 GMT"); @@ -11,6 +12,7 @@ static const char months[] = "Jan\0Feb\0Mar\0Apr\0May\0Jun\0" /* for people on wonky systems only */ #ifndef HAVE_GMTIME_R +# warning using fake gmtime_r static struct tm * my_gmtime_r(time_t *now, struct tm *tm) { struct tm *global = gmtime(now); @@ -42,13 +44,24 @@ static struct tm * my_gmtime_r(time_t *now, struct tm *tm) static VALUE httpdate(VALUE self) { static time_t last; - time_t now = time(NULL); /* not a syscall on modern 64-bit systems */ + struct timeval now; struct tm tm; - if (last == now) + /* + * Favor gettimeofday(2) over time(2), as the latter can return the + * wrong value in the first 1 .. 2.5 ms of every second(!) + * + * https://lore.kernel.org/git/20230320230507.3932018-1-gitster@pobox.com/ + * https://inbox.sourceware.org/libc-alpha/20230306160321.2942372-1-adhemerval.zanella@linaro.org/T/ + * https://sourceware.org/bugzilla/show_bug.cgi?id=30200 + */ + if (gettimeofday(&now, NULL)) + rb_sys_fail("gettimeofday"); + + if (last == now.tv_sec) return buf; - last = now; - gmtime_r(&now, &tm); + last = now.tv_sec; + gmtime_r(&now.tv_sec, &tm); /* we can make this thread-safe later if our Ruby loses the GVL */ snprintf(buf_ptr, buf_capa, diff --git a/ext/unicorn_http/unicorn_http.rl b/ext/unicorn_http/unicorn_http.rl index 8ef23bc..fb5dcde 100644 --- a/ext/unicorn_http/unicorn_http.rl +++ b/ext/unicorn_http/unicorn_http.rl @@ -12,6 +12,7 @@ #include "common_field_optimization.h" #include "global_variables.h" #include "c_util.h" +#include "epollexclusive.h" void init_unicorn_httpdate(void); @@ -27,10 +28,15 @@ void init_unicorn_httpdate(void); #define UH_FL_TO_CLEAR 0x200 #define UH_FL_RESSTART 0x400 /* for check_client_connection */ #define UH_FL_HIJACK 0x800 +#define UH_FL_RES_CHUNK_VER (1U << 12) +#define UH_FL_RES_CHUNK_METHOD (1U << 13) /* all of these flags need to be set for keepalive to be supported */ #define UH_FL_KEEPALIVE (UH_FL_KAVERSION | UH_FL_REQEOF | UH_FL_HASHEADER) +/* we can only chunk responses for non-HEAD HTTP/1.1 requests */ +#define UH_FL_RES_CHUNKABLE (UH_FL_RES_CHUNK_VER | UH_FL_RES_CHUNK_METHOD) + static unsigned int MAX_HEADER_LEN = 1024 * (80 + 32); /* same as Mongrel */ /* this is only intended for use with Rainbows! */ @@ -62,19 +68,8 @@ struct http_parser { } len; }; -static ID id_set_backtrace; - -#ifdef HAVE_RB_HASH_CLEAR /* Ruby >= 2.0 */ -# define my_hash_clear(h) (void)rb_hash_clear(h) -#else /* !HAVE_RB_HASH_CLEAR - Ruby <= 1.9.3 */ - -static ID id_clear; - -static void my_hash_clear(VALUE h) -{ - rb_funcall(h, id_clear, 0); -} -#endif /* HAVE_RB_HASH_CLEAR */ +static ID id_set_backtrace, id_is_chunked_p; +static VALUE cHttpParser; static void finalize_header(struct http_parser *hp); @@ -155,6 +150,9 @@ request_method(struct http_parser *hp, const char *ptr, size_t len) { VALUE v = rb_str_new(ptr, len); + if (len != 4 || memcmp(ptr, "HEAD", 4)) + HP_FL_SET(hp, RES_CHUNK_METHOD); + rb_hash_aset(hp->env, g_request_method, v); } @@ -168,6 +166,7 @@ http_version(struct http_parser *hp, const char *ptr, size_t len) if (CONST_MEM_EQ("HTTP/1.1", ptr, len)) { /* HTTP/1.1 implies keepalive unless "Connection: close" is set */ HP_FL_SET(hp, KAVERSION); + HP_FL_SET(hp, RES_CHUNK_VER); v = g_http_11; } else if (CONST_MEM_EQ("HTTP/1.0", ptr, len)) { v = g_http_10; @@ -220,6 +219,19 @@ static void write_cont_value(struct http_parser *hp, rb_str_buf_cat(hp->cont, vptr, end + 1); } +static int is_chunked(VALUE v) +{ + /* common case first */ + if (STR_CSTR_CASE_EQ(v, "chunked")) + return 1; + + /* + * call Ruby function in unicorn/http_request.rb to deal with unlikely + * comma-delimited case + */ + return rb_funcall(cHttpParser, id_is_chunked_p, 1, v) != Qfalse; +} + static void write_value(struct http_parser *hp, const char *buffer, const char *p) { @@ -246,7 +258,9 @@ static void write_value(struct http_parser *hp, f = uncommon_field(field, flen); } else if (f == g_http_connection) { hp_keepalive_connection(hp, v); - } else if (f == g_content_length) { + } else if (f == g_content_length && !HP_FL_TEST(hp, CHUNKED)) { + if (hp->len.content) + parser_raise(eHttpParserError, "Content-Length already set"); hp->len.content = parse_length(RSTRING_PTR(v), RSTRING_LEN(v)); if (hp->len.content < 0) parser_raise(eHttpParserError, "invalid Content-Length"); @@ -254,9 +268,30 @@ static void write_value(struct http_parser *hp, HP_FL_SET(hp, HASBODY); hp_invalid_if_trailer(hp); } else if (f == g_http_transfer_encoding) { - if (STR_CSTR_CASE_EQ(v, "chunked")) { + if (is_chunked(v)) { + if (HP_FL_TEST(hp, CHUNKED)) + /* + * RFC 7230 3.3.1: + * A sender MUST NOT apply chunked more than once to a message body + * (i.e., chunking an already chunked message is not allowed). + */ + parser_raise(eHttpParserError, "Transfer-Encoding double chunked"); + HP_FL_SET(hp, CHUNKED); HP_FL_SET(hp, HASBODY); + + /* RFC 7230 3.3.3, 3: favor chunked if Content-Length exists */ + hp->len.content = 0; + } else if (HP_FL_TEST(hp, CHUNKED)) { + /* + * RFC 7230 3.3.3, point 3 states: + * If a Transfer-Encoding header field is present in a request and + * the chunked transfer coding is not the final encoding, the + * message body length cannot be determined reliably; the server + * MUST respond with the 400 (Bad Request) status code and then + * close the connection. + */ + parser_raise(eHttpParserError, "invalid Transfer-Encoding"); } hp_invalid_if_trailer(hp); } else if (f == g_http_trailer) { @@ -487,7 +522,7 @@ static void set_url_scheme(VALUE env, VALUE *server_port) * and X-Forwarded-Proto handling from this parser? We've had it * forever and nobody has said anything against it, either. * Anyways, please send comments to our public mailing list: - * unicorn-public@bogomips.org (no HTML mail, no subscription necessary) + * unicorn-public@yhbt.net (no HTML mail, no subscription necessary) */ scheme = rb_hash_aref(env, g_http_x_forwarded_ssl); if (!NIL_P(scheme) && STR_CSTR_EQ(scheme, "on")) { @@ -613,7 +648,7 @@ static VALUE HttpParser_clear(VALUE self) return HttpParser_init(self); http_parser_init(hp); - my_hash_clear(hp->env); + rb_hash_clear(hp->env); return self; } @@ -775,6 +810,14 @@ static VALUE HttpParser_keepalive(VALUE self) return HP_FL_ALL(hp, KEEPALIVE) ? Qtrue : Qfalse; } +/* :nodoc: */ +static VALUE chunkable_response_p(VALUE self) +{ + const struct http_parser *hp = data_get(self); + + return HP_FL_ALL(hp, RES_CHUNKABLE) ? Qtrue : Qfalse; +} + /** * call-seq: * parser.next? => true or false @@ -931,7 +974,7 @@ static VALUE HttpParser_rssget(VALUE self) void Init_unicorn_http(void) { - VALUE mUnicorn, cHttpParser; + VALUE mUnicorn; mUnicorn = rb_define_module("Unicorn"); cHttpParser = rb_define_class_under(mUnicorn, "HttpParser", rb_cObject); @@ -942,6 +985,7 @@ void Init_unicorn_http(void) e414 = rb_define_class_under(mUnicorn, "RequestURITooLongError", eHttpParserError); + id_uminus = rb_intern("-@"); init_globals(); rb_define_alloc_func(cHttpParser, HttpParser_alloc); rb_define_method(cHttpParser, "initialize", HttpParser_init, 0); @@ -954,6 +998,7 @@ void Init_unicorn_http(void) rb_define_method(cHttpParser, "content_length", HttpParser_content_length, 0); rb_define_method(cHttpParser, "body_eof?", HttpParser_body_eof, 0); rb_define_method(cHttpParser, "keepalive?", HttpParser_keepalive, 0); + rb_define_method(cHttpParser, "chunkable_response?", chunkable_response_p, 0); rb_define_method(cHttpParser, "headers?", HttpParser_has_headers, 0); rb_define_method(cHttpParser, "next?", HttpParser_next, 0); rb_define_method(cHttpParser, "buf", HttpParser_buf, 0); @@ -988,8 +1033,8 @@ void Init_unicorn_http(void) id_set_backtrace = rb_intern("set_backtrace"); init_unicorn_httpdate(); -#ifndef HAVE_RB_HASH_CLEAR - id_clear = rb_intern("clear"); -#endif + id_is_chunked_p = rb_intern("is_chunked?"); + + init_epollexclusive(mUnicorn); } #undef SET_GLOBAL diff --git a/ext/unicorn_http/unicorn_http_common.rl b/ext/unicorn_http/unicorn_http_common.rl index 0988b54..507e570 100644 --- a/ext/unicorn_http/unicorn_http_common.rl +++ b/ext/unicorn_http/unicorn_http_common.rl @@ -4,7 +4,7 @@ #### HTTP PROTOCOL GRAMMAR # line endings -Â Â CRLF = ("\r\n" | "\n"); + CRLF = ("\r\n" | "\n"); # character types CTL = (cntrl | 127); diff --git a/lib/unicorn.rb b/lib/unicorn.rb index dd5dff4..564cb30 100644 --- a/lib/unicorn.rb +++ b/lib/unicorn.rb @@ -1,7 +1,6 @@ # -*- encoding: binary -*- require 'etc' require 'stringio' -require 'kgio' require 'raindrops' require 'io/wait' @@ -66,7 +65,6 @@ module Unicorn middleware = { # order matters ContentLength: nil, - Chunked: nil, CommonLogger: [ $stderr ], ShowExceptions: nil, Lint: nil, @@ -75,8 +73,8 @@ module Unicorn # return value, matches rackup defaults based on env # Unicorn does not support persistent connections, but Rainbows! - # and Zbatery both do. Users accustomed to the Rack::Server default - # middlewares will need ContentLength/Chunked middlewares. + # does. Users accustomed to the Rack::Server default + # middlewares will need ContentLength middleware. case ENV["RACK_ENV"] when "development" when "deployment" @@ -96,7 +94,7 @@ module Unicorn # returns an array of strings representing TCP listen socket addresses # and Unix domain socket paths. This is useful for use with - # Raindrops::Middleware under Linux: https://bogomips.org/raindrops/ + # Raindrops::Middleware under Linux: https://yhbt.net/raindrops/ def self.listener_names Unicorn::HttpServer::LISTENERS.map do |io| Unicorn::SocketHelper.sock_name(io) @@ -113,9 +111,7 @@ module Unicorn F_SETPIPE_SZ = 1031 if RUBY_PLATFORM =~ /linux/ def self.pipe # :nodoc: - Kgio::Pipe.new.each do |io| - io.close_on_exec = true # remove this when we only support Ruby >= 2.0 - + IO.pipe.each do |io| # shrink pipes to minimize impact on /proc/sys/fs/pipe-user-pages-soft # limits. if defined?(F_SETPIPE_SZ) diff --git a/lib/unicorn/configurator.rb b/lib/unicorn/configurator.rb index e8b76f5..b21a01d 100644 --- a/lib/unicorn/configurator.rb +++ b/lib/unicorn/configurator.rb @@ -3,11 +3,11 @@ require 'logger' # Implements a simple DSL for configuring a unicorn server. # -# See https://bogomips.org/unicorn/examples/unicorn.conf.rb and -# https://bogomips.org/unicorn/examples/unicorn.conf.minimal.rb +# See https://yhbt.net/unicorn/examples/unicorn.conf.rb and +# https://yhbt.net/unicorn/examples/unicorn.conf.minimal.rb # example configuration files. An example config file for use with # nginx is also available at -# https://bogomips.org/unicorn/examples/nginx.conf +# https://yhbt.net/unicorn/examples/nginx.conf # # See the link:/TUNING.html document for more information on tuning unicorn. class Unicorn::Configurator @@ -53,6 +53,7 @@ class Unicorn::Configurator server.logger.info("worker=#{worker.nr} ready") }, :pid => nil, + :early_hints => false, :worker_exec => false, :preload_app => false, :check_client_connection => false, @@ -215,7 +216,12 @@ class Unicorn::Configurator set_hook(:before_exec, block_given? ? block : args[0], 1) end - # sets the timeout of worker processes to +seconds+. Workers + # Strongly consider using link:/Application_Timeouts.html instead + # of this misfeature. This misfeature has done decades of damage + # to Ruby since it demotivates the use of fine-grained timeout + # mechanisms. + # + # Sets the timeout of worker processes to +seconds+. Workers # handling the request/app.call/response cycle taking longer than # this time period will be forcibly killed (via SIGKILL). This # timeout is enforced by the master process itself and not subject @@ -276,6 +282,15 @@ class Unicorn::Configurator set_bool(:default_middleware, bool) end + # sets whether to enable the proposed early hints Rack API. + # If enabled, Rails 5.2+ will automatically send a 103 Early Hint + # for all the `javascript_include_tag` and `stylesheet_link_tag` + # in your response. See: https://api.rubyonrails.org/v5.2/classes/ActionDispatch/Request.html#method-i-send_early_hints + # See also https://tools.ietf.org/html/rfc8297 + def early_hints(bool) + set_bool(:early_hints, bool) + end + # sets listeners to the given +addresses+, replacing or augmenting the # current set. This is for the global listener pool shared by all # worker processes. For per-worker listeners, see the after_fork example diff --git a/lib/unicorn/http_request.rb b/lib/unicorn/http_request.rb index bcc1f2d..ab3bd6e 100644 --- a/lib/unicorn/http_request.rb +++ b/lib/unicorn/http_request.rb @@ -61,8 +61,7 @@ class Unicorn::HttpParser # returns an environment hash suitable for Rack if successful # This does minimal exception trapping and it is up to the caller # to handle any socket errors (e.g. user aborted upload). - def read(socket) - clear + def read_headers(socket, ai) e = env # From https://www.ietf.org/rfc/rfc3875: @@ -72,17 +71,17 @@ class Unicorn::HttpParser # identify the client for the immediate request to the server; # that client may be a proxy, gateway, or other intermediary # acting on behalf of the actual source client." - e['REMOTE_ADDR'] = socket.kgio_addr + e['REMOTE_ADDR'] = ai.unix? ? '127.0.0.1' : ai.ip_address # short circuit the common case with small GET requests first - socket.kgio_read!(16384, buf) + socket.readpartial(16384, buf) if parse.nil? # Parser is not done, queue up more data to read and continue parsing # an Exception thrown from the parser will throw us out of the loop - false until add_parse(socket.kgio_read!(16384)) + false until add_parse(socket.readpartial(16384)) end - check_client_connection(socket) if @@check_client_connection + check_client_connection(socket, ai) if @@check_client_connection e['rack.input'] = 0 == content_length ? NULL_IO : @@input_class.new(socket, self) @@ -108,8 +107,8 @@ class Unicorn::HttpParser if Raindrops.const_defined?(:TCP_Info) TCPI = Raindrops::TCP_Info.allocate - def check_client_connection(socket) # :nodoc: - if Unicorn::TCPClient === socket + def check_client_connection(socket, ai) # :nodoc: + if ai.ip? # Raindrops::TCP_Info#get!, #state (reads struct tcp_info#tcpi_state) raise Errno::EPIPE, "client closed connection".freeze, EMPTY_ARRAY if closed_state?(TCPI.get!(socket).state) @@ -153,8 +152,8 @@ class Unicorn::HttpParser # Ruby 2.2+ can show struct tcp_info as a string Socket::Option#inspect. # Not that efficient, but probably still better than doing unnecessary # work after a client gives up. - def check_client_connection(socket) # :nodoc: - if Unicorn::TCPClient === socket && @@tcpi_inspect_ok + def check_client_connection(socket, ai) # :nodoc: + if @@tcpi_inspect_ok && ai.ip? opt = socket.getsockopt(Socket::IPPROTO_TCP, Socket::TCP_INFO).inspect if opt =~ /\bstate=(\S+)/ raise Errno::EPIPE, "client closed connection".freeze, @@ -188,4 +187,15 @@ class Unicorn::HttpParser HTTP_RESPONSE_START.each { |c| socket.write(c) } end end + + # called by ext/unicorn_http/unicorn_http.rl via rb_funcall + def self.is_chunked?(v) # :nodoc: + vals = v.split(/[ \t]*,[ \t]*/).map!(&:downcase) + if vals.pop == 'chunked'.freeze + return true unless vals.include?('chunked'.freeze) + raise Unicorn::HttpParserError, 'double chunked', [] + end + return false unless vals.include?('chunked'.freeze) + raise Unicorn::HttpParserError, 'chunked not last', [] + end end diff --git a/lib/unicorn/http_response.rb b/lib/unicorn/http_response.rb index b23e521..0ed0ae3 100644 --- a/lib/unicorn/http_response.rb +++ b/lib/unicorn/http_response.rb @@ -12,6 +12,12 @@ module Unicorn::HttpResponse STATUS_CODES = defined?(Rack::Utils::HTTP_STATUS_CODES) ? Rack::Utils::HTTP_STATUS_CODES : {} + STATUS_WITH_NO_ENTITY_BODY = defined?( + Rack::Utils::STATUS_WITH_NO_ENTITY_BODY) ? + Rack::Utils::STATUS_WITH_NO_ENTITY_BODY : begin + warn 'Rack::Utils::STATUS_WITH_NO_ENTITY_BODY missing' + {} + end # internal API, code will always be common-enough-for-even-old-Rack def err_response(code, response_start_sent) @@ -19,15 +25,28 @@ module Unicorn::HttpResponse "#{code} #{STATUS_CODES[code]}\r\n\r\n" end + def append_header(buf, key, value) + case value + when Array # Rack 3 + value.each { |v| buf << "#{key}: #{v}\r\n" } + when /\n/ # Rack 2 + # avoiding blank, key-only cookies with /\n+/ + value.split(/\n+/).each { |v| buf << "#{key}: #{v}\r\n" } + else + buf << "#{key}: #{value}\r\n" + end + end + # writes the rack_response to socket as an HTTP response def http_response_write(socket, status, headers, body, req = Unicorn::HttpRequest.new) hijack = nil - + do_chunk = false if headers code = status.to_i msg = STATUS_CODES[code] start = req.response_start_sent ? ''.freeze : 'HTTP/1.1 '.freeze + term = STATUS_WITH_NO_ENTITY_BODY.include?(code) || false buf = "#{start}#{msg ? %Q(#{code} #{msg}) : status}\r\n" \ "Date: #{httpdate}\r\n" \ "Connection: close\r\n" @@ -35,25 +54,38 @@ module Unicorn::HttpResponse case key when %r{\A(?:Date|Connection)\z}i next + when %r{\AContent-Length\z}i + append_header(buf, key, value) + term = true + when %r{\ATransfer-Encoding\z}i + append_header(buf, key, value) + term = true if /\bchunked\b/i === value # value may be Array :x when "rack.hijack" # This should only be hit under Rack >= 1.5, as this was an illegal # key in Rack < 1.5 hijack = value else - if value =~ /\n/ - # avoiding blank, key-only cookies with /\n+/ - value.split(/\n+/).each { |v| buf << "#{key}: #{v}\r\n" } - else - buf << "#{key}: #{value}\r\n" - end + append_header(buf, key, value) end end + if !hijack && !term && req.chunkable_response? + do_chunk = true + buf << "Transfer-Encoding: chunked\r\n".freeze + end socket.write(buf << "\r\n".freeze) end if hijack req.hijacked! hijack.call(socket) + elsif do_chunk + begin + body.each do |b| + socket.write("#{b.bytesize.to_s(16)}\r\n", b, "\r\n".freeze) + end + ensure + socket.write("0\r\n\r\n".freeze) + end else body.each { |chunk| socket.write(chunk) } end diff --git a/lib/unicorn/http_server.rb b/lib/unicorn/http_server.rb index 5334fa0..ed5bbf1 100644 --- a/lib/unicorn/http_server.rb +++ b/lib/unicorn/http_server.rb @@ -6,7 +6,7 @@ # forked worker children. # # Users do not need to know the internals of this class, but reading the -# {source}[https://bogomips.org/unicorn.git/tree/lib/unicorn/http_server.rb] +# {source}[https://yhbt.net/unicorn.git/tree/lib/unicorn/http_server.rb] # is education for programmers wishing to learn how unicorn works. # See Unicorn::Configurator for information on how to configure unicorn. class Unicorn::HttpServer @@ -15,7 +15,7 @@ class Unicorn::HttpServer :before_fork, :after_fork, :before_exec, :listener_opts, :preload_app, :orig_app, :config, :ready_pipe, :user, - :default_middleware + :default_middleware, :early_hints attr_writer :after_worker_exit, :after_worker_ready, :worker_exec attr_reader :pid, :logger @@ -69,7 +69,6 @@ class Unicorn::HttpServer # incoming requests on the socket. def initialize(app, options = {}) @app = app - @request = Unicorn::HttpRequest.new @reexec_pid = 0 @default_middleware = true options = options.dup @@ -78,6 +77,7 @@ class Unicorn::HttpServer options[:use_defaults] = true self.config = Unicorn::Configurator.new(options) self.listener_opts = {} + @immortal = [] # immortal inherited sockets from systemd # We use @self_pipe differently in the master and worker processes: # @@ -111,9 +111,7 @@ class Unicorn::HttpServer @worker_data = if worker_data = ENV['UNICORN_WORKER'] worker_data = worker_data.split(',').map!(&:to_i) - worker_data[1] = worker_data.slice!(1..2).map do |i| - Kgio::Pipe.for_fd(i) - end + worker_data[1] = worker_data.slice!(1..2).map { |i| IO.for_fd(i) } worker_data end end @@ -159,6 +157,7 @@ class Unicorn::HttpServer end set_names = listener_names(listeners) dead_names.concat(cur_names - set_names).uniq! + dead_names -= @immortal.map { |io| sock_name(io) } LISTENERS.delete_if do |io| if dead_names.include?(sock_name(io)) @@ -188,7 +187,8 @@ class Unicorn::HttpServer rescue Errno::EEXIST retry end - fp.syswrite("#$$\n") + fp.sync = true + fp.write("#$$\n") File.rename(fp.path, path) fp.close end @@ -241,10 +241,6 @@ class Unicorn::HttpServer tries = opt[:tries] || 5 begin io = bind_listen(address, opt) - unless Kgio::TCPServer === io || Kgio::UNIXServer === io - io.autoclose = false - io = server_cast(io) - end logger.info "listening on addr=#{sock_name(io)} fd=#{io.fileno}" LISTENERS << io io @@ -387,12 +383,13 @@ class Unicorn::HttpServer # the Ruby itself and not require a separate malloc (on 32-bit MRI 1.9+). # Most reads are only one byte here and uncommon, so it's not worth a # persistent buffer, either: - @self_pipe[0].kgio_tryread(11) + @self_pipe[0].read_nonblock(11, exception: false) end def awaken_master return if $$ != @master_pid - @self_pipe[1].kgio_trywrite('.') # wakeup master process from select + # wakeup master process from select + @self_pipe[1].write_nonblock('.', exception: false) end # reaps all unreaped workers @@ -446,11 +443,6 @@ class Unicorn::HttpServer Dir.chdir(START_CTX[:cwd]) cmd = [ START_CTX[0] ].concat(START_CTX[:argv]) - # avoid leaking FDs we don't know about, but let before_exec - # unset FD_CLOEXEC, if anything else in the app eventually - # relies on FD inheritence. - close_sockets_on_exec(listener_fds) - # exec(command, hash) works in at least 1.9.1+, but will only be # required in 1.9.4/2.0.0 at earliest. cmd << listener_fds @@ -472,29 +464,15 @@ class Unicorn::HttpServer worker_info = [worker.nr, worker.to_io.fileno, worker.master.fileno] env['UNICORN_WORKER'] = worker_info.join(',') - close_sockets_on_exec(listener_fds) - Process.spawn(env, START_CTX[0], *START_CTX[:argv], listener_fds) end def listener_sockets listener_fds = {} - LISTENERS.each do |sock| - sock.close_on_exec = false - listener_fds[sock.fileno] = sock - end + LISTENERS.each { |sock| listener_fds[sock.fileno] = sock } listener_fds end - def close_sockets_on_exec(sockets) - (3..1024).each do |io| - next if sockets.include?(io) - io = IO.for_fd(io) rescue next - io.autoclose = false - io.close_on_exec = true - end - end - # forcibly terminate all workers that haven't checked in in timeout seconds. The timeout is implemented using an unlinked File def murder_lazy_workers next_sleep = @timeout - 1 @@ -582,16 +560,25 @@ class Unicorn::HttpServer 500 end if code - client.kgio_trywrite(err_response(code, @request.response_start_sent)) + code = err_response(code, @request.response_start_sent) + client.write_nonblock(code, exception: false) end client.close rescue end + def e103_response_write(client, headers) + rss = @request.response_start_sent + buf = rss ? "103 Early Hints\r\n" : "HTTP/1.1 103 Early Hints\r\n" + headers.each { |key, value| append_header(buf, key, value) } + buf << (rss ? "\r\nHTTP/1.1 ".freeze : "\r\n".freeze) + client.write(buf) + end + def e100_response_write(client, env) # We use String#freeze to avoid allocations under Ruby 2.1+ # Not many users hit this code path, so it's better to reduce the - # constant table sizes even for 1.9.3-2.0 users who'll hit extra + # constant table sizes even for Ruby 2.0 users who'll hit extra # allocations here. client.write(@request.response_start_sent ? "100 Continue\r\n\r\nHTTP/1.1 ".freeze : @@ -601,8 +588,19 @@ class Unicorn::HttpServer # once a client is accepted, it is processed in its entirety here # in 3 easy steps: read request, call app, write app response - def process_client(client) - status, headers, body = @app.call(env = @request.read(client)) + def process_client(client, ai) + @request = Unicorn::HttpRequest.new + env = @request.read_headers(client, ai) + + if early_hints + env["rack.early_hints"] = lambda do |headers| + e103_response_write(client, headers) + end + end + + env["rack.after_reply"] = [] + + status, headers, body = @app.call(env) begin return if @request.hijacked? @@ -624,6 +622,8 @@ class Unicorn::HttpServer end rescue => e handle_error(client, e) + ensure + env["rack.after_reply"].each(&:call) if env end def nuke_listeners!(readers) @@ -654,7 +654,6 @@ class Unicorn::HttpServer LISTENERS.each { |sock| sock.close_on_exec = true } worker.user(*user) if user.kind_of?(Array) && ! worker.switched - self.timeout /= 2.0 # halve it for select() @config = nil build_app! unless preload_app @after_fork = @listener_opts = @orig_app = nil @@ -668,58 +667,54 @@ class Unicorn::HttpServer logger.info "worker=#{worker_nr} reopening logs..." Unicorn::Util.reopen_logs logger.info "worker=#{worker_nr} done reopening logs" + false rescue => e logger.error(e) rescue nil exit!(77) # EX_NOPERM in sysexits.h end + def prep_readers(readers) + wtr = Unicorn::Waiter.prep_readers(readers) + @timeout *= 500 # to milliseconds for epoll, but halved + wtr + rescue + require_relative 'select_waiter' + @timeout /= 2.0 # halved for IO.select + Unicorn::SelectWaiter.new + end + # runs inside each forked worker, this sits around and waits # for connections and doesn't die until the parent dies (or is # given a INT, QUIT, or TERM signal) def worker_loop(worker) - ppid = @master_pid readers = init_worker_process(worker) - nr = 0 # this becomes negative if we need to reopen logs + waiter = prep_readers(readers) + reopen = false # this only works immediately if the master sent us the signal # (which is the normal case) - trap(:USR1) { nr = -65536 } + trap(:USR1) { reopen = true } ready = readers.dup @after_worker_ready.call(self, worker) begin - nr < 0 and reopen_worker_logs(worker.nr) - nr = 0 + reopen = reopen_worker_logs(worker.nr) if reopen worker.tick = time_now.to_i - tmp = ready.dup - while sock = tmp.shift - # Unicorn::Worker#kgio_tryaccept is not like accept(2) at all, - # but that will return false - if client = sock.kgio_tryaccept - process_client(client) - nr += 1 + while sock = ready.shift + client_ai = sock.accept_nonblock(exception: false) + if client_ai != :wait_readable + process_client(*client_ai) worker.tick = time_now.to_i end - break if nr < 0 + break if reopen end - # make the following bet: if we accepted clients this round, - # we're probably reasonably busy, so avoid calling select() - # and do a speculative non-blocking accept() on ready listeners - # before we sleep again in select(). - unless nr == 0 - tmp = ready.dup - redo - end - - ppid == Process.ppid or return - - # timeout used so we can detect parent death: + # timeout so we can .tick and keep parent from SIGKILL-ing us worker.tick = time_now.to_i - ret = IO.select(readers, nil, nil, @timeout) and ready = ret[0] + waiter.get_readers(ready, readers, @timeout) rescue => e - redo if nr < 0 && readers[0] + redo if reopen && readers[0] Unicorn.log_error(@logger, "listen loop error", e) if readers[0] end while readers[0] end @@ -807,21 +802,21 @@ class Unicorn::HttpServer def inherit_listeners! # inherit sockets from parents, they need to be plain Socket objects - # before they become Kgio::UNIXServer or Kgio::TCPServer inherited = ENV['UNICORN_FD'].to_s.split(',') + immortal = [] # emulate sd_listen_fds() for systemd sd_pid, sd_fds = ENV.values_at('LISTEN_PID', 'LISTEN_FDS') if sd_pid.to_i == $$ # n.b. $$ can never be zero # 3 = SD_LISTEN_FDS_START - inherited.concat((3...(3 + sd_fds.to_i)).to_a) + immortal = (3...(3 + sd_fds.to_i)).to_a + inherited.concat(immortal) end # to ease debugging, we will not unset LISTEN_PID and LISTEN_FDS inherited.map! do |fd| io = Socket.for_fd(fd.to_i) - io.autoclose = false - io = server_cast(io) + @immortal << io if immortal.include?(fd) set_server_sockopt(io, listener_opts[sock_name(io)]) logger.info "inherited addr=#{sock_name(io)} fd=#{io.fileno}" io @@ -830,11 +825,9 @@ class Unicorn::HttpServer config_listeners = config[:listeners].dup LISTENERS.replace(inherited) - # we start out with generic Socket objects that get cast to either - # Kgio::TCPServer or Kgio::UNIXServer objects; but since the Socket - # objects share the same OS-level file descriptor as the higher-level - # *Server objects; we need to prevent Socket objects from being - # garbage-collected + # we only use generic Socket objects for aggregate Socket#accept_nonblock + # return value [ Socket, Addrinfo ]. This allows us to avoid having to + # make getpeername(2) syscalls later on to fill in env['REMOTE_ADDR'] config_listeners -= listener_names if config_listeners.empty? && LISTENERS.empty? config_listeners << Unicorn::Const::DEFAULT_LISTEN diff --git a/lib/unicorn/oob_gc.rb b/lib/unicorn/oob_gc.rb index c4741a0..db9f2cb 100644 --- a/lib/unicorn/oob_gc.rb +++ b/lib/unicorn/oob_gc.rb @@ -43,8 +43,8 @@ # use Unicorn::OobGC, 2, %r{\A/(?:expensive/foo|more_expensive/foo)} # # Feedback from users of early implementations of this module: -# * https://bogomips.org/unicorn-public/0BFC98E9-072B-47EE-9A70-05478C20141B@lukemelia.com/ -# * https://bogomips.org/unicorn-public/AANLkTilUbgdyDv9W1bi-s_W6kq9sOhWfmuYkKLoKGOLj@mail.gmail.com/ +# * https://yhbt.net/unicorn-public/0BFC98E9-072B-47EE-9A70-05478C20141B@lukemelia.com/ +# * https://yhbt.net/unicorn-public/AANLkTilUbgdyDv9W1bi-s_W6kq9sOhWfmuYkKLoKGOLj@mail.gmail.com/ module Unicorn::OobGC @@ -60,17 +60,17 @@ module Unicorn::OobGC self.const_set :OOBGC_INTERVAL, interval ObjectSpace.each_object(Unicorn::HttpServer) do |s| s.extend(self) - self.const_set :OOBGC_ENV, s.instance_variable_get(:@request).env end app # pretend to be Rack middleware since it was in the past end #:stopdoc: - def process_client(client) - super(client) # Unicorn::HttpServer#process_client - if OOBGC_PATH =~ OOBGC_ENV['PATH_INFO'] && ((@@nr -= 1) <= 0) + def process_client(*args) + super(*args) # Unicorn::HttpServer#process_client + env = instance_variable_get(:@request).env + if OOBGC_PATH =~ env['PATH_INFO'] && ((@@nr -= 1) <= 0) @@nr = OOBGC_INTERVAL - OOBGC_ENV.clear + env.clear disabled = GC.enable GC.start GC.disable if disabled diff --git a/lib/unicorn/select_waiter.rb b/lib/unicorn/select_waiter.rb new file mode 100644 index 0000000..cb84aab --- /dev/null +++ b/lib/unicorn/select_waiter.rb @@ -0,0 +1,6 @@ +# fallback for non-Linux and Linux <4.5 systems w/o EPOLLEXCLUSIVE +class Unicorn::SelectWaiter # :nodoc: + def get_readers(ready, readers, timeout) # :nodoc: + ret = IO.select(readers, nil, nil, timeout) and ready.replace(ret[0]) + end +end diff --git a/lib/unicorn/socket_helper.rb b/lib/unicorn/socket_helper.rb index 8a6f6ee..06ec2b2 100644 --- a/lib/unicorn/socket_helper.rb +++ b/lib/unicorn/socket_helper.rb @@ -3,18 +3,6 @@ require 'socket' module Unicorn - - # Instead of using a generic Kgio::Socket for everything, - # tag TCP sockets so we can use TCP_INFO under Linux without - # incurring extra syscalls for Unix domain sockets. - # TODO: remove these when we remove kgio - TCPClient = Class.new(Kgio::Socket) # :nodoc: - class TCPSrv < Kgio::TCPServer # :nodoc: - def kgio_tryaccept # :nodoc: - super(TCPClient) - end - end - module SocketHelper # internal interface @@ -91,7 +79,7 @@ module Unicorn def set_server_sockopt(sock, opt) opt = DEFAULTS.merge(opt || {}) - TCPSocket === sock and set_tcp_sockopt(sock, opt) + set_tcp_sockopt(sock, opt) if sock.local_address.ip? rcvbuf, sndbuf = opt.values_at(:rcvbuf, :sndbuf) if rcvbuf || sndbuf @@ -135,7 +123,9 @@ module Unicorn end old_umask = File.umask(opt[:umask] || 0) begin - Kgio::UNIXServer.new(address) + s = Socket.new(:UNIX, :STREAM) + s.bind(Socket.sockaddr_un(address)) + s ensure File.umask(old_umask) end @@ -163,8 +153,7 @@ module Unicorn sock.setsockopt(:SOL_SOCKET, :SO_REUSEPORT, 1) end sock.bind(Socket.pack_sockaddr_in(port, addr)) - sock.autoclose = false - TCPSrv.for_fd(sock.fileno) + sock end # returns rfc2732-style (e.g. "[::1]:666") addresses for IPv6 @@ -180,10 +169,6 @@ module Unicorn def sock_name(sock) case sock when String then sock - when UNIXServer - Socket.unpack_sockaddr_un(sock.getsockname) - when TCPServer - tcp_name(sock) when Socket begin tcp_name(sock) @@ -196,16 +181,5 @@ module Unicorn end module_function :sock_name - - # casts a given Socket to be a TCPServer or UNIXServer - def server_cast(sock) - begin - Socket.unpack_sockaddr_in(sock.getsockname) - TCPSrv.for_fd(sock.fileno) - rescue ArgumentError - Kgio::UNIXServer.for_fd(sock.fileno) - end - end - end # module SocketHelper end # module Unicorn diff --git a/lib/unicorn/stream_input.rb b/lib/unicorn/stream_input.rb index 41d28a0..9246f73 100644 --- a/lib/unicorn/stream_input.rb +++ b/lib/unicorn/stream_input.rb @@ -49,8 +49,7 @@ class Unicorn::StreamInput to_read = length - @rbuf.size rv.replace(@rbuf.slice!(0, @rbuf.size)) until to_read == 0 || eof? || (rv.size > 0 && @chunked) - @socket.kgio_read(to_read, @buf) or eof! - filter_body(@rbuf, @buf) + filter_body(@rbuf, @socket.readpartial(to_read, @buf)) rv << @rbuf to_read -= @rbuf.size end @@ -61,6 +60,8 @@ class Unicorn::StreamInput read_all(rv) end rv + rescue EOFError + return eof! end # :call-seq: @@ -83,9 +84,10 @@ class Unicorn::StreamInput begin @rbuf.sub!(re, '') and return $1 return @rbuf.empty? ? nil : @rbuf.slice!(0, @rbuf.size) if eof? - @socket.kgio_read(@@io_chunk_size, @buf) or eof! - filter_body(once = '', @buf) + filter_body(once = '', @socket.readpartial(@@io_chunk_size, @buf)) @rbuf << once + rescue EOFError + return eof! end while true end @@ -107,14 +109,15 @@ private def eof? if @parser.body_eof? while @chunked && ! @parser.parse - once = @socket.kgio_read(@@io_chunk_size) or eof! - @buf << once + @buf << @socket.readpartial(@@io_chunk_size) end @socket = nil true else false end + rescue EOFError + return eof! end def filter_body(dst, src) @@ -127,10 +130,11 @@ private dst.replace(@rbuf) @socket or return until eof? - @socket.kgio_read(@@io_chunk_size, @buf) or eof! - filter_body(@rbuf, @buf) + filter_body(@rbuf, @socket.readpartial(@@io_chunk_size, @buf)) dst << @rbuf end + rescue EOFError + return eof! ensure @rbuf.clear end diff --git a/lib/unicorn/tmpio.rb b/lib/unicorn/tmpio.rb index db88ed3..0bbf6ec 100644 --- a/lib/unicorn/tmpio.rb +++ b/lib/unicorn/tmpio.rb @@ -11,12 +11,18 @@ class Unicorn::TmpIO < File # immediately, switched to binary mode, and userspace output # buffering is disabled def self.new + path = nil + + # workaround File#path being tainted: + # https://bugs.ruby-lang.org/issues/14485 fp = begin - super("#{Dir::tmpdir}/#{rand}", RDWR|CREAT|EXCL, 0600) + path = "#{Dir::tmpdir}/#{rand}" + super(path, RDWR|CREAT|EXCL, 0600) rescue Errno::EEXIST retry end - unlink(fp.path) + + unlink(path) fp.binmode fp.sync = true fp diff --git a/lib/unicorn/worker.rb b/lib/unicorn/worker.rb index 5ddf379..4af31be 100644 --- a/lib/unicorn/worker.rb +++ b/lib/unicorn/worker.rb @@ -65,15 +65,15 @@ class Unicorn::Worker end # writing and reading 4 bytes on a pipe is atomic on all POSIX platforms # Do not care in the odd case the buffer is full, here. - @master.kgio_trywrite([signum].pack('l')) + @master.write_nonblock([signum].pack('l'), exception: false) rescue Errno::EPIPE # worker will be reaped soon end # this only runs when the Rack app.call is not running - # act like a listener - def kgio_tryaccept # :nodoc: - case buf = @to_io.kgio_tryread(4) + # act like Socket#accept_nonblock(exception: false) + def accept_nonblock(*_unused) # :nodoc: + case buf = @to_io.read_nonblock(4, exception: false) when String # unpack the buffer and trigger the signal handler signum = buf.unpack('l') @@ -82,7 +82,7 @@ class Unicorn::Worker when nil # EOF: master died, but we are at a safe place to exit fake_sig(:QUIT) when :wait_readable # keep waiting - return false + return :wait_readable end while true # loop, as multiple signals may be sent end diff --git a/t/GNUmakefile b/t/GNUmakefile index 5f5d9bc..0ac9b9a 100644 --- a/t/GNUmakefile +++ b/t/GNUmakefile @@ -1,74 +1,5 @@ -# we can run tests in parallel with GNU make +# there used to be more, here, but we stopped relying on recursive make all:: + $(MAKE) -C .. test-integration -pid := $(shell echo $$PPID) - -RUBY = ruby -RAKE = rake --include ../local.mk -ifeq ($(RUBY_VERSION),) - RUBY_VERSION := $(shell $(RUBY) -e 'puts RUBY_VERSION') -endif - -ifeq ($(RUBY_VERSION),) - $(error unable to detect RUBY_VERSION) -endif - -RUBY_ENGINE := $(shell $(RUBY) -e 'puts((RUBY_ENGINE rescue "ruby"))') -export RUBY_ENGINE - -MYLIBS := $(RUBYLIB) - -T = $(wildcard t[0-9][0-9][0-9][0-9]-*.sh) - -all:: $(T) - -# can't rely on "set -o pipefail" since we don't require bash or ksh93 :< -t_pfx = trash/$@-$(RUBY_ENGINE)-$(RUBY_VERSION) -TEST_OPTS = -# TRACER = strace -f -o $(t_pfx).strace -s 100000 -# TRACER = /usr/bin/time -o $(t_pfx).time - -ifdef V - ifeq ($(V),2) - TEST_OPTS += --trace - else - TEST_OPTS += --verbose - endif -endif - -random_blob: - dd if=/dev/urandom bs=1M count=30 of=$@.$(pid) - mv $@.$(pid) $@ - -$(T): random_blob - -dependencies := socat curl -deps := $(addprefix .dep+,$(dependencies)) -$(deps): dep_bin = $(lastword $(subst +, ,$@)) -$(deps): - @which $(dep_bin) > $@.$(pid) 2>/dev/null || : - @test -s $@.$(pid) || \ - { echo >&2 "E '$(dep_bin)' not found in PATH=$(PATH)"; exit 1; } - @mv $@.$(pid) $@ -dep: $(deps) - -test_prefix := $(CURDIR)/../test/$(RUBY_ENGINE)-$(RUBY_VERSION) -$(test_prefix)/.stamp: - $(MAKE) -C .. test-install - -$(T): export RUBY := $(RUBY) -$(T): export RAKE := $(RAKE) -$(T): export PATH := $(test_prefix)/bin:$(PATH) -$(T): export RUBYLIB := $(test_prefix)/lib:$(MYLIBS) -$(T): dep $(test_prefix)/.stamp trash/.gitignore - $(TRACER) $(SHELL) $(SH_TEST_OPTS) $@ $(TEST_OPTS) - -trash/.gitignore: - mkdir -p $(@D) - echo '*' > $@ - -clean: - $(RM) -r trash/* - -.PHONY: $(T) clean +.PHONY: all @@ -5,16 +5,19 @@ TCP ports or Unix domain sockets. They're all designed to run concurrently with other tests to minimize test time, but tests may be run independently as well. -We write our tests in Bourne shell because that's what we're -comfortable writing integration tests with. +New tests are written in Perl 5 because we need a stable language +to test real-world behavior and Ruby introduces incompatibilities +at a far faster rate than Perl 5. Perl is Ruby's older cousin, so +it should be easy-to-learn for Rubyists. + +Old tests are in Bourne shell and slowly being ported to Perl 5. == Requirements -* {Ruby 1.9.3+}[https://www.ruby-lang.org/en/] (duh!) +* {Ruby 2.5.0+}[https://www.ruby-lang.org/en/] +* {Perl 5.14+}[https://www.perl.org/] # your distro should have it * {GNU make}[https://www.gnu.org/software/make/] -* {socat}[http://www.dest-unreach.org/socat/] * {curl}[https://curl.haxx.se/] -* standard UNIX shell utilities (Bourne sh, awk, sed, grep, ...) We do not use bashisms or any non-portable, non-POSIX constructs in our shell code. We use the "pipefail" option if available and @@ -26,9 +29,13 @@ with {dash}[http://gondor.apana.org.au/~herbert/dash/] and To run the entire test suite with 8 tests running at once: - make -j8 + make -j8 && prove -vw + +To run one individual test (Perl5): + + prove -vw t/integration.t -To run one individual test: +To run one individual test (shell): make t0000-simple-http.sh diff --git a/t/active-unix-socket.t b/t/active-unix-socket.t new file mode 100644 index 0000000..ff731b5 --- /dev/null +++ b/t/active-unix-socket.t @@ -0,0 +1,117 @@ +#!perl -w +# Copyright (C) unicorn hackers <unicorn-public@yhbt.net> +# License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> + +use v5.14; BEGIN { require './t/lib.perl' }; +use IO::Socket::UNIX; +use autodie; +no autodie 'kill'; +my %to_kill; +END { kill('TERM', values(%to_kill)) if keys %to_kill } +my $u1 = "$tmpdir/u1.sock"; +my $u2 = "$tmpdir/u2.sock"; +{ + open my $fh, '>', "$tmpdir/u1.conf.rb"; + print $fh <<EOM; +pid "$tmpdir/u.pid" +listen "$u1" +stderr_path "$err_log" +EOM + close $fh; + + open $fh, '>', "$tmpdir/u2.conf.rb"; + print $fh <<EOM; +pid "$tmpdir/u.pid" +listen "$u2" +stderr_path "$tmpdir/err2.log" +EOM + close $fh; + + open $fh, '>', "$tmpdir/u3.conf.rb"; + print $fh <<EOM; +pid "$tmpdir/u3.pid" +listen "$u1" +stderr_path "$tmpdir/err3.log" +EOM + close $fh; +} + +my @uarg = qw(-D -E none t/integration.ru); + +# this pipe will be used to notify us when all daemons die: +pipe(my $p0, my $p1); +fcntl($p1, POSIX::F_SETFD, 0); + +# start the first instance +unicorn('-c', "$tmpdir/u1.conf.rb", @uarg)->join; +is($?, 0, 'daemonized 1st process'); +chomp($to_kill{u1} = slurp("$tmpdir/u.pid")); +like($to_kill{u1}, qr/\A\d+\z/s, 'read pid file'); + +chomp(my $worker_pid = readline(unix_start($u1, 'GET /pid'))); +like($worker_pid, qr/\A\d+\z/s, 'captured worker pid'); +ok(kill(0, $worker_pid), 'worker is kill-able'); + + +# 2nd process conflicts on PID +unicorn('-c', "$tmpdir/u2.conf.rb", @uarg)->join; +isnt($?, 0, 'conflicting PID file fails to start'); + +chomp(my $pidf = slurp("$tmpdir/u.pid")); +is($pidf, $to_kill{u1}, 'pid file contents unchanged after start failure'); + +chomp(my $pid2 = readline(unix_start($u1, 'GET /pid'))); +is($worker_pid, $pid2, 'worker PID unchanged'); + + +# 3rd process conflicts on socket +unicorn('-c', "$tmpdir/u3.conf.rb", @uarg)->join; +isnt($?, 0, 'conflicting UNIX socket fails to start'); + +chomp($pid2 = readline(unix_start($u1, 'GET /pid'))); +is($worker_pid, $pid2, 'worker PID still unchanged'); + +chomp($pidf = slurp("$tmpdir/u.pid")); +is($pidf, $to_kill{u1}, 'pid file contents unchanged after 2nd start failure'); + +{ # teardown initial process via SIGKILL + ok(kill('KILL', delete $to_kill{u1}), 'SIGKILL initial daemon'); + close $p1; + vec(my $rvec = '', fileno($p0), 1) = 1; + is(select($rvec, undef, undef, 5), 1, 'timeout for pipe HUP'); + is(my $undef = <$p0>, undef, 'process closed pipe writer at exit'); + ok(-f "$tmpdir/u.pid", 'pid file stayed after SIGKILL'); + ok(-S $u1, 'socket stayed after SIGKILL'); + is(IO::Socket::UNIX->new(Peer => $u1, Type => SOCK_STREAM), undef, + 'fail to connect to u1'); + for (1..50) { # wait for init process to reap worker + kill(0, $worker_pid) or last; + sleep 0.011; + } + ok(!kill(0, $worker_pid), 'worker gone after parent dies'); +} + +# restart the first instance +{ + pipe($p0, $p1); + fcntl($p1, POSIX::F_SETFD, 0); + unicorn('-c', "$tmpdir/u1.conf.rb", @uarg)->join; + is($?, 0, 'daemonized 1st process'); + chomp($to_kill{u1} = slurp("$tmpdir/u.pid")); + like($to_kill{u1}, qr/\A\d+\z/s, 'read pid file'); + + chomp($pid2 = readline(unix_start($u1, 'GET /pid'))); + like($pid2, qr/\A\d+\z/, 'worker running'); + + ok(kill('TERM', delete $to_kill{u1}), 'SIGTERM restarted daemon'); + close $p1; + vec(my $rvec = '', fileno($p0), 1) = 1; + is(select($rvec, undef, undef, 5), 1, 'timeout for pipe HUP'); + is(my $undef = <$p0>, undef, 'process closed pipe writer at exit'); + ok(!-f "$tmpdir/u.pid", 'pid file gone after SIGTERM'); + ok(-S $u1, 'socket stays after SIGTERM'); +} + +check_stderr; +undef $tmpdir; +done_testing; diff --git a/t/bin/content-md5-put b/t/bin/content-md5-put deleted file mode 100755 index 01da0bb..0000000 --- a/t/bin/content-md5-put +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env ruby -# -*- encoding: binary -*- -# simple chunked HTTP PUT request generator (and just that), -# it reads stdin and writes to stdout so socat can write to a -# UNIX or TCP socket (or to another filter or file) along with -# a Content-MD5 trailer. -require 'digest/md5' -$stdout.sync = $stderr.sync = true -$stdout.binmode -$stdin.binmode - -bs = ENV['bs'] ? ENV['bs'].to_i : 4096 - -if ARGV.grep("--no-headers").empty? - $stdout.write( - "PUT / HTTP/1.1\r\n" \ - "Host: example.com\r\n" \ - "Transfer-Encoding: chunked\r\n" \ - "Trailer: Content-MD5\r\n" \ - "\r\n" - ) -end - -digest = Digest::MD5.new -if buf = $stdin.readpartial(bs) - begin - digest.update(buf) - $stdout.write("%x\r\n" % [ buf.size ]) - $stdout.write(buf) - $stdout.write("\r\n") - end while $stdin.read(bs, buf) -end - -digest = [ digest.digest ].pack('m').strip -$stdout.write("0\r\n") -$stdout.write("Content-MD5: #{digest}\r\n\r\n") diff --git a/t/bin/sha1sum.rb b/t/bin/sha1sum.rb deleted file mode 100755 index 53d68ce..0000000 --- a/t/bin/sha1sum.rb +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env ruby -# -*- encoding: binary -*- -# Reads from stdin and outputs the SHA1 hex digest of the input - -require 'digest/sha1' -$stdout.sync = $stderr.sync = true -$stdout.binmode -$stdin.binmode -bs = 16384 -digest = Digest::SHA1.new -if buf = $stdin.read(bs) - begin - digest.update(buf) - end while $stdin.read(bs, buf) -end - -$stdout.syswrite("#{digest.hexdigest}\n") diff --git a/t/t0116.ru b/t/client_body_buffer_size.ru index fab5fce..44161a5 100644 --- a/t/t0116.ru +++ b/t/client_body_buffer_size.ru @@ -1,6 +1,4 @@ #\ -E none -use Rack::ContentLength -use Rack::ContentType, 'text/plain' app = lambda do |env| input = env['rack.input'] case env["PATH_INFO"] diff --git a/t/client_body_buffer_size.t b/t/client_body_buffer_size.t new file mode 100644 index 0000000..d479901 --- /dev/null +++ b/t/client_body_buffer_size.t @@ -0,0 +1,80 @@ +#!perl -w +# Copyright (C) unicorn hackers <unicorn-public@yhbt.net> +# License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> + +use v5.14; BEGIN { require './t/lib.perl' }; +use autodie; +open my $conf_fh, '>', $u_conf; +$conf_fh->autoflush(1); +print $conf_fh <<EOM; +client_body_buffer_size 0 +EOM +my $srv = tcp_server(); +my $host_port = tcp_host_port($srv); +my @uarg = (qw(-E none t/client_body_buffer_size.ru -c), $u_conf); +my $ar = unicorn(@uarg, { 3 => $srv }); +my ($c, $status, $hdr); +my $mem_class = 'StringIO'; +my $fs_class = 'Unicorn::TmpIO'; + +$c = tcp_start($srv, "PUT /input_class HTTP/1.0\r\nContent-Length: 0"); +($status, $hdr) = slurp_hdr($c); +like($status, qr!\AHTTP/1\.[01] 200\b!, 'status line valid'); +is(readline($c), $mem_class, 'zero-byte file is StringIO'); + +$c = tcp_start($srv, "PUT /tmp_class HTTP/1.0\r\nContent-Length: 1"); +print $c '.'; +($status, $hdr) = slurp_hdr($c); +like($status, qr!\AHTTP/1\.[01] 200\b!, 'status line valid'); +is(readline($c), $fs_class, '1 byte file is filesystem-backed'); + + +my $fifo = "$tmpdir/fifo"; +POSIX::mkfifo($fifo, 0600) or die "mkfifo: $!"; +seek($conf_fh, 0, SEEK_SET); +truncate($conf_fh, 0); +print $conf_fh <<EOM; +after_fork { |_,_| File.open('$fifo', 'w') { |fp| fp.write "pid=#\$\$" } } +EOM +$ar->do_kill('HUP'); +open my $fifo_fh, '<', $fifo; +like(my $wpid = readline($fifo_fh), qr/\Apid=\d+\z/a , + 'reloaded w/ default client_body_buffer_size'); + + +$c = tcp_start($srv, "PUT /tmp_class HTTP/1.0\r\nContent-Length: 1"); +($status, $hdr) = slurp_hdr($c); +like($status, qr!\AHTTP/1\.[01] 200\b!, 'status line valid'); +is(readline($c), $mem_class, 'class for a 1 byte file is memory-backed'); + + +my $one_meg = 1024 ** 2; +$c = tcp_start($srv, "PUT /tmp_class HTTP/1.0\r\nContent-Length: $one_meg"); +($status, $hdr) = slurp_hdr($c); +like($status, qr!\AHTTP/1\.[01] 200\b!, 'status line valid'); +is(readline($c), $fs_class, '1 megabyte file is FS-backed'); + +# reload with bigger client_body_buffer_size +say $conf_fh "client_body_buffer_size $one_meg"; +$ar->do_kill('HUP'); +open $fifo_fh, '<', $fifo; +like($wpid = readline($fifo_fh), qr/\Apid=\d+\z/a , + 'reloaded w/ bigger client_body_buffer_size'); + + +$c = tcp_start($srv, "PUT /tmp_class HTTP/1.0\r\nContent-Length: $one_meg"); +($status, $hdr) = slurp_hdr($c); +like($status, qr!\AHTTP/1\.[01] 200\b!, 'status line valid'); +is(readline($c), $mem_class, '1 megabyte file is now memory-backed'); + +my $too_big = $one_meg + 1; +$c = tcp_start($srv, "PUT /tmp_class HTTP/1.0\r\nContent-Length: $too_big"); +($status, $hdr) = slurp_hdr($c); +like($status, qr!\AHTTP/1\.[01] 200\b!, 'status line valid'); +is(readline($c), $fs_class, '1 megabyte + 1 byte file is FS-backed'); + + +undef $ar; +check_stderr; +undef $tmpdir; +done_testing; diff --git a/t/heartbeat-timeout.ru b/t/heartbeat-timeout.ru index d9904e8..3eeb5d6 100644 --- a/t/heartbeat-timeout.ru +++ b/t/heartbeat-timeout.ru @@ -1,5 +1,5 @@ use Rack::ContentLength -headers = { 'Content-Type' => 'text/plain' } +headers = { 'content-type' => 'text/plain' } run lambda { |env| case env['PATH_INFO'] when "/block-forever" @@ -7,6 +7,6 @@ run lambda { |env| sleep # in case STOP signal is not received in time [ 500, headers, [ "Should never get here\n" ] ] else - [ 200, headers, [ "#$$\n" ] ] + [ 200, headers, [ "#$$" ] ] end } diff --git a/t/heartbeat-timeout.t b/t/heartbeat-timeout.t new file mode 100644 index 0000000..694867a --- /dev/null +++ b/t/heartbeat-timeout.t @@ -0,0 +1,62 @@ +#!perl -w +# Copyright (C) unicorn hackers <unicorn-public@yhbt.net> +# License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> +use v5.14; BEGIN { require './t/lib.perl' }; +use autodie; +use Time::HiRes qw(clock_gettime CLOCK_MONOTONIC); +mkdir "$tmpdir/alt"; +my $srv = tcp_server(); +open my $fh, '>', $u_conf; +print $fh <<EOM; +pid "$tmpdir/pid" +preload_app true +stderr_path "$err_log" +timeout 3 # WORST FEATURE EVER +EOM +close $fh; + +my $ar = unicorn(qw(-E none t/heartbeat-timeout.ru -c), $u_conf, { 3 => $srv }); + +my ($status, $hdr, $wpid) = do_req($srv, 'GET /pid HTTP/1.0'); +like($status, qr!\AHTTP/1\.[01] 200\b!, 'PID request succeeds'); +like($wpid, qr/\A[0-9]+\z/, 'worker is running'); + +my $t0 = clock_gettime(CLOCK_MONOTONIC); +my $c = tcp_start($srv, 'GET /block-forever HTTP/1.0'); +vec(my $rvec = '', fileno($c), 1) = 1; +is(select($rvec, undef, undef, 6), 1, 'got readiness'); +$c->blocking(0); +is(sysread($c, my $buf, 128), 0, 'got EOF response'); +my $elapsed = clock_gettime(CLOCK_MONOTONIC) - $t0; +ok($elapsed > 3, 'timeout took >3s'); + +my @timeout_err = slurp($err_log); +truncate($err_log, 0); +is(grep(/timeout \(\d+s > 3s\), killing/, @timeout_err), 1, + 'noted timeout error') or diag explain(\@timeout_err); + +# did it respawn? +($status, $hdr, my $new_pid) = do_req($srv, 'GET /pid HTTP/1.0'); +like($status, qr!\AHTTP/1\.[01] 200\b!, 'PID request succeeds'); +isnt($new_pid, $wpid, 'spawned new worker'); + +diag 'SIGSTOP for 4 seconds...'; +$ar->do_kill('STOP'); +sleep 4; +$ar->do_kill('CONT'); +for my $i (1..2) { + ($status, $hdr, my $spid) = do_req($srv, 'GET /pid HTTP/1.0'); + like($status, qr!\AHTTP/1\.[01] 200\b!, + "PID request succeeds #$i after STOP+CONT"); + is($new_pid, $spid, "worker pid unchanged after STOP+CONT #$i"); + if ($i == 1) { + diag 'sleeping 2s to ensure timeout is not delayed'; + sleep 2; + } +} + +$ar->join('TERM'); +check_stderr; +undef $tmpdir; + +done_testing; diff --git a/t/hijack.ru b/t/hijack.ru deleted file mode 100644 index 02260e2..0000000 --- a/t/hijack.ru +++ /dev/null @@ -1,55 +0,0 @@ -use Rack::Lint -use Rack::ContentLength -use Rack::ContentType, "text/plain" -class DieIfUsed - @@n = 0 - def each - abort "body.each called after response hijack\n" - end - - def close - warn "closed DieIfUsed #{@@n += 1}\n" - end -end - -envs = [] - -run lambda { |env| - case env["PATH_INFO"] - when "/hijack_req" - if env["rack.hijack?"] - io = env["rack.hijack"].call - envs << env - if io.respond_to?(:read_nonblock) && - env["rack.hijack_io"].respond_to?(:read_nonblock) - - # exercise both, since we Rack::Lint may use different objects - env["rack.hijack_io"].write("HTTP/1.0 200 OK\r\n\r\n") - io.write("request.hijacked") - io.close - return [ 500, {}, DieIfUsed.new ] - end - end - [ 500, {}, [ "hijack BAD\n" ] ] - when "/hijack_res" - r = "response.hijacked" - [ 200, - { - "Content-Length" => r.bytesize.to_s, - "rack.hijack" => proc do |io| - envs << env - io.write(r) - io.close - end - }, - DieIfUsed.new - ] - when "/normal_env_id" - b = "#{env.object_id}\n" - h = { - 'Content-Type' => 'text/plain', - 'Content-Length' => b.bytesize.to_s, - } - [ 200, h, [ b ] ] - end -} diff --git a/t/integration.ru b/t/integration.ru new file mode 100644 index 0000000..888833a --- /dev/null +++ b/t/integration.ru @@ -0,0 +1,115 @@ +#!ruby +# Copyright (C) unicorn hackers <unicorn-public@80x24.org> +# License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> + +# this goes for t/integration.t We'll try to put as many tests +# in here as possible to avoid startup overhead of Ruby. + +def early_hints(env, val) + env['rack.early_hints'].call('link' => val) # val may be ary or string + [ 200, {}, [ val.class.to_s ] ] +end + +$orig_rack_200 = nil +def tweak_status_code + $orig_rack_200 = Rack::Utils::HTTP_STATUS_CODES[200] + Rack::Utils::HTTP_STATUS_CODES[200] = "HI" + [ 200, {}, [] ] +end + +def restore_status_code + $orig_rack_200 or return [ 500, {}, [] ] + Rack::Utils::HTTP_STATUS_CODES[200] = $orig_rack_200 + [ 200, {}, [] ] +end + +class WriteOnClose + def each(&block) + @callback = block + end + + def close + @callback.call "7\r\nGoodbye\r\n0\r\n\r\n" + end +end + +def write_on_close + [ 200, { 'transfer-encoding' => 'chunked' }, WriteOnClose.new ] +end + +def env_dump(env) + require 'json' + h = {} + env.each do |k,v| + case v + when String, Integer, true, false; h[k] = v + else + case k + when 'rack.version', 'rack.after_reply'; h[k] = v + end + end + end + h.to_json +end + +def rack_input_tests(env) + return [ 100, {}, [] ] if /\A100-continue\z/i =~ env['HTTP_EXPECT'] + cap = 16384 + require 'digest/md5' + dig = Digest::MD5.new + input = env['rack.input'] + case env['PATH_INFO'] + when '/rack_input/size_first'; input.size + when '/rack_input/rewind_first'; input.rewind + when '/rack_input'; # OK + else + abort "bad path: #{env['PATH_INFO']}" + end + if buf = input.read(rand(cap)) + begin + raise "#{buf.size} > #{cap}" if buf.size > cap + dig.update(buf) + end while input.read(rand(cap), buf) + buf.clear # remove this call if Ruby ever gets escape analysis + end + h = { 'content-type' => 'text/plain' } + if env['HTTP_TRAILER'] =~ /\bContent-MD5\b/i + cmd5_b64 = env['HTTP_CONTENT_MD5'] or return [500, {}, ['No Content-MD5']] + cmd5_bin = cmd5_b64.unpack('m')[0] + if cmd5_bin != dig.digest + h['content-length'] = cmd5_b64.size.to_s + return [ 500, h, [ cmd5_b64 ] ] + end + end + h['content-length'] = '32' + [ 200, h, [ dig.hexdigest ] ] +end + +run(lambda do |env| + case env['REQUEST_METHOD'] + when 'GET' + case env['PATH_INFO'] + when '/rack-2-newline-headers'; [ 200, { 'X-R2' => "a\nb\nc" }, [] ] + when '/rack-3-array-headers'; [ 200, { 'x-r3' => %w(a b c) }, [] ] + when '/nil-header-value'; [ 200, { 'X-Nil' => nil }, [] ] + when '/unknown-status-pass-through'; [ '666 I AM THE BEAST', {}, [] ] + when '/env_dump'; [ 200, {}, [ env_dump(env) ] ] + when '/write_on_close'; write_on_close + when '/pid'; [ 200, {}, [ "#$$\n" ] ] + when '/early_hints_rack2'; early_hints(env, "r\n2") + when '/early_hints_rack3'; early_hints(env, %w(r 3)) + when '/broken_app'; raise RuntimeError, 'hello' + else '/'; [ 200, {}, [ env_dump(env) ] ] + end # case PATH_INFO (GET) + when 'POST' + case env['PATH_INFO'] + when '/tweak-status-code'; tweak_status_code + when '/restore-status-code'; restore_status_code + end # case PATH_INFO (POST) + # ... + when 'PUT' + case env['PATH_INFO'] + when %r{\A/rack_input}; rack_input_tests(env) + end + end # case REQUEST_METHOD +end) # run diff --git a/t/integration.t b/t/integration.t new file mode 100644 index 0000000..7310ff2 --- /dev/null +++ b/t/integration.t @@ -0,0 +1,356 @@ +#!perl -w +# Copyright (C) unicorn hackers <unicorn-public@yhbt.net> +# License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> + +# This is the main integration test for fast-ish things to minimize +# Ruby startup time penalties. + +use v5.14; BEGIN { require './t/lib.perl' }; +use autodie; +use Socket qw(SOL_SOCKET SO_KEEPALIVE SHUT_WR); +our $srv = tcp_server(); +our $host_port = tcp_host_port($srv); + +if ('ensure Perl does not set SO_KEEPALIVE by default') { + my $val = getsockopt($srv, SOL_SOCKET, SO_KEEPALIVE); + unpack('i', $val) == 0 or + setsockopt($srv, SOL_SOCKET, SO_KEEPALIVE, pack('i', 0)); + $val = getsockopt($srv, SOL_SOCKET, SO_KEEPALIVE); +} +my $t0 = time; +open my $conf_fh, '>', $u_conf; +$conf_fh->autoflush(1); +my $u1 = "$tmpdir/u1"; +print $conf_fh <<EOM; +early_hints true +listen "$u1" +EOM +my $ar = unicorn(qw(-E none t/integration.ru -c), $u_conf, { 3 => $srv }); +my $curl = which('curl'); +my $fifo = "$tmpdir/fifo"; +POSIX::mkfifo($fifo, 0600) or die "mkfifo: $!"; +my %PUT = ( + chunked_md5 => sub { + my ($in, $out, $path, %opt) = @_; + my $dig = Digest::MD5->new; + print $out <<EOM; +PUT $path HTTP/1.1\r +Transfer-Encoding: chunked\r +Trailer: Content-MD5\r +\r +EOM + my ($buf, $r); + while (1) { + $r = read($in, $buf, 999 + int(rand(0xffff))); + last if $r == 0; + printf $out "%x\r\n", length($buf); + print $out $buf, "\r\n"; + $dig->add($buf); + } + print $out "0\r\nContent-MD5: ", $dig->b64digest, "\r\n\r\n"; + }, + identity => sub { + my ($in, $out, $path, %opt) = @_; + my $clen = $opt{-s} // -s $in; + print $out <<EOM; +PUT $path HTTP/1.0\r +Content-Length: $clen\r +\r +EOM + my ($buf, $r, $len, $bs); + while ($clen) { + $bs = 999 + int(rand(0xffff)); + $len = $clen > $bs ? $bs : $clen; + $r = read($in, $buf, $len); + die 'premature EOF' if $r == 0; + print $out $buf; + $clen -= $r; + } + }, +); + +my ($c, $status, $hdr, $bdy); + +# response header tests +($status, $hdr) = do_req($srv, 'GET /rack-2-newline-headers HTTP/1.0'); +like($status, qr!\AHTTP/1\.[01] 200\b!, 'status line valid'); +my $orig_200_status = $status; +is_deeply([ grep(/^X-R2: /, @$hdr) ], + [ 'X-R2: a', 'X-R2: b', 'X-R2: c' ], + 'rack 2 LF-delimited headers supported') or diag(explain($hdr)); + +{ + my $val = getsockopt($srv, SOL_SOCKET, SO_KEEPALIVE); + is(unpack('i', $val), 1, 'SO_KEEPALIVE set on inherited socket'); +} + +SKIP: { # Date header check + my @d = grep(/^Date: /i, @$hdr); + is(scalar(@d), 1, 'got one date header') or diag(explain(\@d)); + eval { require HTTP::Date } or skip "HTTP::Date missing: $@", 1; + $d[0] =~ s/^Date: //i or die 'BUG: did not strip date: prefix'; + my $t = HTTP::Date::str2time($d[0]); + my $now = time; + ok($t >= ($t0 - 1) && $t > 0 && $t <= ($now + 1), 'valid date') or + diag(explain(["t=$t t0=$t0 now=$now", $!, \@d])); +}; + + +($status, $hdr) = do_req($srv, 'GET /rack-3-array-headers HTTP/1.0'); +is_deeply([ grep(/^x-r3: /, @$hdr) ], + [ 'x-r3: a', 'x-r3: b', 'x-r3: c' ], + 'rack 3 array headers supported') or diag(explain($hdr)); + +SKIP: { + eval { require JSON::PP } or skip "JSON::PP missing: $@", 1; + ($status, $hdr, my $json) = do_req $srv, 'GET /env_dump'; + is($status, undef, 'no status for HTTP/0.9'); + is($hdr, undef, 'no header for HTTP/0.9'); + unlike($json, qr/^Connection: /smi, 'no connection header for 0.9'); + unlike($json, qr!\AHTTP/!s, 'no HTTP/1.x prefix for 0.9'); + my $env = JSON::PP->new->decode($json); + is(ref($env), 'HASH', 'JSON decoded body to hashref'); + is($env->{SERVER_PROTOCOL}, 'HTTP/0.9', 'SERVER_PROTOCOL is 0.9'); +} + +# cf. <CAO47=rJa=zRcLn_Xm4v2cHPr6c0UswaFC_omYFEH+baSxHOWKQ@mail.gmail.com> +($status, $hdr) = do_req($srv, 'GET /nil-header-value HTTP/1.0'); +is_deeply([grep(/^X-Nil:/, @$hdr)], ['X-Nil: '], + 'nil header value accepted for broken apps') or diag(explain($hdr)); + +check_stderr; +($status, $hdr, $bdy) = do_req($srv, 'GET /broken_app HTTP/1.0'); +like($status, qr!\AHTTP/1\.[0-1] 500\b!, 'got 500 error on broken endpoint'); +is($bdy, undef, 'no response body after exception'); +truncate($errfh, 0); + +my $ck_early_hints = sub { + my ($note) = @_; + $c = unix_start($u1, 'GET /early_hints_rack2 HTTP/1.0'); + ($status, $hdr) = slurp_hdr($c); + like($status, qr!\AHTTP/1\.[01] 103\b!, 'got 103 for rack 2 value'); + is_deeply(['link: r', 'link: 2'], $hdr, 'rack 2 hints match '.$note); + ($status, $hdr) = slurp_hdr($c); + like($status, qr!\AHTTP/1\.[01] 200\b!, 'got 200 afterwards'); + is(readline($c), 'String', 'early hints used a String for rack 2'); + + $c = unix_start($u1, 'GET /early_hints_rack3 HTTP/1.0'); + ($status, $hdr) = slurp_hdr($c); + like($status, qr!\AHTTP/1\.[01] 103\b!, 'got 103 for rack 3'); + is_deeply(['link: r', 'link: 3'], $hdr, 'rack 3 hints match '.$note); + ($status, $hdr) = slurp_hdr($c); + like($status, qr!\AHTTP/1\.[01] 200\b!, 'got 200 afterwards'); + is(readline($c), 'Array', 'early hints used a String for rack 3'); +}; +$ck_early_hints->('ccc off'); # we'll retest later + +if ('TODO: ensure Rack::Utils::HTTP_STATUS_CODES is available') { + ($status, $hdr) = do_req $srv, 'POST /tweak-status-code HTTP/1.0'; + like($status, qr!\AHTTP/1\.[01] 200 HI\b!, 'status tweaked'); + + ($status, $hdr) = do_req $srv, 'POST /restore-status-code HTTP/1.0'; + is($status, $orig_200_status, 'original status restored'); +} + +SKIP: { + eval { require HTTP::Tiny } or skip "HTTP::Tiny missing: $@", 1; + my $ht = HTTP::Tiny->new; + my $res = $ht->get("http://$host_port/write_on_close"); + is($res->{content}, 'Goodbye', 'write-on-close body read'); +} + +if ('bad requests') { + ($status, $hdr) = do_req $srv, 'GET /env_dump HTTP/1/1'; + like($status, qr!\AHTTP/1\.[01] 400 \b!, 'got 400 on bad request'); + + $c = tcp_start($srv); + print $c 'GET /'; + my $buf = join('', (0..9), 'ab'); + for (0..1023) { print $c $buf } + print $c " HTTP/1.0\r\n\r\n"; + ($status, $hdr) = slurp_hdr($c); + like($status, qr!\AHTTP/1\.[01] 414 \b!, + '414 on REQUEST_PATH > (12 * 1024)'); + + $c = tcp_start($srv); + print $c 'GET /hello-world?a'; + $buf = join('', (0..9)); + for (0..1023) { print $c $buf } + print $c " HTTP/1.0\r\n\r\n"; + ($status, $hdr) = slurp_hdr($c); + like($status, qr!\AHTTP/1\.[01] 414 \b!, + '414 on QUERY_STRING > (10 * 1024)'); + + $c = tcp_start($srv); + print $c 'GET /hello-world#a'; + $buf = join('', (0..9), 'a'..'f'); + for (0..63) { print $c $buf } + print $c " HTTP/1.0\r\n\r\n"; + ($status, $hdr) = slurp_hdr($c); + like($status, qr!\AHTTP/1\.[01] 414 \b!, '414 on FRAGMENT > (1024)'); +} + +# input tests +my ($blob_size, $blob_hash); +SKIP: { + skip 'SKIP_EXPENSIVE on', 1 if $ENV{SKIP_EXPENSIVE}; + CORE::open(my $rh, '<', 't/random_blob') or + skip "t/random_blob not generated $!", 1; + $blob_size = -s $rh; + require Digest::MD5; + $blob_hash = Digest::MD5->new->addfile($rh)->hexdigest; + + my $ck_hash = sub { + my ($sub, $path, %opt) = @_; + seek($rh, 0, SEEK_SET); + $c = tcp_start($srv); + $c->autoflush($opt{sync} // 0); + $PUT{$sub}->($rh, $c, $path, %opt); + defined($opt{overwrite}) and + print { $c } ('x' x $opt{overwrite}); + $c->flush or die $!; + shutdown($c, SHUT_WR); + ($status, $hdr) = slurp_hdr($c); + is(readline($c), $blob_hash, "$sub $path"); + }; + $ck_hash->('identity', '/rack_input', -s => $blob_size); + $ck_hash->('chunked_md5', '/rack_input'); + $ck_hash->('identity', '/rack_input/size_first', -s => $blob_size); + $ck_hash->('identity', '/rack_input/rewind_first', -s => $blob_size); + $ck_hash->('chunked_md5', '/rack_input/size_first'); + $ck_hash->('chunked_md5', '/rack_input/rewind_first'); + + $ck_hash->('identity', '/rack_input', -s => $blob_size, sync => 1); + $ck_hash->('chunked_md5', '/rack_input', sync => 1); + + # ensure small overwrites don't get checksummed + $ck_hash->('identity', '/rack_input', -s => $blob_size, + overwrite => 1); # one extra byte + unlike(slurp($err_log), qr/ClientShutdown/, + 'no overreads after client SHUT_WR'); + + # excessive overwrite truncated + $c = tcp_start($srv); + $c->autoflush(0); + print $c "PUT /rack_input HTTP/1.0\r\nContent-Length: 1\r\n\r\n"; + if (1) { + local $SIG{PIPE} = 'IGNORE'; + my $buf = "\0" x 8192; + my $n = 0; + my $end = time + 5; + $! = 0; + while (print $c $buf and time < $end) { ++$n } + ok($!, 'overwrite truncated') or diag "n=$n err=$! ".time; + undef $c; + } + + # client shutdown early + $c = tcp_start($srv); + $c->autoflush(0); + print $c "PUT /rack_input HTTP/1.0\r\nContent-Length: 16384\r\n\r\n"; + if (1) { + local $SIG{PIPE} = 'IGNORE'; + print $c 'too short body'; + shutdown($c, SHUT_WR); + vec(my $rvec = '', fileno($c), 1) = 1; + select($rvec, undef, undef, 10) or BAIL_OUT "timed out"; + my $buf = <$c>; + is($buf, undef, 'server aborted after client SHUT_WR'); + undef $c; + } + + $curl // skip 'no curl found in PATH', 1; + + my ($copt, $cout); + my $url = "http://$host_port/rack_input"; + my $do_curl = sub { + my (@arg) = @_; + pipe(my $cout, $copt->{1}); + open $copt->{2}, '>', "$tmpdir/curl.err"; + my $cpid = spawn($curl, '-sSf', @arg, $url, $copt); + close(delete $copt->{1}); + is(readline($cout), $blob_hash, "curl @arg response"); + is(waitpid($cpid, 0), $cpid, "curl @arg exited"); + is($?, 0, "no error from curl @arg"); + is(slurp("$tmpdir/curl.err"), '', "no stderr from curl @arg"); + }; + + $do_curl->(qw(-T t/random_blob)); + + seek($rh, 0, SEEK_SET); + $copt->{0} = $rh; + $do_curl->('-T-'); + + diag 'testing Unicorn::PrereadInput...'; + local $srv = tcp_server(); + local $host_port = tcp_host_port($srv); + check_stderr; + truncate($errfh, 0); + + my $pri = unicorn(qw(-E none t/preread_input.ru), { 3 => $srv }); + $url = "http://$host_port/"; + + $do_curl->(qw(-T t/random_blob)); + seek($rh, 0, SEEK_SET); + $copt->{0} = $rh; + $do_curl->('-T-'); + + my @pr_err = slurp("$tmpdir/err.log"); + is(scalar(grep(/app dispatch:/, @pr_err)), 2, 'app dispatched twice'); + + # abort a chunked request by blocking curl on a FIFO: + $c = tcp_start($srv, "PUT / HTTP/1.1\r\nTransfer-Encoding: chunked"); + close $c; + @pr_err = slurp("$tmpdir/err.log"); + is(scalar(grep(/app dispatch:/, @pr_err)), 2, + 'app did not dispatch on aborted request'); + undef $pri; + check_stderr; + diag 'Unicorn::PrereadInput middleware tests done'; +} + +# ... more stuff here + +# SIGHUP-able stuff goes here + +if ('check_client_connection') { + print $conf_fh <<EOM; # appending to existing +check_client_connection true +after_fork { |_,_| File.open('$fifo', 'w') { |fp| fp.write "pid=#\$\$" } } +EOM + $ar->do_kill('HUP'); + open my $fifo_fh, '<', $fifo; + my $wpid = readline($fifo_fh); + like($wpid, qr/\Apid=\d+\z/a , 'new worker ready'); + $ck_early_hints->('ccc on'); +} + +if ('max_header_len internal API') { + undef $c; + my $req = 'GET / HTTP/1.0'; + my $len = length($req."\r\n\r\n"); + print $conf_fh <<EOM; # appending to existing +Unicorn::HttpParser.max_header_len = $len +EOM + $ar->do_kill('HUP'); + open my $fifo_fh, '<', $fifo; + my $wpid = readline($fifo_fh); + like($wpid, qr/\Apid=\d+\z/a , 'new worker ready'); + close $fifo_fh; + $wpid =~ s/\Apid=// or die; + ok(CORE::kill(0, $wpid), 'worker PID retrieved'); + + ($status, $hdr) = do_req($srv, $req); + like($status, qr!\AHTTP/1\.[01] 200\b!, 'minimal request succeeds'); + + ($status, $hdr) = do_req($srv, 'GET /xxxxxx HTTP/1.0'); + like($status, qr!\AHTTP/1\.[01] 413\b!, 'big request fails'); +} + + +undef $ar; + +check_stderr; + +undef $tmpdir; +done_testing; diff --git a/t/lib.perl b/t/lib.perl new file mode 100644 index 0000000..9254b23 --- /dev/null +++ b/t/lib.perl @@ -0,0 +1,258 @@ +#!perl -w +# Copyright (C) unicorn hackers <unicorn-public@80x24.org> +# License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> +package UnicornTest; +use v5.14; +use parent qw(Exporter); +use autodie; +use Test::More; +use Time::HiRes qw(sleep time); +use IO::Socket::INET; +use POSIX qw(dup2 _exit setpgid :signal_h SEEK_SET F_SETFD); +use File::Temp 0.19 (); # 0.19 for ->newdir +our ($tmpdir, $errfh, $err_log, $u_sock, $u_conf, $daemon_pid, + $pid_file); +our @EXPORT = qw(unicorn slurp tcp_server tcp_start unicorn + $tmpdir $errfh $err_log $u_sock $u_conf $daemon_pid $pid_file + SEEK_SET tcp_host_port which spawn check_stderr unix_start slurp_hdr + do_req stop_daemon sleep time); + +my ($base) = ($0 =~ m!\b([^/]+)\.[^\.]+\z!); +$tmpdir = File::Temp->newdir("unicorn-$base-XXXX", TMPDIR => 1); +$err_log = "$tmpdir/err.log"; +$pid_file = "$tmpdir/pid"; +$u_sock = "$tmpdir/u.sock"; +$u_conf = "$tmpdir/u.conf.rb"; +open($errfh, '>>', $err_log); + +sub stop_daemon (;$) { + my ($is_END) = @_; + kill('TERM', $daemon_pid); + my $tries = 1000; + while (CORE::kill(0, $daemon_pid) && --$tries) { sleep(0.01) } + if ($is_END && CORE::kill(0, $daemon_pid)) { # after done_testing + CORE::kill('KILL', $daemon_pid); + die "daemon_pid=$daemon_pid did not die"; + } else { + ok(!CORE::kill(0, $daemon_pid), 'daemonized unicorn gone'); + undef $daemon_pid; + } +}; + +END { + diag slurp($err_log) if $tmpdir; + stop_daemon(1) if defined $daemon_pid; +}; + +sub check_stderr () { + my @log = slurp($err_log); + diag("@log") if $ENV{V}; + my @err = grep(!/NameError.*Unicorn::Waiter/, grep(/error/i, @log)); + @err = grep(!/failed to set accept_filter=/, @err); + @err = grep(!/perhaps accf_.*? needs to be loaded/, @err); + is_deeply(\@err, [], 'no unexpected errors in stderr'); + is_deeply([grep(/SIGKILL/, @log)], [], 'no SIGKILL in stderr'); +} + +sub slurp_hdr { + my ($c) = @_; + local $/ = "\r\n\r\n"; # affects both readline+chomp + chomp(my $hdr = readline($c)); + my ($status, @hdr) = split(/\r\n/, $hdr); + diag explain([ $status, \@hdr ]) if $ENV{V}; + ($status, \@hdr); +} + +sub tcp_server { + my %opt = ( + ReuseAddr => 1, + Proto => 'tcp', + Type => SOCK_STREAM, + Listen => SOMAXCONN, + Blocking => 0, + @_, + ); + eval { + die 'IPv4-only' if $ENV{TEST_IPV4_ONLY}; + require IO::Socket::INET6; + IO::Socket::INET6->new(%opt, LocalAddr => '[::1]') + } || eval { + die 'IPv6-only' if $ENV{TEST_IPV6_ONLY}; + IO::Socket::INET->new(%opt, LocalAddr => '127.0.0.1') + } || BAIL_OUT "failed to create TCP server: $! ($@)"; +} + +sub tcp_host_port { + my ($s) = @_; + my ($h, $p) = ($s->sockhost, $s->sockport); + my $ipv4 = $s->sockdomain == AF_INET; + if (wantarray) { + $ipv4 ? ($h, $p) : ("[$h]", $p); + } else { + $ipv4 ? "$h:$p" : "[$h]:$p"; + } +} + +sub unix_start ($@) { + my ($dst, @req) = @_; + my $s = IO::Socket::UNIX->new(Peer => $dst, Type => SOCK_STREAM) or + BAIL_OUT "unix connect $dst: $!"; + $s->autoflush(1); + print $s @req, "\r\n\r\n" if @req; + $s; +} + +sub tcp_start ($@) { + my ($dst, @req) = @_; + my $addr = tcp_host_port($dst); + my $s = ref($dst)->new( + Proto => 'tcp', + Type => SOCK_STREAM, + PeerAddr => $addr, + ) or BAIL_OUT "failed to connect to $addr: $!"; + $s->autoflush(1); + print $s @req, "\r\n\r\n" if @req; + $s; +} + +sub slurp { + open my $fh, '<', $_[0]; + local $/ if !wantarray; + readline($fh); +} + +sub spawn { + my $env = ref($_[0]) eq 'HASH' ? shift : undef; + my $opt = ref($_[-1]) eq 'HASH' ? pop : {}; + my @cmd = @_; + my $old = POSIX::SigSet->new; + my $set = POSIX::SigSet->new; + $set->fillset or die "sigfillset: $!"; + sigprocmask(SIG_SETMASK, $set, $old) or die "SIG_SETMASK: $!"; + pipe(my $r, my $w); + my $pid = fork; + if ($pid == 0) { + close $r; + $SIG{__DIE__} = sub { + warn(@_); + syswrite($w, my $num = $! + 0); + _exit(1); + }; + + # pretend to be systemd (cf. sd_listen_fds(3)) + my $cfd; + for ($cfd = 0; ($cfd < 3) || defined($opt->{$cfd}); $cfd++) { + my $io = $opt->{$cfd} // next; + my $pfd = fileno($io); + if ($pfd == $cfd) { + fcntl($io, F_SETFD, 0); + } else { + dup2($pfd, $cfd) // die "dup2($pfd, $cfd): $!"; + } + } + if (($cfd - 3) > 0) { + $env->{LISTEN_PID} = $$; + $env->{LISTEN_FDS} = $cfd - 3; + } + + if (defined(my $pgid = $opt->{pgid})) { + setpgid(0, $pgid) // die "setpgid(0, $pgid): $!"; + } + $SIG{$_} = 'DEFAULT' for grep(!/^__/, keys %SIG); + if (defined(my $cd = $opt->{-C})) { chdir $cd } + $old->delset(POSIX::SIGCHLD) or die "sigdelset CHLD: $!"; + sigprocmask(SIG_SETMASK, $old) or die "SIG_SETMASK: ~CHLD: $!"; + @ENV{keys %$env} = values(%$env) if $env; + exec { $cmd[0] } @cmd; + die "exec @cmd: $!"; + } + close $w; + sigprocmask(SIG_SETMASK, $old) or die "SIG_SETMASK(old): $!"; + if (my $cerrnum = do { local $/, <$r> }) { + $! = $cerrnum; + die "@cmd PID=$pid died: $!"; + } + $pid; +} + +sub which { + my ($file) = @_; + return $file if index($file, '/') >= 0; + for my $p (split(/:/, $ENV{PATH})) { + $p .= "/$file"; + return $p if -x $p; + } + undef; +} + +# returns an AutoReap object +sub unicorn { + my %env; + if (ref($_[0]) eq 'HASH') { + my $e = shift; + %env = %$e; + } + my @args = @_; + push(@args, {}) if ref($args[-1]) ne 'HASH'; + $args[-1]->{2} //= $errfh; # stderr default + + state $ruby = which($ENV{RUBY} // 'ruby'); + state $lib = File::Spec->rel2abs('lib'); + state $ver = $ENV{TEST_RUBY_VERSION} // `$ruby -e 'print RUBY_VERSION'`; + state $eng = $ENV{TEST_RUBY_ENGINE} // `$ruby -e 'print RUBY_ENGINE'`; + state $ext = File::Spec->rel2abs("test/$eng-$ver/ext/unicorn_http"); + state $exe = File::Spec->rel2abs('bin/unicorn'); + my $pid = spawn(\%env, $ruby, '-I', $lib, '-I', $ext, $exe, @args); + UnicornTest::AutoReap->new($pid); +} + +sub do_req ($@) { + my ($dst, @req) = @_; + my $c = ref($dst) ? tcp_start($dst, @req) : unix_start($dst, @req); + return $c if !wantarray; + my ($status, $hdr); + # read headers iff HTTP/1.x request, HTTP/0.9 remains supported + my ($first) = (join('', @req) =~ m!\A([^\r\n]+)!); + ($status, $hdr) = slurp_hdr($c) if $first =~ m{\s*HTTP/\S+$}; + my $bdy = do { local $/; <$c> }; + close $c; + ($status, $hdr, $bdy); +} + +# automatically kill + reap children when this goes out-of-scope +package UnicornTest::AutoReap; +use v5.14; +use autodie; + +sub new { + my (undef, $pid) = @_; + bless { pid => $pid, owner => $$ }, __PACKAGE__ +} + +sub do_kill { + my ($self, $sig) = @_; + kill($sig // 'TERM', $self->{pid}); +} + +sub join { + my ($self, $sig) = @_; + my $pid = delete $self->{pid} or return; + kill($sig, $pid) if defined $sig; + my $ret = waitpid($pid, 0); + $ret == $pid or die "BUG: waitpid($pid) != $ret"; +} + +sub DESTROY { + my ($self) = @_; + return if $self->{owner} != $$; + $self->join('TERM'); +} + +package main; # inject ourselves into the t/*.t script +UnicornTest->import; +Test::More->import; +# try to ensure ->DESTROY fires: +$SIG{TERM} = sub { exit(15 + 128) }; +$SIG{INT} = sub { exit(2 + 128) }; +$SIG{PIPE} = sub { exit(13 + 128) }; +1; diff --git a/t/oob_gc.ru b/t/oob_gc.ru index c253540..224cb06 100644 --- a/t/oob_gc.ru +++ b/t/oob_gc.ru @@ -7,9 +7,6 @@ $gc_started = false # Mock GC.start def GC.start - ObjectSpace.each_object(Kgio::Socket) do |x| - x.closed? or abort "not closed #{x}" - end $gc_started = true end run lambda { |env| diff --git a/t/oob_gc_path.ru b/t/oob_gc_path.ru index af8e3b9..7f40601 100644 --- a/t/oob_gc_path.ru +++ b/t/oob_gc_path.ru @@ -7,9 +7,6 @@ $gc_started = false # Mock GC.start def GC.start - ObjectSpace.each_object(Kgio::Socket) do |x| - x.closed? or abort "not closed #{x}" - end $gc_started = true end run lambda { |env| diff --git a/t/preread_input.ru b/t/preread_input.ru index 79685c4..18af221 100644 --- a/t/preread_input.ru +++ b/t/preread_input.ru @@ -1,17 +1,22 @@ #\-E none -require 'digest/sha1' +require 'digest/md5' require 'unicorn/preread_input' -use Rack::ContentLength -use Rack::ContentType, "text/plain" use Unicorn::PrereadInput nr = 0 run lambda { |env| $stderr.write "app dispatch: #{nr += 1}\n" input = env["rack.input"] - dig = Digest::SHA1.new - while buf = input.read(16384) - dig.update(buf) + dig = Digest::MD5.new + if buf = input.read(16384) + begin + dig.update(buf) + end while input.read(16384, buf) + buf.clear # remove this call if Ruby ever gets escape analysis end - - [ 200, {}, [ "#{dig.hexdigest}\n" ] ] + if env['HTTP_TRAILER'] =~ /\bContent-MD5\b/i + cmd5_b64 = env['HTTP_CONTENT_MD5'] or return [500, {}, ['No Content-MD5']] + cmd5_bin = cmd5_b64.unpack('m')[0] + return [500, {}, [ cmd5_b64 ] ] if cmd5_bin != dig.digest + end + [ 200, {}, [ dig.hexdigest ] ] } diff --git a/t/rack-input-tests.ru b/t/rack-input-tests.ru deleted file mode 100644 index 8c35630..0000000 --- a/t/rack-input-tests.ru +++ /dev/null @@ -1,21 +0,0 @@ -# SHA1 checksum generator -require 'digest/sha1' -use Rack::ContentLength -cap = 16384 -app = lambda do |env| - /\A100-continue\z/i =~ env['HTTP_EXPECT'] and - return [ 100, {}, [] ] - digest = Digest::SHA1.new - input = env['rack.input'] - input.size if env["PATH_INFO"] == "/size_first" - input.rewind if env["PATH_INFO"] == "/rewind_first" - if buf = input.read(rand(cap)) - begin - raise "#{buf.size} > #{cap}" if buf.size > cap - digest.update(buf) - end while input.read(rand(cap), buf) - end - - [ 200, {'Content-Type' => 'text/plain'}, [ digest.hexdigest << "\n" ] ] -end -run app diff --git a/t/reload-bad-config.t b/t/reload-bad-config.t new file mode 100644 index 0000000..c023b88 --- /dev/null +++ b/t/reload-bad-config.t @@ -0,0 +1,54 @@ +#!perl -w +# Copyright (C) unicorn hackers <unicorn-public@yhbt.net> +# License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> +use v5.14; BEGIN { require './t/lib.perl' }; +use autodie; +my $srv = tcp_server(); +my $host_port = tcp_host_port($srv); +my $ru = "$tmpdir/config.ru"; +my $u_conf = "$tmpdir/u.conf.rb"; + +open my $fh, '>', $ru; +print $fh <<'EOM'; +use Rack::ContentLength +use Rack::ContentType, 'text/plain' +config = ru = "hello world\n" # check for config variable conflicts, too +run lambda { |env| [ 200, {}, [ ru.to_s ] ] } +EOM +close $fh; + +open $fh, '>', $u_conf; +print $fh <<EOM; +preload_app true +stderr_path "$err_log" +EOM +close $fh; + +my $ar = unicorn(qw(-E none -c), $u_conf, $ru, { 3 => $srv }); +my ($status, $hdr, $bdy) = do_req($srv, 'GET / HTTP/1.0'); +like($status, qr!\AHTTP/1\.[01] 200\b!, 'status line valid at start'); +is($bdy, "hello world\n", 'body matches expected'); + +open $fh, '>>', $ru; +say $fh '....this better be a syntax error in any version of ruby...'; +close $fh; + +$ar->do_kill('HUP'); # reload +my @l; +for (1..1000) { + @l = grep(/(?:done|error) reloading/, slurp($err_log)) and + last; + sleep 0.011; +} +diag slurp($err_log) if $ENV{V}; +ok(grep(/error reloading/, @l), 'got error reloading'); +open $fh, '>', $err_log; +close $fh; + +($status, $hdr, $bdy) = do_req($srv, 'GET / HTTP/1.0'); +like($status, qr!\AHTTP/1\.[01] 200\b!, 'status line valid afte reload'); +is($bdy, "hello world\n", 'body matches expected after reload'); + +check_stderr; +undef $tmpdir; # quiet t/lib.perl END{} +done_testing; diff --git a/t/t0006.ru b/t/reopen-logs.ru index c39e8f6..c39e8f6 100644 --- a/t/t0006.ru +++ b/t/reopen-logs.ru diff --git a/t/reopen-logs.t b/t/reopen-logs.t new file mode 100644 index 0000000..76a4dbd --- /dev/null +++ b/t/reopen-logs.t @@ -0,0 +1,39 @@ +#!perl -w +# Copyright (C) unicorn hackers <unicorn-public@yhbt.net> +# License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> +use v5.14; BEGIN { require './t/lib.perl' }; +use autodie; +my $srv = tcp_server(); +my $u_conf = "$tmpdir/u.conf.rb"; +my $out_log = "$tmpdir/out.log"; +open my $fh, '>', $u_conf; +print $fh <<EOM; +stderr_path "$err_log" +stdout_path "$out_log" +EOM +close $fh; + +my $auto_reap = unicorn('-c', $u_conf, 't/reopen-logs.ru', { 3 => $srv } ); +my ($status, $hdr, $bdy) = do_req($srv, 'GET / HTTP/1.0'); +is($bdy, "true\n", 'logs opened'); + +rename($err_log, "$err_log.rot"); +rename($out_log, "$out_log.rot"); + +$auto_reap->do_kill('USR1'); + +my $tries = 1000; +while (!-f $err_log && --$tries) { sleep 0.01 }; +while (!-f $out_log && --$tries) { sleep 0.01 }; + +ok(-f $out_log, 'stdout_path recreated after USR1'); +ok(-f $err_log, 'stderr_path recreated after USR1'); + +($status, $hdr, $bdy) = do_req($srv, 'GET / HTTP/1.0'); +is($bdy, "true\n", 'logs reopened with sync==true'); + +$auto_reap->join('QUIT'); +is($?, 0, 'no error on exit'); +check_stderr; +undef $tmpdir; +done_testing; diff --git a/t/t0000-http-basic.sh b/t/t0000-http-basic.sh deleted file mode 100755 index 8ab58ac..0000000 --- a/t/t0000-http-basic.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/sh -. ./test-lib.sh -t_plan 8 "simple HTTP connection tests" - -t_begin "setup and start" && { - unicorn_setup - unicorn -D -c $unicorn_config env.ru - unicorn_wait_start -} - -t_begin "single request" && { - curl -sSfv http://$listen/ -} - -t_begin "check stderr has no errors" && { - check_stderr -} - -t_begin "HTTP/0.9 request should not return headers" && { - ( - printf 'GET /\r\n' - cat $fifo > $tmp & - wait - echo ok > $ok - ) | socat - TCP:$listen > $fifo -} - -t_begin "env.inspect should've put everything on one line" && { - test 1 -eq $(count_lines < $tmp) -} - -t_begin "no headers in output" && { - if grep ^Connection: $tmp - then - die "Connection header found in $tmp" - elif grep ^HTTP/ $tmp - then - die "HTTP/ found in $tmp" - fi -} - -t_begin "killing succeeds" && { - kill $unicorn_pid -} - -t_begin "check stderr has no errors" && { - check_stderr -} - -t_done diff --git a/t/t0001-reload-bad-config.sh b/t/t0001-reload-bad-config.sh deleted file mode 100755 index 55bb355..0000000 --- a/t/t0001-reload-bad-config.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/sh -. ./test-lib.sh -t_plan 7 "reload config.ru error with preload_app true" - -t_begin "setup and start" && { - unicorn_setup - rtmpfiles ru - - cat > $ru <<\EOF -use Rack::ContentLength -use Rack::ContentType, "text/plain" -x = { "hello" => "world" } -run lambda { |env| [ 200, {}, [ x.inspect << "\n" ] ] } -EOF - echo 'preload_app true' >> $unicorn_config - unicorn -D -c $unicorn_config $ru - unicorn_wait_start -} - -t_begin "hit with curl" && { - out=$(curl -sSf http://$listen/) - test x"$out" = x'{"hello"=>"world"}' -} - -t_begin "introduce syntax error in rackup file" && { - echo '...' >> $ru -} - -t_begin "reload signal succeeds" && { - kill -HUP $unicorn_pid - while ! egrep '(done|error) reloading' $r_err >/dev/null - do - sleep 1 - done - - grep 'error reloading' $r_err >/dev/null - > $r_err -} - -t_begin "hit with curl" && { - out=$(curl -sSf http://$listen/) - test x"$out" = x'{"hello"=>"world"}' -} - -t_begin "killing succeeds" && { - kill $unicorn_pid -} - -t_begin "check stderr" && { - check_stderr -} - -t_done diff --git a/t/t0002-config-conflict.sh b/t/t0002-config-conflict.sh deleted file mode 100755 index d7b2181..0000000 --- a/t/t0002-config-conflict.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/sh -. ./test-lib.sh -t_plan 6 "config variables conflict with preload_app" - -t_begin "setup and start" && { - unicorn_setup - rtmpfiles ru rutmp - - cat > $ru <<\EOF -use Rack::ContentLength -use Rack::ContentType, "text/plain" -config = ru = { "hello" => "world" } -run lambda { |env| [ 200, {}, [ ru.inspect << "\n" ] ] } -EOF - echo 'preload_app true' >> $unicorn_config - unicorn -D -c $unicorn_config $ru - unicorn_wait_start -} - -t_begin "hit with curl" && { - out=$(curl -sSf http://$listen/) - test x"$out" = x'{"hello"=>"world"}' -} - -t_begin "modify rackup file" && { - sed -e 's/world/WORLD/' < $ru > $rutmp - mv $rutmp $ru -} - -t_begin "reload signal succeeds" && { - kill -HUP $unicorn_pid - while ! egrep '(done|error) reloading' < $r_err >/dev/null - do - sleep 1 - done - - grep 'done reloading' $r_err >/dev/null -} - -t_begin "hit with curl" && { - out=$(curl -sSf http://$listen/) - test x"$out" = x'{"hello"=>"WORLD"}' -} - -t_begin "killing succeeds" && { - kill $unicorn_pid -} - -t_done diff --git a/t/t0002-parser-error.sh b/t/t0002-parser-error.sh deleted file mode 100755 index 9dc1cd2..0000000 --- a/t/t0002-parser-error.sh +++ /dev/null @@ -1,94 +0,0 @@ -#!/bin/sh -. ./test-lib.sh -t_plan 11 "parser error test" - -t_begin "setup and startup" && { - unicorn_setup - unicorn -D env.ru -c $unicorn_config - unicorn_wait_start -} - -t_begin "send a bad request" && { - ( - printf 'GET / HTTP/1/1\r\nHost: example.com\r\n\r\n' - cat $fifo > $tmp & - wait - echo ok > $ok - ) | socat - TCP:$listen > $fifo - test xok = x$(cat $ok) -} - -dbgcat tmp - -t_begin "response should be a 400" && { - grep -F 'HTTP/1.1 400 Bad Request' $tmp -} - -t_begin "send a huge Request URI (REQUEST_PATH > (12 * 1024))" && { - rm -f $tmp - cat $fifo > $tmp & - ( - set -e - trap 'echo ok > $ok' EXIT - printf 'GET /' - for i in $(awk </dev/null 'BEGIN{for(i=0;i<1024;i++) print i}') - do - printf '0123456789ab' - done - printf ' HTTP/1.1\r\nHost: example.com\r\n\r\n' - ) | socat - TCP:$listen > $fifo || : - test xok = x$(cat $ok) - wait -} - -t_begin "response should be a 414 (REQUEST_PATH)" && { - grep -F 'HTTP/1.1 414 ' $tmp -} - -t_begin "send a huge Request URI (QUERY_STRING > (10 * 1024))" && { - rm -f $tmp - cat $fifo > $tmp & - ( - set -e - trap 'echo ok > $ok' EXIT - printf 'GET /hello-world?a' - for i in $(awk </dev/null 'BEGIN{for(i=0;i<1024;i++) print i}') - do - printf '0123456789' - done - printf ' HTTP/1.1\r\nHost: example.com\r\n\r\n' - ) | socat - TCP:$listen > $fifo || : - test xok = x$(cat $ok) - wait -} - -t_begin "response should be a 414 (QUERY_STRING)" && { - grep -F 'HTTP/1.1 414 ' $tmp -} - -t_begin "send a huge Request URI (FRAGMENT > 1024)" && { - rm -f $tmp - cat $fifo > $tmp & - ( - set -e - trap 'echo ok > $ok' EXIT - printf 'GET /hello-world#a' - for i in $(awk </dev/null 'BEGIN{for(i=0;i<64;i++) print i}') - do - printf '0123456789abcdef' - done - printf ' HTTP/1.1\r\nHost: example.com\r\n\r\n' - ) | socat - TCP:$listen > $fifo || : - test xok = x$(cat $ok) - wait -} - -t_begin "response should be a 414 (FRAGMENT)" && { - grep -F 'HTTP/1.1 414 ' $tmp -} - -t_begin "server stderr should be clean" && check_stderr - -t_begin "term signal sent" && kill $unicorn_pid - -t_done diff --git a/t/t0003-working_directory.sh b/t/t0003-working_directory.sh deleted file mode 100755 index 79988d8..0000000 --- a/t/t0003-working_directory.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/sh -. ./test-lib.sh - -t_plan 4 "config.ru inside alt working_directory" - -t_begin "setup and start" && { - unicorn_setup - rtmpfiles unicorn_config_tmp - rm -rf $t_pfx.app - mkdir $t_pfx.app - - cat > $t_pfx.app/config.ru <<EOF -#\--daemonize --host $host --port $port -use Rack::ContentLength -use Rack::ContentType, "text/plain" -run lambda { |env| [ 200, {}, [ "#{\$master_ppid}\\n" ] ] } -EOF - # we have --host/--port in config.ru instead - grep -v ^listen $unicorn_config > $unicorn_config_tmp - - # the whole point of this exercise - echo "working_directory '$t_pfx.app'" >> $unicorn_config_tmp - - # allows ppid to be 1 in before_fork - echo "preload_app true" >> $unicorn_config_tmp - cat >> $unicorn_config_tmp <<\EOF -before_fork do |server,worker| - $master_ppid = Process.ppid # should be zero to detect daemonization -end -EOF - - mv $unicorn_config_tmp $unicorn_config - - # rely on --daemonize switch, no & or -D - unicorn -c $unicorn_config - unicorn_wait_start -} - -t_begin "hit with curl" && { - body=$(curl -sSf http://$listen/) -} - -t_begin "killing succeeds" && { - kill $unicorn_pid -} - -t_begin "response body ppid == 1 (daemonized)" && { - test "$body" -eq 1 -} - -t_done diff --git a/t/t0004-heartbeat-timeout.sh b/t/t0004-heartbeat-timeout.sh deleted file mode 100755 index 2965283..0000000 --- a/t/t0004-heartbeat-timeout.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/sh -. ./test-lib.sh - -t_plan 11 "heartbeat/timeout test" - -t_begin "setup and startup" && { - unicorn_setup - echo timeout 3 >> $unicorn_config - echo preload_app true >> $unicorn_config - unicorn -D heartbeat-timeout.ru -c $unicorn_config - unicorn_wait_start -} - -t_begin "read worker PID" && { - worker_pid=$(curl -sSf http://$listen/) - t_info "worker_pid=$worker_pid" -} - -t_begin "sleep for a bit, ensure worker PID does not change" && { - sleep 4 - test $(curl -sSf http://$listen/) -eq $worker_pid -} - -t_begin "block the worker process to force it to die" && { - rm $ok - t0=$(unix_time) - err="$(curl -sSf http://$listen/block-forever 2>&1 || > $ok)" - t1=$(unix_time) - elapsed=$(($t1 - $t0)) - t_info "elapsed=$elapsed err=$err" - test x"$err" != x"Should never get here" - test x"$err" != x"$worker_pid" -} - -t_begin "ensure worker was killed" && { - test -e $ok - test 1 -eq $(grep timeout $r_err | grep killing | count_lines) -} - -t_begin "ensure timeout took at least 3 seconds" && { - test $elapsed -ge 3 -} - -t_begin "we get a fresh new worker process" && { - new_worker_pid=$(curl -sSf http://$listen/) - test $new_worker_pid -ne $worker_pid -} - -t_begin "truncate the server error log" && { - > $r_err -} - -t_begin "SIGSTOP and SIGCONT on unicorn master does not kill worker" && { - kill -STOP $unicorn_pid - sleep 4 - kill -CONT $unicorn_pid - sleep 2 - test $new_worker_pid -eq $(curl -sSf http://$listen/) -} - -t_begin "stop server" && { - kill -QUIT $unicorn_pid -} - -t_begin "check stderr" && check_stderr - -dbgcat r_err - -t_done diff --git a/t/t0004-working_directory_broken.sh b/t/t0004-working_directory_broken.sh deleted file mode 100755 index ca9d382..0000000 --- a/t/t0004-working_directory_broken.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/sh -. ./test-lib.sh - -t_plan 3 "config.ru is missing inside alt working_directory" - -t_begin "setup" && { - unicorn_setup - rtmpfiles unicorn_config_tmp ok - rm -rf $t_pfx.app - mkdir $t_pfx.app - - # the whole point of this exercise - echo "working_directory '$t_pfx.app'" >> $unicorn_config_tmp -} - -t_begin "fails to start up w/o config.ru" && { - unicorn -c $unicorn_config_tmp || echo ok > $ok -} - -t_begin "fallback code was run" && { - test x"$(cat $ok)" = xok -} - -t_done diff --git a/t/t0005-working_directory_app.rb.sh b/t/t0005-working_directory_app.rb.sh deleted file mode 100755 index 0fbab4f..0000000 --- a/t/t0005-working_directory_app.rb.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/sh -. ./test-lib.sh - -t_plan 4 "fooapp.rb inside alt working_directory" - -t_begin "setup and start" && { - unicorn_setup - rm -rf $t_pfx.app - mkdir $t_pfx.app - - cat > $t_pfx.app/fooapp.rb <<\EOF -class Fooapp - def self.call(env) - # Rack::Lint in 1.5.0 requires headers to be a hash - h = [%w(Content-Type text/plain), %w(Content-Length 2)] - h = Rack::Utils::HeaderHash.new(h) - [ 200, h, %w(HI) ] - end -end -EOF - # the whole point of this exercise - echo "working_directory '$t_pfx.app'" >> $unicorn_config - cd / - unicorn -D -c $unicorn_config -I. fooapp.rb - unicorn_wait_start -} - -t_begin "hit with curl" && { - body=$(curl -sSf http://$listen/) -} - -t_begin "killing succeeds" && { - kill $unicorn_pid -} - -t_begin "response body expected" && { - test x"$body" = xHI -} - -t_done diff --git a/t/t0006-reopen-logs.sh b/t/t0006-reopen-logs.sh deleted file mode 100755 index a6e7a17..0000000 --- a/t/t0006-reopen-logs.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/bin/sh -. ./test-lib.sh - -t_plan 15 "reopen rotated logs" - -t_begin "setup and startup" && { - rtmpfiles curl_out curl_err r_rot - unicorn_setup - unicorn -D t0006.ru -c $unicorn_config - unicorn_wait_start -} - -t_begin "ensure server is responsive" && { - test xtrue = x$(curl -sSf http://$listen/ 2> $curl_err) -} - -t_begin "ensure stderr log is clean" && check_stderr - -t_begin "external log rotation" && { - rm -f $r_rot - mv $r_err $r_rot -} - -t_begin "send reopen log signal (USR1)" && { - kill -USR1 $unicorn_pid -} - -t_begin "wait for rotated log to reappear" && { - nr=60 - while ! test -f $r_err && test $nr -ge 0 - do - sleep 1 - nr=$(( $nr - 1 )) - done -} - -t_begin "ensure server is still responsive" && { - test xtrue = x$(curl -sSf http://$listen/ 2> $curl_err) -} - -t_begin "wait for worker to reopen logs" && { - nr=60 - re="worker=.* done reopening logs" - while ! grep "$re" < $r_err >/dev/null && test $nr -ge 0 - do - sleep 1 - nr=$(( $nr - 1 )) - done -} - -dbgcat r_rot -dbgcat r_err - -t_begin "ensure no errors from curl" && { - test ! -s $curl_err -} - -t_begin "current server stderr is clean" && check_stderr - -t_begin "rotated stderr is clean" && { - check_stderr $r_rot -} - -t_begin "server is now writing logs to new stderr" && { - before_rot=$(count_bytes < $r_rot) - before_err=$(count_bytes < $r_err) - test xtrue = x$(curl -sSf http://$listen/ 2> $curl_err) - after_rot=$(count_bytes < $r_rot) - after_err=$(count_bytes < $r_err) - test $after_rot -eq $before_rot - test $after_err -gt $before_err -} - -t_begin "stop server" && { - kill $unicorn_pid -} - -dbgcat r_err - -t_begin "current server stderr is clean" && check_stderr -t_begin "rotated stderr is clean" && check_stderr $r_rot - -t_done diff --git a/t/t0007-working_directory_no_embed_cli.sh b/t/t0007-working_directory_no_embed_cli.sh deleted file mode 100755 index 77d6707..0000000 --- a/t/t0007-working_directory_no_embed_cli.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/sh -. ./test-lib.sh - -t_plan 4 "config.ru inside alt working_directory (no embedded switches)" - -t_begin "setup and start" && { - unicorn_setup - rm -rf $t_pfx.app - mkdir $t_pfx.app - - cat > $t_pfx.app/config.ru <<EOF -use Rack::ContentLength -use Rack::ContentType, "text/plain" -run lambda { |env| [ 200, {}, [ "#{\$master_ppid}\\n" ] ] } -EOF - # the whole point of this exercise - echo "working_directory '$t_pfx.app'" >> $unicorn_config - - # allows ppid to be 1 in before_fork - echo "preload_app true" >> $unicorn_config - cat >> $unicorn_config <<\EOF -before_fork do |server,worker| - $master_ppid = Process.ppid # should be zero to detect daemonization -end -EOF - - cd / - unicorn -D -c $unicorn_config - unicorn_wait_start -} - -t_begin "hit with curl" && { - body=$(curl -sSf http://$listen/) -} - -t_begin "killing succeeds" && { - kill $unicorn_pid -} - -t_begin "response body ppid == 1 (daemonized)" && { - test "$body" -eq 1 -} - -t_done diff --git a/t/t0009-winch_ttin.sh b/t/t0009-winch_ttin.sh deleted file mode 100755 index 6e56e30..0000000 --- a/t/t0009-winch_ttin.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/sh -. ./test-lib.sh -t_plan 8 "SIGTTIN succeeds after SIGWINCH" - -t_begin "setup and start" && { - unicorn_setup -cat >> $unicorn_config <<EOF -after_fork do |server, worker| - # test script will block while reading from $fifo, - File.open("$fifo", "wb") { |fp| fp.syswrite worker.nr.to_s } -end -EOF - unicorn -D -c $unicorn_config pid.ru - unicorn_wait_start - test 0 -eq $(cat $fifo) || die "worker.nr != 0" -} - -t_begin "read worker pid" && { - orig_worker_pid=$(curl -sSf http://$listen/) - test -n "$orig_worker_pid" && kill -0 $orig_worker_pid -} - -t_begin "stop all workers" && { - kill -WINCH $unicorn_pid -} - -# we have to do this next step before delivering TTIN -# signals aren't guaranteed to delivered in order -t_begin "wait for worker to die" && { - i=0 - while kill -0 $orig_worker_pid 2>/dev/null - do - i=$(( $i + 1 )) - test $i -lt 600 || die "timed out" - sleep 1 - done -} - -t_begin "start one worker back up" && { - kill -TTIN $unicorn_pid -} - -t_begin "wait for new worker to start" && { - test 0 -eq $(cat $fifo) || die "worker.nr != 0" - new_worker_pid=$(curl -sSf http://$listen/) - test -n "$new_worker_pid" && kill -0 $new_worker_pid - test $orig_worker_pid -ne $new_worker_pid || \ - die "worker wasn't replaced" -} - -t_begin "killing succeeds" && { - kill $unicorn_pid -} - -t_begin "check stderr" && check_stderr - -dbgcat r_err - -t_done diff --git a/t/t0011-active-unix-socket.sh b/t/t0011-active-unix-socket.sh deleted file mode 100755 index fae0b6c..0000000 --- a/t/t0011-active-unix-socket.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/sh -. ./test-lib.sh -t_plan 11 "existing UNIX domain socket check" - -read_pid_unix () { - x=$(printf 'GET / HTTP/1.0\r\n\r\n' | \ - socat - UNIX:$unix_socket | \ - tail -1) - test -n "$x" - y="$(expr "$x" : '\([0-9][0-9]*\)')" - test x"$x" = x"$y" - test -n "$y" - echo "$y" -} - -t_begin "setup and start" && { - rtmpfiles unix_socket unix_config - rm -f $unix_socket - unicorn_setup - grep -v ^listen < $unicorn_config > $unix_config - echo "listen '$unix_socket'" >> $unix_config - unicorn -D -c $unix_config pid.ru - unicorn_wait_start - orig_master_pid=$unicorn_pid -} - -t_begin "get pid of worker" && { - worker_pid=$(read_pid_unix) - t_info "worker_pid=$worker_pid" -} - -t_begin "fails to start with existing pid file" && { - rm -f $ok - unicorn -D -c $unix_config pid.ru || echo ok > $ok - test x"$(cat $ok)" = xok -} - -t_begin "worker pid unchanged" && { - test x"$(read_pid_unix)" = x$worker_pid - > $r_err -} - -t_begin "fails to start with listening UNIX domain socket bound" && { - rm $ok $pid - unicorn -D -c $unix_config pid.ru || echo ok > $ok - test x"$(cat $ok)" = xok - > $r_err -} - -t_begin "worker pid unchanged (again)" && { - test x"$(read_pid_unix)" = x$worker_pid -} - -t_begin "nuking the existing Unicorn succeeds" && { - kill -9 $unicorn_pid - while kill -0 $unicorn_pid - do - sleep 1 - done - check_stderr -} - -t_begin "succeeds in starting with leftover UNIX domain socket bound" && { - test -S $unix_socket - unicorn -D -c $unix_config pid.ru - unicorn_wait_start -} - -t_begin "worker pid changed" && { - test x"$(read_pid_unix)" != x$worker_pid -} - -t_begin "killing succeeds" && { - kill $unicorn_pid -} - -t_begin "no errors" && check_stderr - -t_done diff --git a/t/t0018-write-on-close.sh b/t/t0018-write-on-close.sh deleted file mode 100755 index 3afefea..0000000 --- a/t/t0018-write-on-close.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh -. ./test-lib.sh -t_plan 4 "write-on-close tests for funky response-bodies" - -t_begin "setup and start" && { - unicorn_setup - unicorn -D -c $unicorn_config write-on-close.ru - unicorn_wait_start -} - -t_begin "write-on-close response body succeeds" && { - test xGoodbye = x"$(curl -sSf http://$listen/)" -} - -t_begin "killing succeeds" && { - kill $unicorn_pid -} - -t_begin "check stderr" && { - check_stderr -} - -t_done diff --git a/t/t0019-max_header_len.sh b/t/t0019-max_header_len.sh deleted file mode 100755 index 6a355b4..0000000 --- a/t/t0019-max_header_len.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/sh -. ./test-lib.sh -t_plan 5 "max_header_len setting (only intended for Rainbows!)" - -t_begin "setup and start" && { - unicorn_setup - req='GET / HTTP/1.0\r\n\r\n' - len=$(printf "$req" | count_bytes) - echo Unicorn::HttpParser.max_header_len = $len >> $unicorn_config - unicorn -D -c $unicorn_config env.ru - unicorn_wait_start -} - -t_begin "minimal request succeeds" && { - rm -f $tmp - ( - cat $fifo > $tmp & - printf "$req" - wait - echo ok > $ok - ) | socat - TCP:$listen > $fifo - test xok = x$(cat $ok) - - fgrep "HTTP/1.1 200 OK" $tmp -} - -t_begin "big request fails" && { - rm -f $tmp - ( - cat $fifo > $tmp & - printf 'GET /xxxxxx HTTP/1.0\r\n\r\n' - wait - echo ok > $ok - ) | socat - TCP:$listen > $fifo - test xok = x$(cat $ok) - fgrep "HTTP/1.1 413" $tmp -} - -dbgcat tmp - -t_begin "killing succeeds" && { - kill $unicorn_pid -} - -t_begin "check stderr" && { - check_stderr -} - -t_done diff --git a/t/t0100-rack-input-tests.sh b/t/t0100-rack-input-tests.sh deleted file mode 100755 index ee7a437..0000000 --- a/t/t0100-rack-input-tests.sh +++ /dev/null @@ -1,124 +0,0 @@ -#!/bin/sh -. ./test-lib.sh -test -r random_blob || die "random_blob required, run with 'make $0'" - -t_plan 10 "rack.input read tests" - -t_begin "setup and startup" && { - rtmpfiles curl_out curl_err - unicorn_setup - unicorn -E none -D rack-input-tests.ru -c $unicorn_config - blob_sha1=$(rsha1 < random_blob) - blob_size=$(count_bytes < random_blob) - t_info "blob_sha1=$blob_sha1" - unicorn_wait_start -} - -t_begin "corked identity request" && { - rm -f $tmp - ( - cat $fifo > $tmp & - printf 'PUT / HTTP/1.0\r\n' - printf 'Content-Length: %d\r\n\r\n' $blob_size - cat random_blob - wait - echo ok > $ok - ) | ( sleep 1 && socat - TCP4:$listen > $fifo ) - test 1 -eq $(grep $blob_sha1 $tmp |count_lines) - test x"$(cat $ok)" = xok -} - -t_begin "corked chunked request" && { - rm -f $tmp - ( - cat $fifo > $tmp & - content-md5-put < random_blob - wait - echo ok > $ok - ) | ( sleep 1 && socat - TCP4:$listen > $fifo ) - test 1 -eq $(grep $blob_sha1 $tmp |count_lines) - test x"$(cat $ok)" = xok -} - -t_begin "corked identity request (input#size first)" && { - rm -f $tmp - ( - cat $fifo > $tmp & - printf 'PUT /size_first HTTP/1.0\r\n' - printf 'Content-Length: %d\r\n\r\n' $blob_size - cat random_blob - wait - echo ok > $ok - ) | ( sleep 1 && socat - TCP4:$listen > $fifo ) - test 1 -eq $(grep $blob_sha1 $tmp |count_lines) - test x"$(cat $ok)" = xok -} - -t_begin "corked identity request (input#rewind first)" && { - rm -f $tmp - ( - cat $fifo > $tmp & - printf 'PUT /rewind_first HTTP/1.0\r\n' - printf 'Content-Length: %d\r\n\r\n' $blob_size - cat random_blob - wait - echo ok > $ok - ) | ( sleep 1 && socat - TCP4:$listen > $fifo ) - test 1 -eq $(grep $blob_sha1 $tmp |count_lines) - test x"$(cat $ok)" = xok -} - -t_begin "corked chunked request (input#size first)" && { - rm -f $tmp - ( - cat $fifo > $tmp & - printf 'PUT /size_first HTTP/1.1\r\n' - printf 'Host: example.com\r\n' - printf 'Transfer-Encoding: chunked\r\n' - printf 'Trailer: Content-MD5\r\n' - printf '\r\n' - content-md5-put --no-headers < random_blob - wait - echo ok > $ok - ) | ( sleep 1 && socat - TCP4:$listen > $fifo ) - test 1 -eq $(grep $blob_sha1 $tmp |count_lines) - test 1 -eq $(grep $blob_sha1 $tmp |count_lines) - test x"$(cat $ok)" = xok -} - -t_begin "corked chunked request (input#rewind first)" && { - rm -f $tmp - ( - cat $fifo > $tmp & - printf 'PUT /rewind_first HTTP/1.1\r\n' - printf 'Host: example.com\r\n' - printf 'Transfer-Encoding: chunked\r\n' - printf 'Trailer: Content-MD5\r\n' - printf '\r\n' - content-md5-put --no-headers < random_blob - wait - echo ok > $ok - ) | ( sleep 1 && socat - TCP4:$listen > $fifo ) - test 1 -eq $(grep $blob_sha1 $tmp |count_lines) - test x"$(cat $ok)" = xok -} - -t_begin "regular request" && { - curl -sSf -T random_blob http://$listen/ > $curl_out 2> $curl_err - test x$blob_sha1 = x$(cat $curl_out) - test ! -s $curl_err -} - -t_begin "chunked request" && { - curl -sSf -T- < random_blob http://$listen/ > $curl_out 2> $curl_err - test x$blob_sha1 = x$(cat $curl_out) - test ! -s $curl_err -} - -dbgcat r_err - -t_begin "shutdown" && { - kill $unicorn_pid -} - -t_done diff --git a/t/t0116-client_body_buffer_size.sh b/t/t0116-client_body_buffer_size.sh deleted file mode 100755 index c9e17c7..0000000 --- a/t/t0116-client_body_buffer_size.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/sh -. ./test-lib.sh -t_plan 12 "client_body_buffer_size settings" - -t_begin "setup and start" && { - unicorn_setup - rtmpfiles unicorn_config_tmp one_meg - dd if=/dev/zero bs=1M count=1 of=$one_meg - cat >> $unicorn_config <<EOF -after_fork do |server, worker| - File.open("$fifo", "wb") { |fp| fp.syswrite "START" } -end -EOF - cat $unicorn_config > $unicorn_config_tmp - echo client_body_buffer_size 0 >> $unicorn_config - unicorn -D -c $unicorn_config t0116.ru - unicorn_wait_start - fs_class=Unicorn::TmpIO - mem_class=StringIO - - test x"$(cat $fifo)" = xSTART -} - -t_begin "class for a zero-byte file should be StringIO" && { - > $tmp - test xStringIO = x"$(curl -T $tmp -sSf http://$listen/input_class)" -} - -t_begin "class for a 1 byte file should be filesystem-backed" && { - echo > $tmp - test x$fs_class = x"$(curl -T $tmp -sSf http://$listen/tmp_class)" -} - -t_begin "reload with default client_body_buffer_size" && { - mv $unicorn_config_tmp $unicorn_config - kill -HUP $unicorn_pid - test x"$(cat $fifo)" = xSTART -} - -t_begin "class for a 1 byte file should be memory-backed" && { - echo > $tmp - test x$mem_class = x"$(curl -T $tmp -sSf http://$listen/tmp_class)" -} - -t_begin "class for a random blob file should be filesystem-backed" && { - resp="$(curl -T random_blob -sSf http://$listen/tmp_class)" - test x$fs_class = x"$resp" -} - -t_begin "one megabyte file should be filesystem-backed" && { - resp="$(curl -T $one_meg -sSf http://$listen/tmp_class)" - test x$fs_class = x"$resp" -} - -t_begin "reload with a big client_body_buffer_size" && { - echo "client_body_buffer_size(1024 * 1024)" >> $unicorn_config - kill -HUP $unicorn_pid - test x"$(cat $fifo)" = xSTART -} - -t_begin "one megabyte file should be memory-backed" && { - resp="$(curl -T $one_meg -sSf http://$listen/tmp_class)" - test x$mem_class = x"$resp" -} - -t_begin "one megabyte + 1 byte file should be filesystem-backed" && { - echo >> $one_meg - resp="$(curl -T $one_meg -sSf http://$listen/tmp_class)" - test x$fs_class = x"$resp" -} - -t_begin "killing succeeds" && { - kill $unicorn_pid -} - -t_begin "check stderr" && { - check_stderr -} - -t_done diff --git a/t/t0200-rack-hijack.sh b/t/t0200-rack-hijack.sh deleted file mode 100755 index fee0791..0000000 --- a/t/t0200-rack-hijack.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/sh -. ./test-lib.sh -t_plan 9 "rack.hijack tests (Rack 1.5+ (Rack::VERSION >= [ 1,2]))" - -t_begin "setup and start" && { - unicorn_setup - unicorn -D -c $unicorn_config hijack.ru - unicorn_wait_start -} - -t_begin "normal env reused between requests" && { - env_a="$(curl -sSf http://$listen/normal_env_id)" - b="$(curl -sSf http://$listen/normal_env_id)" - test x"$env_a" = x"$b" -} - -t_begin "check request hijack" && { - test "xrequest.hijacked" = x"$(curl -sSfv http://$listen/hijack_req)" -} - -t_begin "env changed after request hijack" && { - env_b="$(curl -sSf http://$listen/normal_env_id)" - test x"$env_a" != x"$env_b" -} - -t_begin "check response hijack" && { - test "xresponse.hijacked" = x"$(curl -sSfv http://$listen/hijack_res)" -} - -t_begin "env changed after response hijack" && { - env_c="$(curl -sSf http://$listen/normal_env_id)" - test x"$env_b" != x"$env_c" -} - -t_begin "env continues to be reused between requests" && { - b="$(curl -sSf http://$listen/normal_env_id)" - test x"$env_c" = x"$b" -} - -t_begin "killing succeeds after hijack" && { - kill $unicorn_pid -} - -t_begin "check stderr for hijacked body close" && { - check_stderr - grep 'closed DieIfUsed 1\>' $r_err - grep 'closed DieIfUsed 2\>' $r_err - ! grep 'closed DieIfUsed 3\>' $r_err -} - -t_done diff --git a/t/t0300-no-default-middleware.sh b/t/t0300-no-default-middleware.sh index 779dc02..00feacc 100644 --- a/t/t0300-no-default-middleware.sh +++ b/t/t0300-no-default-middleware.sh @@ -9,7 +9,7 @@ t_begin "setup and start" && { } t_begin "check exit status with Rack::Lint not present" && { - test 42 -eq "$(curl -sf -o/dev/null -w'%{http_code}' http://$listen/)" + test 500 -ne "$(curl -sf -o/dev/null -w'%{http_code}' http://$listen/)" } t_begin "killing succeeds" && { @@ -6,8 +6,8 @@ run(lambda do |env| "lint=#{caller.grep(%r{rack/lint\.rb})[0].split(':')[0]}\n" end h = { - 'Content-Length' => b.size.to_s, - 'Content-Type' => 'text/plain', + 'content-length' => b.size.to_s, + 'content-type' => 'text/plain', } [ 200, h, [ b ] ] end) diff --git a/t/t9000-preread-input.sh b/t/t9000-preread-input.sh deleted file mode 100755 index d6c73ab..0000000 --- a/t/t9000-preread-input.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/sh -. ./test-lib.sh -t_plan 9 "PrereadInput middleware tests" - -t_begin "setup and start" && { - random_blob_sha1=$(rsha1 < random_blob) - unicorn_setup - unicorn -D -c $unicorn_config preread_input.ru - unicorn_wait_start -} - -t_begin "single identity request" && { - curl -sSf -T random_blob http://$listen/ > $tmp -} - -t_begin "sha1 matches" && { - test x"$(cat $tmp)" = x"$random_blob_sha1" -} - -t_begin "single chunked request" && { - curl -sSf -T- < random_blob http://$listen/ > $tmp -} - -t_begin "sha1 matches" && { - test x"$(cat $tmp)" = x"$random_blob_sha1" -} - -t_begin "app only dispatched twice" && { - test 2 -eq "$(grep 'app dispatch:' < $r_err | count_lines )" -} - -t_begin "aborted chunked request" && { - rm -f $tmp - curl -sSf -T- < $fifo http://$listen/ > $tmp & - curl_pid=$! - kill -9 $curl_pid - wait -} - -t_begin "app only dispatched twice" && { - test 2 -eq "$(grep 'app dispatch:' < $r_err | count_lines )" -} - -t_begin "killing succeeds" && { - kill -QUIT $unicorn_pid -} - -t_done diff --git a/t/test-lib.sh b/t/test-lib.sh index 7f97958..8613144 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -94,7 +94,8 @@ check_stderr () { set +u _r_err=${1-${r_err}} set -u - if grep -v $T $_r_err | grep -i Error + if grep -v $T $_r_err | grep -i Error | \ + grep -v NameError.*Unicorn::Waiter then die "Errors found in $_r_err" elif grep SIGKILL $_r_err @@ -122,7 +123,3 @@ unicorn_wait_start () { # no need to play tricks with FIFOs since we got "ready_pipe" now unicorn_pid=$(cat $pid) } - -rsha1 () { - sha1sum.rb -} diff --git a/t/winch_ttin.t b/t/winch_ttin.t new file mode 100644 index 0000000..c507959 --- /dev/null +++ b/t/winch_ttin.t @@ -0,0 +1,67 @@ +#!perl -w +# Copyright (C) unicorn hackers <unicorn-public@yhbt.net> +# License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> +use v5.14; BEGIN { require './t/lib.perl' }; +use autodie; +use POSIX qw(mkfifo); +my $u_conf = "$tmpdir/u.conf.rb"; +my $u_sock = "$tmpdir/u.sock"; +my $fifo = "$tmpdir/fifo"; +mkfifo($fifo, 0666) or die "mkfifo($fifo): $!"; + +open my $fh, '>', $u_conf; +print $fh <<EOM; +pid "$tmpdir/pid" +listen "$u_sock" +stderr_path "$err_log" +after_fork do |server, worker| + # test script will block while reading from $fifo, + File.open("$fifo", "wb") { |fp| fp.syswrite worker.nr.to_s } +end +EOM +close $fh; + +unicorn('-D', '-c', $u_conf, 't/integration.ru')->join; +is($?, 0, 'daemonized properly'); +open $fh, '<', "$tmpdir/pid"; +chomp(my $pid = <$fh>); +ok(kill(0, $pid), 'daemonized PID works'); +my $quit = sub { kill('QUIT', $pid) if $pid; $pid = undef }; +END { $quit->() }; + +open $fh, '<', $fifo; +my $worker_nr = <$fh>; +close $fh; +is($worker_nr, '0', 'initial worker spawned'); + +my ($status, $hdr, $worker_pid) = do_req($u_sock, 'GET /pid HTTP/1.0'); +like($status, qr/ 200\b/, 'got 200 response'); +like($worker_pid, qr/\A[0-9]+\n\z/s, 'PID in response'); +chomp $worker_pid; +ok(kill(0, $worker_pid), 'worker_pid is valid'); + +ok(kill('WINCH', $pid), 'SIGWINCH can be sent'); + +my $tries = 1000; +while (CORE::kill(0, $worker_pid) && --$tries) { sleep 0.01 } +ok(!CORE::kill(0, $worker_pid), 'worker not running'); + +ok(kill('TTIN', $pid), 'SIGTTIN to restart worker'); + +open $fh, '<', $fifo; +$worker_nr = <$fh>; +close $fh; +is($worker_nr, '0', 'worker restarted'); + +($status, $hdr, my $new_worker_pid) = do_req($u_sock, 'GET /pid HTTP/1.0'); +like($status, qr/ 200\b/, 'got 200 response'); +like($new_worker_pid, qr/\A[0-9]+\n\z/, 'got new worker PID'); +chomp $new_worker_pid; +ok(kill(0, $new_worker_pid), 'got a valid worker PID'); +isnt($worker_pid, $new_worker_pid, 'worker PID changed'); + +$quit->(); + +check_stderr; +undef $tmpdir; +done_testing; diff --git a/t/working_directory.t b/t/working_directory.t new file mode 100644 index 0000000..f9254eb --- /dev/null +++ b/t/working_directory.t @@ -0,0 +1,94 @@ +#!perl -w +# Copyright (C) unicorn hackers <unicorn-public@yhbt.net> +# License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> +use v5.14; BEGIN { require './t/lib.perl' }; +use autodie; +mkdir "$tmpdir/alt"; +my $ru = "$tmpdir/alt/config.ru"; +open my $fh, '>', $u_conf; +print $fh <<EOM; +pid "$pid_file" +preload_app true +stderr_path "$err_log" +working_directory "$tmpdir/alt" # the whole point of this test +before_fork { |_,_| \$master_ppid = Process.ppid } +EOM +close $fh; + +my $common_ru = <<'EOM'; +use Rack::ContentLength +use Rack::ContentType, 'text/plain' +run lambda { |env| [ 200, {}, [ "#{$master_ppid}\n" ] ] } +EOM + +open $fh, '>', $ru; +print $fh <<EOM; +#\\--daemonize --listen $u_sock +$common_ru +EOM +close $fh; + +unicorn('-c', $u_conf)->join; # will daemonize +chomp($daemon_pid = slurp($pid_file)); + +my ($status, $hdr, $bdy) = do_req($u_sock, 'GET / HTTP/1.0'); +is($bdy, "1\n", 'got expected $master_ppid'); + +stop_daemon; +check_stderr; + +if ('test without CLI switches in config.ru') { + truncate $err_log, 0; + open $fh, '>', $ru; + print $fh $common_ru; + close $fh; + + unicorn('-D', '-l', $u_sock, '-c', $u_conf)->join; # will daemonize + chomp($daemon_pid = slurp($pid_file)); + + ($status, $hdr, $bdy) = do_req($u_sock, 'GET / HTTP/1.0'); + is($bdy, "1\n", 'got expected $master_ppid'); + + stop_daemon; + check_stderr; +} + +if ('ensures broken working_directory (missing config.ru) is OK') { + truncate $err_log, 0; + unlink $ru; + + my $auto_reap = unicorn('-c', $u_conf); + $auto_reap->join; + isnt($?, 0, 'exited with error due to missing config.ru'); + + like(slurp($err_log), qr/rackup file \Q(config.ru)\E not readable/, + 'noted unreadability of config.ru in stderr'); +} + +if ('fooapp.rb (not config.ru) works with working_directory') { + truncate $err_log, 0; + my $fooapp = "$tmpdir/alt/fooapp.rb"; + open $fh, '>', $fooapp; + print $fh <<EOM; +class Fooapp + def self.call(env) + b = "dir=#{Dir.pwd}" + h = { 'content-type' => 'text/plain', 'content-length' => b.bytesize.to_s } + [ 200, h, [ b ] ] + end +end +EOM + close $fh; + my $srv = tcp_server; + my $auto_reap = unicorn(qw(-c), $u_conf, qw(-I. fooapp.rb), + { -C => '/', 3 => $srv }); + ($status, $hdr, $bdy) = do_req($srv, 'GET / HTTP/1.0'); + is($bdy, "dir=$tmpdir/alt", + 'fooapp.rb (w/o config.ru) w/ working_directory'); + $auto_reap->join('TERM'); + is($?, 0, 'fooapp.rb process exited'); + check_stderr; +} + +undef $tmpdir; +done_testing; diff --git a/t/write-on-close.ru b/t/write-on-close.ru deleted file mode 100644 index 54a2f2e..0000000 --- a/t/write-on-close.ru +++ /dev/null @@ -1,11 +0,0 @@ -class WriteOnClose - def each(&block) - @callback = block - end - - def close - @callback.call "7\r\nGoodbye\r\n0\r\n\r\n" - end -end -use Rack::ContentType, "text/plain" -run(lambda { |_| [ 200, [%w(Transfer-Encoding chunked)], WriteOnClose.new ] }) diff --git a/test/benchmark/README b/test/benchmark/README index 1d3cdd0..cd929f3 100644 --- a/test/benchmark/README +++ b/test/benchmark/README @@ -42,9 +42,19 @@ The benchmark client is usually httperf. Another gentle reminder: performance with slow networks/clients is NOT our problem. That is the job of nginx (or similar). +== ddstream.ru + +Standalone Rack app intended to show how BAD we are at slow clients. +See usage in comments. + +== readinput.ru + +Standalone Rack app intended to show how bad we are with slow uploaders. +See usage in comments. + == Contributors -This directory is maintained independently in the "benchmark" branch -based against v0.1.0. Only changes to this directory (test/benchmarks) -are committed to this branch although the master branch may merge this -branch occassionaly. +This directory is intended to remain stable. Do not make changes +to benchmarking code which can change performance and invalidate +results across revisions. Instead, write new benchmarks and update +coments/documentation as necessary. diff --git a/test/benchmark/ddstream.ru b/test/benchmark/ddstream.ru new file mode 100644 index 0000000..b14c973 --- /dev/null +++ b/test/benchmark/ddstream.ru @@ -0,0 +1,50 @@ +# This app is intended to test large HTTP responses with or without +# a fully-buffering reverse proxy such as nginx. Without a fully-buffering +# reverse proxy, unicorn will be unresponsive when client count exceeds +# worker_processes. +# +# To demonstrate how bad unicorn is at slowly reading clients: +# +# # in one terminal, start unicorn with one worker: +# unicorn -E none -l 127.0.0.1:8080 test/benchmark/ddstream.ru +# +# # in a different terminal, start more slow curl processes than +# # unicorn workers and watch time outputs +# curl --limit-rate 8K --trace-time -vsN http://127.0.0.1:8080/ >/dev/null & +# curl --limit-rate 8K --trace-time -vsN http://127.0.0.1:8080/ >/dev/null & +# wait +# +# The last client won't see a response until the first one is done reading +# +# nginx note: do not change the default "proxy_buffering" behavior. +# Setting "proxy_buffering off" prevents nginx from protecting unicorn. + +# totally standalone rack app to stream a giant response +class BigResponse + def initialize(bs, count) + @buf = "#{bs.to_s(16)}\r\n#{' ' * bs}\r\n" + @count = count + @res = [ 200, + { 'Transfer-Encoding' => -'chunked', 'Content-Type' => 'text/plain' }, + self + ] + end + + # rack response body iterator + def each + (1..@count).each { yield @buf } + yield -"0\r\n\r\n" + end + + # rack app entry endpoint + def call(_env) + @res + end +end + +# default to a giant (128M) response because kernel socket buffers +# can be ridiculously large on some systems +bs = ENV['bs'] ? ENV['bs'].to_i : 65536 +count = ENV['count'] ? ENV['count'].to_i : 2048 +warn "serving response with bs=#{bs} count=#{count} (#{bs*count} bytes)" +run BigResponse.new(bs, count) diff --git a/test/benchmark/readinput.ru b/test/benchmark/readinput.ru new file mode 100644 index 0000000..c91bec3 --- /dev/null +++ b/test/benchmark/readinput.ru @@ -0,0 +1,40 @@ +# This app is intended to test large HTTP requests with or without +# a fully-buffering reverse proxy such as nginx. Without a fully-buffering +# reverse proxy, unicorn will be unresponsive when client count exceeds +# worker_processes. + +DOC = <<DOC +To demonstrate how bad unicorn is at slowly uploading clients: + + # in one terminal, start unicorn with one worker: + unicorn -E none -l 127.0.0.1:8080 test/benchmark/readinput.ru + + # in a different terminal, upload 45M from multiple curl processes: + dd if=/dev/zero bs=45M count=1 | curl -T- -HExpect: --limit-rate 1M \ + --trace-time -v http://127.0.0.1:8080/ & + dd if=/dev/zero bs=45M count=1 | curl -T- -HExpect: --limit-rate 1M \ + --trace-time -v http://127.0.0.1:8080/ & + wait + +# The last client won't see a response until the first one is done uploading +# You also won't be able to make GET requests to view this documentation +# while clients are uploading. You can also view the stderr debug output +# of unicorn (see logging code in #{__FILE__}). +DOC + +run(lambda do |env| + input = env['rack.input'] + buf = ''.b + + # default logger contains timestamps, rely on that so users can + # see what the server is doing + l = env['rack.logger'] + + l.debug('BEGIN reading input ...') if l + :nop while input.read(16384, buf) + l.debug('DONE reading input ...') if l + + buf.clear + [ 200, [ %W(Content-Length #{DOC.size}), %w(Content-Type text/plain) ], + [ DOC ] ] +end) diff --git a/test/benchmark/uconnect.perl b/test/benchmark/uconnect.perl new file mode 100755 index 0000000..230445e --- /dev/null +++ b/test/benchmark/uconnect.perl @@ -0,0 +1,66 @@ +#!/usr/bin/perl -w +# Benchmark script to spawn some processes and hammer a local unicorn +# to test accept loop performance. This only does Unix sockets. +# There's plenty of TCP benchmarking tools out there, and TCP port reuse +# has predictability problems since unicorn can't do persistent connections. +# Written in Perl for the same reason: predictability. +# Ruby GC is not as predictable as Perl refcounting. +use strict; +use Socket qw(AF_UNIX SOCK_STREAM sockaddr_un); +use POSIX qw(:sys_wait_h); +use Getopt::Std; +# -c / -n switches stolen from ab(1) +my $usage = "$0 [-c CONCURRENCY] [-n NUM_REQUESTS] SOCKET_PATH\n"; +our $opt_c = 2; +our $opt_n = 1000; +getopts('c:n:') or die $usage; +my $unix_path = shift or die $usage; +use constant REQ => "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n"; +use constant REQ_LEN => length(REQ); +use constant BUFSIZ => 8192; +$^F = 99; # don't waste syscall time with FD_CLOEXEC + +my %workers; # pid => worker num +die "-n $opt_n not evenly divisible by -c $opt_c\n" if $opt_n % $opt_c; +my $n_per_worker = $opt_n / $opt_c; +my $addr = sockaddr_un($unix_path); + +for my $num (1..$opt_c) { + defined(my $pid = fork) or die "fork failed: $!\n"; + if ($pid) { + $workers{$pid} = $num; + } else { + work($n_per_worker); + } +} + +reap_worker(0) while scalar keys %workers; +exit; + +sub work { + my ($n) = @_; + my ($buf, $x); + for (1..$n) { + socket(S, AF_UNIX, SOCK_STREAM, 0) or die "socket: $!"; + connect(S, $addr) or die "connect: $!"; + defined($x = syswrite(S, REQ)) or die "write: $!"; + $x == REQ_LEN or die "short write: $x != ".REQ_LEN."\n"; + do { + $x = sysread(S, $buf, BUFSIZ); + unless (defined $x) { + next if $!{EINTR}; + die "sysread: $!\n"; + } + } until ($x == 0); + } + exit 0; +} + +sub reap_worker { + my ($flags) = @_; + my $pid = waitpid(-1, $flags); + return if !defined $pid || $pid <= 0; + my $p = delete $workers{$pid} || '(unknown)'; + warn("$pid [$p] exited with $?\n") if $?; + $p; +} diff --git a/test/exec/test_exec.rb b/test/exec/test_exec.rb index 8a6b43e..8494452 100644 --- a/test/exec/test_exec.rb +++ b/test/exec/test_exec.rb @@ -1,6 +1,5 @@ # -*- encoding: binary -*- - -# Copyright (c) 2009 Eric Wong +# Don't add to this file, new tests are in Perl 5. See t/README FLOCK_PATH = File.expand_path(__FILE__) require './test/test_helper' @@ -25,28 +24,29 @@ class ExecTest < Test::Unit::TestCase HI = <<-EOS use Rack::ContentLength -run proc { |env| [ 200, { 'Content-Type' => 'text/plain' }, [ "HI\\n" ] ] } +run proc { |env| [ 200, { 'content-type' => 'text/plain' }, [ "HI\\n" ] ] } EOS SHOW_RACK_ENV = <<-EOS use Rack::ContentLength run proc { |env| - [ 200, { 'Content-Type' => 'text/plain' }, [ ENV['RACK_ENV'] ] ] + [ 200, { 'content-type' => 'text/plain' }, [ ENV['RACK_ENV'] ] ] } EOS HELLO = <<-EOS class Hello def call(env) - [ 200, { 'Content-Type' => 'text/plain' }, [ "HI\\n" ] ] + [ 200, { 'content-type' => 'text/plain' }, [ "HI\\n" ] ] end end EOS COMMON_TMP = Tempfile.new('unicorn_tmp') unless defined?(COMMON_TMP) + HEAVY_WORKERS = 2 HEAVY_CFG = <<-EOS -worker_processes 4 +worker_processes #{HEAVY_WORKERS} timeout 30 logger Logger.new('#{COMMON_TMP.path}') before_fork do |server, worker| @@ -61,9 +61,9 @@ run lambda { |env| a = ::File.stat(pwd) b = ::File.stat(Dir.pwd) if (a.ino == b.ino && a.dev == b.dev) - [ 200, { 'Content-Type' => 'text/plain' }, [ pwd ] ] + [ 200, { 'content-type' => 'text/plain' }, [ pwd ] ] else - [ 404, { 'Content-Type' => 'text/plain' }, [] ] + [ 404, { 'content-type' => 'text/plain' }, [] ] end } EOS @@ -96,59 +96,6 @@ run lambda { |env| end end - def test_sd_listen_fds_emulation - # [ruby-core:69895] [Bug #11336] fixed by r51576 - return if RUBY_VERSION.to_f < 2.3 - - File.open("config.ru", "wb") { |fp| fp.write(HI) } - sock = TCPServer.new(@addr, @port) - - [ %W(-l #@addr:#@port), nil ].each do |l| - sock.setsockopt(:SOL_SOCKET, :SO_KEEPALIVE, 0) - - pid = xfork do - redirect_test_io do - # pretend to be systemd - ENV['LISTEN_PID'] = "#$$" - ENV['LISTEN_FDS'] = '1' - - # 3 = SD_LISTEN_FDS_START - args = [ $unicorn_bin ] - args.concat(l) if l - args << { 3 => sock } - exec(*args) - end - end - res = hit(["http://#@addr:#@port/"]) - assert_equal [ "HI\n" ], res - assert_shutdown(pid) - assert sock.getsockopt(:SOL_SOCKET, :SO_KEEPALIVE).bool, - 'unicorn should always set SO_KEEPALIVE on inherited sockets' - end - ensure - sock.close if sock - end - - def test_inherit_listener_unspecified - File.open("config.ru", "wb") { |fp| fp.write(HI) } - sock = TCPServer.new(@addr, @port) - sock.setsockopt(:SOL_SOCKET, :SO_KEEPALIVE, 0) - - pid = xfork do - redirect_test_io do - ENV['UNICORN_FD'] = sock.fileno.to_s - exec($unicorn_bin, sock.fileno => sock.fileno) - end - end - res = hit(["http://#@addr:#@port/"]) - assert_equal [ "HI\n" ], res - assert_shutdown(pid) - assert sock.getsockopt(:SOL_SOCKET, :SO_KEEPALIVE).bool, - 'unicorn should always set SO_KEEPALIVE on inherited sockets' - ensure - sock.close if sock - end - def test_working_directory_rel_path_config_file other = Tempfile.new('unicorn.wd') File.unlink(other.path) @@ -254,7 +201,13 @@ after_fork do |server, worker| end EOF pid = xfork { redirect_test_io { exec($unicorn_bin, "-c#{tmp.path}") } } - File.open("#{other.path}/fifo", "rb").close + begin + fifo = File.open("#{other.path}/fifo", "rb") + rescue Errno::EINTR + # OpenBSD raises Errno::EINTR when opening + return if RUBY_PLATFORM =~ /openbsd/ + end + fifo.close assert ! File.exist?("stderr_log_here") assert ! File.exist?("stdout_log_here") @@ -291,16 +244,6 @@ EOF end end - def test_basic - File.open("config.ru", "wb") { |fp| fp.syswrite(HI) } - pid = fork do - redirect_test_io { exec($unicorn_bin, "-l", "#{@addr}:#{@port}") } - end - results = retry_hit(["http://#{@addr}:#{@port}/"]) - assert_equal String, results[0].class - assert_shutdown(pid) - end - def test_rack_env_unset File.open("config.ru", "wb") { |fp| fp.syswrite(SHOW_RACK_ENV) } pid = fork { redirect_test_io { exec($unicorn_bin, "-l#@addr:#@port") } } @@ -556,7 +499,7 @@ EOF def test_unicorn_config_per_worker_listen port2 = unused_port pid_spit = 'use Rack::ContentLength;' \ - 'run proc { |e| [ 200, {"Content-Type"=>"text/plain"}, ["#$$\\n"] ] }' + 'run proc { |e| [ 200, {"content-type"=>"text/plain"}, ["#$$\\n"] ] }' File.open("config.ru", "wb") { |fp| fp.syswrite(pid_spit) } tmp = Tempfile.new('test.socket') File.unlink(tmp.path) @@ -573,7 +516,7 @@ EOF assert_equal String, results[0].class worker_pid = results[0].to_i assert_not_equal pid, worker_pid - s = UNIXSocket.new(tmp.path) + s = unix_socket(tmp.path) s.syswrite("GET / HTTP/1.0\r\n\r\n") results = '' loop { results << s.sysread(4096) } rescue nil @@ -606,6 +549,7 @@ EOF def test_weird_config_settings File.open("config.ru", "wb") { |fp| fp.syswrite(HI) } ucfg = Tempfile.new('unicorn_test_config') + proc_total = HEAVY_WORKERS + 1 # + 1 for master ucfg.syswrite(HEAVY_CFG) pid = xfork do redirect_test_io do @@ -616,9 +560,9 @@ EOF results = retry_hit(["http://#{@addr}:#{@port}/"]) assert_equal String, results[0].class wait_master_ready(COMMON_TMP.path) - wait_workers_ready(COMMON_TMP.path, 4) + wait_workers_ready(COMMON_TMP.path, HEAVY_WORKERS) bf = File.readlines(COMMON_TMP.path).grep(/\bbefore_fork: worker=/) - assert_equal 4, bf.size + assert_equal HEAVY_WORKERS, bf.size rotate = Tempfile.new('unicorn_rotate') File.rename(COMMON_TMP.path, rotate.path) @@ -630,20 +574,20 @@ EOF tries = DEFAULT_TRIES log = File.readlines(rotate.path) while (tries -= 1) > 0 && - log.grep(/reopening logs\.\.\./).size < 5 + log.grep(/reopening logs\.\.\./).size < proc_total sleep DEFAULT_RES log = File.readlines(rotate.path) end - assert_equal 5, log.grep(/reopening logs\.\.\./).size + assert_equal proc_total, log.grep(/reopening logs\.\.\./).size assert_equal 0, log.grep(/done reopening logs/).size tries = DEFAULT_TRIES log = File.readlines(COMMON_TMP.path) - while (tries -= 1) > 0 && log.grep(/done reopening logs/).size < 5 + while (tries -= 1) > 0 && log.grep(/done reopening logs/).size < proc_total sleep DEFAULT_RES log = File.readlines(COMMON_TMP.path) end - assert_equal 5, log.grep(/done reopening logs/).size + assert_equal proc_total, log.grep(/done reopening logs/).size assert_equal 0, log.grep(/reopening logs\.\.\./).size Process.kill(:QUIT, pid) @@ -663,20 +607,6 @@ EOF assert_shutdown(pid) end - def test_config_ru_alt_path - config_path = "#{@tmpdir}/foo.ru" - File.open(config_path, "wb") { |fp| fp.syswrite(HI) } - pid = fork do - redirect_test_io do - Dir.chdir("/") - exec($unicorn_bin, "-l#{@addr}:#{@port}", config_path) - end - end - results = retry_hit(["http://#{@addr}:#{@port}/"]) - assert_equal String, results[0].class - assert_shutdown(pid) - end - def test_load_module libdir = "#{@tmpdir}/lib" FileUtils.mkpath([ libdir ]) @@ -730,7 +660,7 @@ EOF wait_for_file(sock_path) assert File.socket?(sock_path) - sock = UNIXSocket.new(sock_path) + sock = unix_socket(sock_path) sock.syswrite("GET / HTTP/1.0\r\n\r\n") results = sock.sysread(4096) @@ -740,7 +670,7 @@ EOF wait_for_file(sock_path) assert File.socket?(sock_path) - sock = UNIXSocket.new(sock_path) + sock = unix_socket(sock_path) sock.syswrite("GET / HTTP/1.0\r\n\r\n") results = sock.sysread(4096) @@ -775,7 +705,7 @@ EOF assert_equal pid, File.read(pid_file).to_i assert File.socket?(sock_path), "socket created" - sock = UNIXSocket.new(sock_path) + sock = unix_socket(sock_path) sock.syswrite("GET / HTTP/1.0\r\n\r\n") results = sock.sysread(4096) @@ -801,7 +731,7 @@ EOF wait_for_file(new_sock_path) assert File.socket?(new_sock_path), "socket exists" @sockets.each do |path| - sock = UNIXSocket.new(path) + sock = unix_socket(path) sock.syswrite("GET / HTTP/1.0\r\n\r\n") results = sock.sysread(4096) assert_equal String, results.class diff --git a/test/test_helper.rb b/test/test_helper.rb index c21f75d..d86f83b 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -28,22 +28,43 @@ require 'tempfile' require 'fileutils' require 'logger' require 'unicorn' +require 'io/nonblock' if ENV['DEBUG'] require 'ruby-debug' Debugger.start end +unless RUBY_VERSION < '3.1' + warn "Unicorn was only tested against MRI up to 3.0.\n" \ + "It might not properly work with #{RUBY_VERSION}" +end + def redirect_test_io orig_err = STDERR.dup orig_out = STDOUT.dup - STDERR.reopen("test_stderr.#{$$}.log", "a") - STDOUT.reopen("test_stdout.#{$$}.log", "a") + rdr_pid = $$ + new_out = File.open("test_stdout.#$$.log", "a") + new_err = File.open("test_stderr.#$$.log", "a") + new_out.sync = new_err.sync = true + + if tail = ENV['TAIL'] # "tail -F" if GNU, "tail -f" otherwise + require 'shellwords' + cmd = tail.shellsplit + cmd << new_out.path + cmd << new_err.path + pid = Process.spawn(*cmd, { 1 => 2, :pgroup => true }) + sleep 0.1 # wait for tail(1) to startup + end + STDERR.reopen(new_err) + STDOUT.reopen(new_out) STDERR.sync = STDOUT.sync = true at_exit do - File.unlink("test_stderr.#{$$}.log") rescue nil - File.unlink("test_stdout.#{$$}.log") rescue nil + if rdr_pid == $$ + File.unlink(new_out.path) rescue nil + File.unlink(new_err.path) rescue nil + end end begin @@ -51,6 +72,7 @@ def redirect_test_io ensure STDERR.reopen(orig_err) STDOUT.reopen(orig_out) + Process.kill(:TERM, pid) if pid end end @@ -265,34 +287,20 @@ def wait_for_death(pid) raise "PID:#{pid} never died!" end -# executes +cmd+ and chunks its STDOUT -def chunked_spawn(stdout, *cmd) - fork { - crd, cwr = IO.pipe - crd.binmode - cwr.binmode - crd.sync = cwr.sync = true - - pid = fork { - STDOUT.reopen(cwr) - crd.close - cwr.close - exec(*cmd) - } - cwr.close - begin - buf = crd.readpartial(16384) - stdout.write("#{'%x' % buf.size}\r\n#{buf}") - rescue EOFError - stdout.write("0\r\n") - pid, status = Process.waitpid(pid) - exit status.exitstatus - end while true - } -end - def reset_sig_handlers %w(WINCH QUIT INT TERM USR1 USR2 HUP TTIN TTOU CHLD).each do |sig| trap(sig, "DEFAULT") end end + +def tcp_socket(*args) + sock = TCPSocket.new(*args) + sock.nonblock = false + sock +end + +def unix_socket(*args) + sock = UNIXSocket.new(*args) + sock.nonblock = false + sock +end diff --git a/test/unit/test_ccc.rb b/test/unit/test_ccc.rb index 3be1439..f518230 100644 --- a/test/unit/test_ccc.rb +++ b/test/unit/test_ccc.rb @@ -3,6 +3,7 @@ require 'unicorn' require 'io/wait' require 'tempfile' require 'test/unit' +require './test/test_helper' class TestCccTCPI < Test::Unit::TestCase def test_ccc_tcpi @@ -28,7 +29,7 @@ class TestCccTCPI < Test::Unit::TestCase # will wake up when writer closes sleep_pipe[0].read if env['PATH_INFO'] == '/sleep' - [ 200, [ %w(Content-Length 0), %w(Content-Type text/plain) ], [] ] + [ 200, {'content-length'=>'0', 'content-type'=>'text/plain'}, [] ] end ENV['UNICORN_FD'] = srv.fileno.to_s opts = { @@ -42,7 +43,7 @@ class TestCccTCPI < Test::Unit::TestCase wr.close # make sure the server is running, at least - client = TCPSocket.new(host, port) + client = tcp_socket(host, port) client.write("GET / HTTP/1.1\r\nHost: example.com\r\n\r\n") assert client.wait(10), 'never got response from server' res = client.read @@ -51,13 +52,13 @@ class TestCccTCPI < Test::Unit::TestCase client.close # start a slow request... - sleeper = TCPSocket.new(host, port) + sleeper = tcp_socket(host, port) sleeper.write("GET /sleep HTTP/1.1\r\nHost: example.com\r\n\r\n") # and a bunch of aborted ones nr = 100 nr.times do |i| - client = TCPSocket.new(host, port) + client = tcp_socket(host, port) client.write("GET /collections/#{rand(10000)} HTTP/1.1\r\n" \ "Host: example.com\r\n\r\n") client.close diff --git a/test/unit/test_http_parser_ng.rb b/test/unit/test_http_parser_ng.rb index d186f5a..425d5ad 100644 --- a/test/unit/test_http_parser_ng.rb +++ b/test/unit/test_http_parser_ng.rb @@ -11,6 +11,20 @@ class HttpParserNgTest < Test::Unit::TestCase @parser = HttpParser.new end + # RFC 7230 allows gzip/deflate/compress Transfer-Encoding, + # but "chunked" must be last if used + def test_is_chunked + [ 'chunked,chunked', 'chunked,gzip', 'chunked,gzip,chunked' ].each do |x| + assert_raise(HttpParserError) { HttpParser.is_chunked?(x) } + end + [ 'gzip, chunked', 'gzip,chunked', 'gzip ,chunked' ].each do |x| + assert HttpParser.is_chunked?(x) + end + [ 'gzip', 'xhunked', 'xchunked' ].each do |x| + assert !HttpParser.is_chunked?(x) + end + end + def test_parser_max_len assert_raises(RangeError) do HttpParser.max_header_len = 0xffffffff + 1 @@ -566,6 +580,73 @@ class HttpParserNgTest < Test::Unit::TestCase end end + def test_duplicate_content_length + str = "PUT / HTTP/1.1\r\n" \ + "Content-Length: 1\r\n" \ + "Content-Length: 9\r\n" \ + "\r\n" + assert_raises(HttpParserError) { @parser.headers({}, str) } + end + + def test_chunked_overrides_content_length + order = [ 'Transfer-Encoding: chunked', 'Content-Length: 666' ] + %w(a b).each do |x| + str = "PUT /#{x} HTTP/1.1\r\n" \ + "#{order.join("\r\n")}" \ + "\r\n\r\na\r\nhelloworld\r\n0\r\n\r\n" + order.reverse! + env = @parser.headers({}, str) + assert_nil @parser.content_length + assert_equal 'chunked', env['HTTP_TRANSFER_ENCODING'] + assert_equal '666', env['CONTENT_LENGTH'], + 'Content-Length logged so the app can log a possible client bug/attack' + @parser.filter_body(dst = '', str) + assert_equal 'helloworld', dst + @parser.parse # handle the non-existent trailer + assert @parser.next? + end + end + + def test_chunked_order_good + str = "PUT /x HTTP/1.1\r\n" \ + "Transfer-Encoding: gzip\r\n" \ + "Transfer-Encoding: chunked\r\n" \ + "\r\n" + env = @parser.headers({}, str) + assert_equal 'gzip,chunked', env['HTTP_TRANSFER_ENCODING'] + assert_nil @parser.content_length + + @parser.clear + str = "PUT /x HTTP/1.1\r\n" \ + "Transfer-Encoding: gzip, chunked\r\n" \ + "\r\n" + env = @parser.headers({}, str) + assert_equal 'gzip, chunked', env['HTTP_TRANSFER_ENCODING'] + assert_nil @parser.content_length + end + + def test_chunked_order_bad + str = "PUT /x HTTP/1.1\r\n" \ + "Transfer-Encoding: chunked\r\n" \ + "Transfer-Encoding: gzip\r\n" \ + "\r\n" + assert_raise(HttpParserError) { @parser.headers({}, str) } + end + + def test_double_chunked + str = "PUT /x HTTP/1.1\r\n" \ + "Transfer-Encoding: chunked\r\n" \ + "Transfer-Encoding: chunked\r\n" \ + "\r\n" + assert_raise(HttpParserError) { @parser.headers({}, str) } + + @parser.clear + str = "PUT /x HTTP/1.1\r\n" \ + "Transfer-Encoding: chunked,chunked\r\n" \ + "\r\n" + assert_raise(HttpParserError) { @parser.headers({}, str) } + end + def test_backtrace_is_empty begin @parser.headers({}, "AAADFSFDSFD\r\n\r\n") diff --git a/test/unit/test_request.rb b/test/unit/test_request.rb index 6cb0268..53ae944 100644 --- a/test/unit/test_request.rb +++ b/test/unit/test_request.rb @@ -10,19 +10,14 @@ include Unicorn class RequestTest < Test::Unit::TestCase - class MockRequest < StringIO - alias_method :readpartial, :sysread - alias_method :kgio_read!, :sysread - alias_method :read_nonblock, :sysread - def kgio_addr - '127.0.0.1' - end - end + MockRequest = Class.new(StringIO) + + AI = Addrinfo.new(Socket.sockaddr_un('/unicorn/sucks')) def setup @request = HttpRequest.new @app = lambda do |env| - [ 200, { 'Content-Length' => '0', 'Content-Type' => 'text/plain' }, [] ] + [ 200, { 'content-length' => '0', 'content-type' => 'text/plain' }, [] ] end @lint = Rack::Lint.new(@app) end @@ -30,7 +25,7 @@ class RequestTest < Test::Unit::TestCase def test_options client = MockRequest.new("OPTIONS * HTTP/1.1\r\n" \ "Host: foo\r\n\r\n") - env = @request.read(client) + env = @request.read_headers(client, AI) assert_equal '', env['REQUEST_PATH'] assert_equal '', env['PATH_INFO'] assert_equal '*', env['REQUEST_URI'] @@ -40,7 +35,7 @@ class RequestTest < Test::Unit::TestCase def test_absolute_uri_with_query client = MockRequest.new("GET http://e:3/x?y=z HTTP/1.1\r\n" \ "Host: foo\r\n\r\n") - env = @request.read(client) + env = @request.read_headers(client, AI) assert_equal '/x', env['REQUEST_PATH'] assert_equal '/x', env['PATH_INFO'] assert_equal 'y=z', env['QUERY_STRING'] @@ -50,7 +45,7 @@ class RequestTest < Test::Unit::TestCase def test_absolute_uri_with_fragment client = MockRequest.new("GET http://e:3/x#frag HTTP/1.1\r\n" \ "Host: foo\r\n\r\n") - env = @request.read(client) + env = @request.read_headers(client, AI) assert_equal '/x', env['REQUEST_PATH'] assert_equal '/x', env['PATH_INFO'] assert_equal '', env['QUERY_STRING'] @@ -61,7 +56,7 @@ class RequestTest < Test::Unit::TestCase def test_absolute_uri_with_query_and_fragment client = MockRequest.new("GET http://e:3/x?a=b#frag HTTP/1.1\r\n" \ "Host: foo\r\n\r\n") - env = @request.read(client) + env = @request.read_headers(client, AI) assert_equal '/x', env['REQUEST_PATH'] assert_equal '/x', env['PATH_INFO'] assert_equal 'a=b', env['QUERY_STRING'] @@ -73,7 +68,7 @@ class RequestTest < Test::Unit::TestCase %w(ssh+http://e/ ftp://e/x http+ssh://e/x).each do |abs_uri| client = MockRequest.new("GET #{abs_uri} HTTP/1.1\r\n" \ "Host: foo\r\n\r\n") - assert_raises(HttpParserError) { @request.read(client) } + assert_raises(HttpParserError) { @request.read_headers(client, AI) } end end @@ -81,7 +76,7 @@ class RequestTest < Test::Unit::TestCase client = MockRequest.new("GET / HTTP/1.1\r\n" \ "X-Forwarded-Proto: https\r\n" \ "Host: foo\r\n\r\n") - env = @request.read(client) + env = @request.read_headers(client, AI) assert_equal "https", env['rack.url_scheme'] assert_kind_of Array, @lint.call(env) end @@ -90,7 +85,7 @@ class RequestTest < Test::Unit::TestCase client = MockRequest.new("GET / HTTP/1.1\r\n" \ "X-Forwarded-Proto: http\r\n" \ "Host: foo\r\n\r\n") - env = @request.read(client) + env = @request.read_headers(client, AI) assert_equal "http", env['rack.url_scheme'] assert_kind_of Array, @lint.call(env) end @@ -99,14 +94,14 @@ class RequestTest < Test::Unit::TestCase client = MockRequest.new("GET / HTTP/1.1\r\n" \ "X-Forwarded-Proto: ftp\r\n" \ "Host: foo\r\n\r\n") - env = @request.read(client) + env = @request.read_headers(client, AI) assert_equal "http", env['rack.url_scheme'] assert_kind_of Array, @lint.call(env) end def test_rack_lint_get client = MockRequest.new("GET / HTTP/1.1\r\nHost: foo\r\n\r\n") - env = @request.read(client) + env = @request.read_headers(client, AI) assert_equal "http", env['rack.url_scheme'] assert_equal '127.0.0.1', env['REMOTE_ADDR'] assert_kind_of Array, @lint.call(env) @@ -114,7 +109,7 @@ class RequestTest < Test::Unit::TestCase def test_no_content_stringio client = MockRequest.new("GET / HTTP/1.1\r\nHost: foo\r\n\r\n") - env = @request.read(client) + env = @request.read_headers(client, AI) assert_equal StringIO, env['rack.input'].class end @@ -122,7 +117,7 @@ class RequestTest < Test::Unit::TestCase client = MockRequest.new("PUT / HTTP/1.1\r\n" \ "Content-Length: 0\r\n" \ "Host: foo\r\n\r\n") - env = @request.read(client) + env = @request.read_headers(client, AI) assert_equal StringIO, env['rack.input'].class end @@ -130,7 +125,7 @@ class RequestTest < Test::Unit::TestCase client = MockRequest.new("PUT / HTTP/1.1\r\n" \ "Content-Length: 1\r\n" \ "Host: foo\r\n\r\n") - env = @request.read(client) + env = @request.read_headers(client, AI) assert_equal Unicorn::TeeInput, env['rack.input'].class end @@ -141,7 +136,7 @@ class RequestTest < Test::Unit::TestCase "Content-Length: 5\r\n" \ "\r\n" \ "abcde") - env = @request.read(client) + env = @request.read_headers(client, AI) assert ! env.include?(:http_body) assert_kind_of Array, @lint.call(env) end @@ -152,14 +147,6 @@ class RequestTest < Test::Unit::TestCase buf = (' ' * bs).freeze length = bs * count client = Tempfile.new('big_put') - def client.kgio_addr; '127.0.0.1'; end - def client.kgio_read(*args) - readpartial(*args) - rescue EOFError - end - def client.kgio_read!(*args) - readpartial(*args) - end client.syswrite( "PUT / HTTP/1.1\r\n" \ "Host: foo\r\n" \ @@ -167,7 +154,7 @@ class RequestTest < Test::Unit::TestCase "\r\n") count.times { assert_equal bs, client.syswrite(buf) } assert_equal 0, client.sysseek(0) - env = @request.read(client) + env = @request.read_headers(client, AI) assert ! env.include?(:http_body) assert_equal length, env['rack.input'].size count.times { diff --git a/test/unit/test_response.rb b/test/unit/test_response.rb deleted file mode 100644 index fbe433f..0000000 --- a/test/unit/test_response.rb +++ /dev/null @@ -1,111 +0,0 @@ -# -*- encoding: binary -*- - -# Copyright (c) 2005 Zed A. Shaw -# You can redistribute it and/or modify it under the same terms as Ruby 1.8 or -# the GPLv2+ (GPLv3+ preferred) -# -# Additional work donated by contributors. See git history -# for more information. - -require './test/test_helper' -require 'time' - -include Unicorn - -class ResponseTest < Test::Unit::TestCase - include Unicorn::HttpResponse - - def test_httpdate - before = Time.now.to_i - 1 - str = httpdate - assert_kind_of(String, str) - middle = Time.parse(str).to_i - after = Time.now.to_i - assert before <= middle - assert middle <= after - end - - def test_response_headers - out = StringIO.new - http_response_write(out, 200, {"X-Whatever" => "stuff"}, ["cool"]) - assert ! out.closed? - - assert out.length > 0, "output didn't have data" - end - - # ref: <CAO47=rJa=zRcLn_Xm4v2cHPr6c0UswaFC_omYFEH+baSxHOWKQ@mail.gmail.com> - def test_response_header_broken_nil - out = StringIO.new - http_response_write(out, 200, {"Nil" => nil}, %w(hysterical raisin)) - assert ! out.closed? - - assert_match %r{^Nil: \r\n}sm, out.string, 'nil accepted' - end - - def test_response_string_status - out = StringIO.new - http_response_write(out,'200', {}, []) - assert ! out.closed? - assert out.length > 0, "output didn't have data" - end - - def test_response_200 - io = StringIO.new - http_response_write(io, 200, {}, []) - assert ! io.closed? - assert io.length > 0, "output didn't have data" - end - - def test_response_with_default_reason - code = 400 - io = StringIO.new - http_response_write(io, code, {}, []) - assert ! io.closed? - lines = io.string.split(/\r\n/) - assert_match(/.* Bad Request$/, lines.first, - "wrong default reason phrase") - end - - def test_rack_multivalue_headers - out = StringIO.new - http_response_write(out,200, {"X-Whatever" => "stuff\nbleh"}, []) - assert ! out.closed? - assert_match(/^X-Whatever: stuff\r\nX-Whatever: bleh\r\n/, out.string) - end - - # Even though Rack explicitly forbids "Status" in the header hash, - # some broken clients still rely on it - def test_status_header_added - out = StringIO.new - http_response_write(out,200, {"X-Whatever" => "stuff"}, []) - assert ! out.closed? - end - - def test_unknown_status_pass_through - out = StringIO.new - http_response_write(out,"666 I AM THE BEAST", {}, [] ) - assert ! out.closed? - headers = out.string.split(/\r\n\r\n/).first.split(/\r\n/) - assert %r{\AHTTP/\d\.\d 666 I AM THE BEAST\z}.match(headers[0]) - end - - def test_modified_rack_http_status_codes_late - r, w = IO.pipe - pid = fork do - r.close - # Users may want to globally override the status text associated - # with an HTTP status code in their app. - Rack::Utils::HTTP_STATUS_CODES[200] = "HI" - http_response_write(w, 200, {}, []) - w.close - end - w.close - assert_equal "HTTP/1.1 200 HI\r\n", r.gets - r.read # just drain the pipe - pid, status = Process.waitpid2(pid) - assert status.success?, status.inspect - ensure - r.close - w.close unless w.closed? - end -end diff --git a/test/unit/test_server.rb b/test/unit/test_server.rb index 8096955..7ffa48f 100644 --- a/test/unit/test_server.rb +++ b/test/unit/test_server.rb @@ -16,13 +16,30 @@ class TestHandler def call(env) while env['rack.input'].read(4096) end - [200, { 'Content-Type' => 'text/plain' }, ['hello!\n']] + [200, { 'content-type' => 'text/plain' }, ['hello!\n']] rescue Unicorn::ClientShutdown, Unicorn::HttpParserError => e $stderr.syswrite("#{e.class}: #{e.message} #{e.backtrace.empty?}\n") raise e end end +class TestRackAfterReply + def initialize + @called = false + end + + def call(env) + while env['rack.input'].read(4096) + end + + env["rack.after_reply"] << -> { @called = true } + + [200, { 'content-type' => 'text/plain' }, ["after_reply_called: #{@called}"]] + rescue Unicorn::ClientShutdown, Unicorn::HttpParserError => e + $stderr.syswrite("#{e.class}: #{e.message} #{e.backtrace.empty?}\n") + raise e + end +end class WebServerTest < Test::Unit::TestCase @@ -53,7 +70,7 @@ class WebServerTest < Test::Unit::TestCase tmp.sysseek(0) tmp.truncate(0) tmp.syswrite($$) - lambda { |env| [ 200, { 'Content-Type' => 'text/plain' }, [ "#$$\n" ] ] } + lambda { |env| [ 200, { 'content-type' => 'text/plain' }, [ "#$$\n" ] ] } } redirect_test_io do @server = HttpServer.new(app, :listeners => [ "127.0.0.1:#@port"] ) @@ -84,18 +101,30 @@ class WebServerTest < Test::Unit::TestCase tmp.close! end - def test_broken_app + def test_after_reply teardown - app = lambda { |env| raise RuntimeError, "hello" } - # [200, {}, []] } + redirect_test_io do - @server = HttpServer.new(app, :listeners => [ "127.0.0.1:#@port"] ) + @server = HttpServer.new(TestRackAfterReply.new, + :listeners => [ "127.0.0.1:#@port"]) @server.start end - sock = TCPSocket.new('127.0.0.1', @port) + + sock = tcp_socket('127.0.0.1', @port) sock.syswrite("GET / HTTP/1.0\r\n\r\n") - assert_match %r{\AHTTP/1.[01] 500\b}, sock.sysread(4096) - assert_nil sock.close + + responses = sock.read(4096) + assert_match %r{\AHTTP/1.[01] 200\b}, responses + assert_match %r{^after_reply_called: false}, responses + + sock = tcp_socket('127.0.0.1', @port) + sock.syswrite("GET / HTTP/1.0\r\n\r\n") + + responses = sock.read(4096) + assert_match %r{\AHTTP/1.[01] 200\b}, responses + assert_match %r{^after_reply_called: true}, responses + + sock.close end def test_simple_server @@ -103,62 +132,9 @@ class WebServerTest < Test::Unit::TestCase assert_equal 'hello!\n', results[0], "Handler didn't really run" end - def test_client_shutdown_writes - bs = 15609315 * rand - sock = TCPSocket.new('127.0.0.1', @port) - sock.syswrite("PUT /hello HTTP/1.1\r\n") - sock.syswrite("Host: example.com\r\n") - sock.syswrite("Transfer-Encoding: chunked\r\n") - sock.syswrite("Trailer: X-Foo\r\n") - sock.syswrite("\r\n") - sock.syswrite("%x\r\n" % [ bs ]) - sock.syswrite("F" * bs) - sock.syswrite("\r\n0\r\nX-") - "Foo: bar\r\n\r\n".each_byte do |x| - sock.syswrite x.chr - sleep 0.05 - end - # we wrote the entire request before shutting down, server should - # continue to process our request and never hit EOFError on our sock - sock.shutdown(Socket::SHUT_WR) - buf = sock.read - assert_equal 'hello!\n', buf.split(/\r\n\r\n/).last - next_client = Net::HTTP.get(URI.parse("http://127.0.0.1:#@port/")) - assert_equal 'hello!\n', next_client - lines = File.readlines("test_stderr.#$$.log") - assert lines.grep(/^Unicorn::ClientShutdown: /).empty? - assert_nil sock.close - end - - def test_client_shutdown_write_truncates - bs = 15609315 * rand - sock = TCPSocket.new('127.0.0.1', @port) - sock.syswrite("PUT /hello HTTP/1.1\r\n") - sock.syswrite("Host: example.com\r\n") - sock.syswrite("Transfer-Encoding: chunked\r\n") - sock.syswrite("Trailer: X-Foo\r\n") - sock.syswrite("\r\n") - sock.syswrite("%x\r\n" % [ bs ]) - sock.syswrite("F" * (bs / 2.0)) - - # shutdown prematurely, this will force the server to abort - # processing on us even during app dispatch - sock.shutdown(Socket::SHUT_WR) - IO.select([sock], nil, nil, 60) or raise "Timed out" - buf = sock.read - assert_equal "", buf - next_client = Net::HTTP.get(URI.parse("http://127.0.0.1:#@port/")) - assert_equal 'hello!\n', next_client - lines = File.readlines("test_stderr.#$$.log") - lines = lines.grep(/^Unicorn::ClientShutdown: bytes_read=\d+/) - assert_equal 1, lines.size - assert_match %r{\AUnicorn::ClientShutdown: bytes_read=\d+ true$}, lines[0] - assert_nil sock.close - end - def test_client_malformed_body bs = 15653984 - sock = TCPSocket.new('127.0.0.1', @port) + sock = tcp_socket('127.0.0.1', @port) sock.syswrite("PUT /hello HTTP/1.1\r\n") sock.syswrite("Host: example.com\r\n") sock.syswrite("Transfer-Encoding: chunked\r\n") @@ -180,7 +156,7 @@ class WebServerTest < Test::Unit::TestCase def do_test(string, chunk, close_after=nil, shutdown_delay=0) # Do not use instance variables here, because it needs to be thread safe - socket = TCPSocket.new("127.0.0.1", @port); + socket = tcp_socket("127.0.0.1", @port); request = StringIO.new(string) chunks_out = 0 @@ -225,14 +201,14 @@ class WebServerTest < Test::Unit::TestCase end def test_bad_client_400 - sock = TCPSocket.new('127.0.0.1', @port) + sock = tcp_socket('127.0.0.1', @port) sock.syswrite("GET / HTTP/1.0\r\nHost: foo\rbar\r\n\r\n") assert_match %r{\AHTTP/1.[01] 400\b}, sock.sysread(4096) assert_nil sock.close end def test_http_0_9 - sock = TCPSocket.new('127.0.0.1', @port) + sock = tcp_socket('127.0.0.1', @port) sock.syswrite("GET /hello\r\n") assert_match 'hello!\n', sock.sysread(4096) assert_nil sock.close diff --git a/test/unit/test_signals.rb b/test/unit/test_signals.rb index 4d9fdc5..6c48754 100644 --- a/test/unit/test_signals.rb +++ b/test/unit/test_signals.rb @@ -47,16 +47,16 @@ class SignalsTest < Test::Unit::TestCase def test_worker_dies_on_dead_master pid = fork { - app = lambda { |env| [ 200, {'X-Pid' => "#$$" }, [] ] } + app = lambda { |env| [ 200, {'x-pid' => "#$$" }, [] ] } opts = @server_opts.merge(:timeout => 3) redirect_test_io { HttpServer.new(app, opts).start.join } } wait_workers_ready("test_stderr.#{pid}.log", 1) - sock = TCPSocket.new('127.0.0.1', @port) + sock = tcp_socket('127.0.0.1', @port) sock.syswrite("GET / HTTP/1.0\r\n\r\n") buf = sock.readpartial(4096) assert_nil sock.close - buf =~ /\bX-Pid: (\d+)\b/ or raise Exception + buf =~ /\bx-pid: (\d+)\b/ or raise Exception child = $1.to_i wait_master_ready("test_stderr.#{pid}.log") wait_workers_ready("test_stderr.#{pid}.log", 1) @@ -79,7 +79,7 @@ class SignalsTest < Test::Unit::TestCase } wr.close wait_workers_ready("test_stderr.#{pid}.log", 1) - sock = TCPSocket.new('127.0.0.1', @port) + sock = tcp_socket('127.0.0.1', @port) sock.syswrite("GET / HTTP/1.0\r\n\r\n") buf = rd.readpartial(1) wait_master_ready("test_stderr.#{pid}.log") @@ -102,7 +102,7 @@ class SignalsTest < Test::Unit::TestCase } t0 = Time.now wait_workers_ready("test_stderr.#{pid}.log", 1) - sock = TCPSocket.new('127.0.0.1', @port) + sock = tcp_socket('127.0.0.1', @port) sock.syswrite("GET / HTTP/1.0\r\n\r\n") buf = nil @@ -120,17 +120,17 @@ class SignalsTest < Test::Unit::TestCase def test_response_write app = lambda { |env| - [ 200, { 'Content-Type' => 'text/plain', 'X-Pid' => Process.pid.to_s }, + [ 200, { 'content-type' => 'text/plain', 'x-pid' => Process.pid.to_s }, Dd.new(@bs, @count) ] } redirect_test_io { @server = HttpServer.new(app, @server_opts).start } wait_workers_ready("test_stderr.#{$$}.log", 1) - sock = TCPSocket.new('127.0.0.1', @port) + sock = tcp_socket('127.0.0.1', @port) sock.syswrite("GET / HTTP/1.0\r\n\r\n") buf = '' header_len = pid = nil buf = sock.sysread(16384, buf) - pid = buf[/\r\nX-Pid: (\d+)\r\n/, 1].to_i + pid = buf[/\r\nx-pid: (\d+)\r\n/, 1].to_i header_len = buf[/\A(.+?\r\n\r\n)/m, 1].size assert pid > 0, "pid not positive: #{pid.inspect}" read = buf.size @@ -158,18 +158,18 @@ class SignalsTest < Test::Unit::TestCase app = lambda { |env| while env['rack.input'].read(4096) end - [ 200, {'Content-Type'=>'text/plain', 'X-Pid'=>Process.pid.to_s}, [] ] + [ 200, {'content-type'=>'text/plain', 'x-pid'=>Process.pid.to_s}, [] ] } redirect_test_io { @server = HttpServer.new(app, @server_opts).start } wait_workers_ready("test_stderr.#{$$}.log", 1) - sock = TCPSocket.new('127.0.0.1', @port) + sock = tcp_socket('127.0.0.1', @port) sock.syswrite("GET / HTTP/1.0\r\n\r\n") - pid = sock.sysread(4096)[/\r\nX-Pid: (\d+)\r\n/, 1].to_i + pid = sock.sysread(4096)[/\r\nx-pid: (\d+)\r\n/, 1].to_i assert_nil sock.close assert pid > 0, "pid not positive: #{pid.inspect}" - sock = TCPSocket.new('127.0.0.1', @port) + sock = tcp_socket('127.0.0.1', @port) sock.syswrite("PUT / HTTP/1.0\r\n") sock.syswrite("Content-Length: #{@bs * @count}\r\n\r\n") 1000.times { Process.kill(:HUP, pid) } @@ -182,7 +182,7 @@ class SignalsTest < Test::Unit::TestCase redirect_test_io { @server.stop(true) } # can't check for == since pending signals get merged assert size_before < @tmp.stat.size - assert_equal pid, sock.sysread(4096)[/\r\nX-Pid: (\d+)\r\n/, 1].to_i + assert_equal pid, sock.sysread(4096)[/\r\nx-pid: (\d+)\r\n/, 1].to_i assert_nil sock.close end end diff --git a/test/unit/test_socket_helper.rb b/test/unit/test_socket_helper.rb index fbc7bb9..a446f06 100644 --- a/test/unit/test_socket_helper.rb +++ b/test/unit/test_socket_helper.rb @@ -24,7 +24,8 @@ class TestSocketHelper < Test::Unit::TestCase port = unused_port @test_addr @tcp_listener_name = "#@test_addr:#{port}" @tcp_listener = bind_listen(@tcp_listener_name) - assert TCPServer === @tcp_listener + assert Socket === @tcp_listener + assert @tcp_listener.local_address.ip? assert_equal @tcp_listener_name, sock_name(@tcp_listener) end @@ -38,10 +39,10 @@ class TestSocketHelper < Test::Unit::TestCase { :backlog => 16, :rcvbuf => 4096, :sndbuf => 4096 } ].each do |opts| tcp_listener = bind_listen(tcp_listener_name, opts) - assert TCPServer === tcp_listener + assert tcp_listener.local_address.ip? tcp_listener.close unix_listener = bind_listen(unix_listener_name, opts) - assert UNIXServer === unix_listener + assert unix_listener.local_address.unix? unix_listener.close end end @@ -52,11 +53,13 @@ class TestSocketHelper < Test::Unit::TestCase @unix_listener_path = tmp.path File.unlink(@unix_listener_path) @unix_listener = bind_listen(@unix_listener_path) - assert UNIXServer === @unix_listener + assert Socket === @unix_listener + assert @unix_listener.local_address.unix? assert_equal @unix_listener_path, sock_name(@unix_listener) assert File.readable?(@unix_listener_path), "not readable" assert File.writable?(@unix_listener_path), "not writable" assert_equal 0777, File.umask + assert_equal @unix_listener, bind_listen(@unix_listener) ensure File.umask(old_umask) end @@ -67,7 +70,6 @@ class TestSocketHelper < Test::Unit::TestCase @unix_listener_path = tmp.path File.unlink(@unix_listener_path) @unix_listener = bind_listen(@unix_listener_path, :umask => 077) - assert UNIXServer === @unix_listener assert_equal @unix_listener_path, sock_name(@unix_listener) assert_equal 0140700, File.stat(@unix_listener_path).mode assert_equal 0777, File.umask @@ -75,28 +77,6 @@ class TestSocketHelper < Test::Unit::TestCase File.umask(old_umask) end - def test_bind_listen_unix_idempotent - test_bind_listen_unix - a = bind_listen(@unix_listener) - assert_equal a.fileno, @unix_listener.fileno - unix_server = server_cast(@unix_listener) - assert UNIXServer === unix_server - a = bind_listen(unix_server) - assert_equal a.fileno, unix_server.fileno - assert_equal a.fileno, @unix_listener.fileno - end - - def test_bind_listen_tcp_idempotent - test_bind_listen_tcp - a = bind_listen(@tcp_listener) - assert_equal a.fileno, @tcp_listener.fileno - tcp_server = server_cast(@tcp_listener) - assert TCPServer === tcp_server - a = bind_listen(tcp_server) - assert_equal a.fileno, tcp_server.fileno - assert_equal a.fileno, @tcp_listener.fileno - end - def test_bind_listen_unix_rebind test_bind_listen_unix new_listener = nil @@ -107,48 +87,26 @@ class TestSocketHelper < Test::Unit::TestCase File.unlink(@unix_listener_path) new_listener = bind_listen(@unix_listener_path) - assert UNIXServer === new_listener assert new_listener.fileno != @unix_listener.fileno assert_equal sock_name(new_listener), sock_name(@unix_listener) assert_equal @unix_listener_path, sock_name(new_listener) pid = fork do - client = server_cast(new_listener).accept - client.syswrite('abcde') - exit 0 + begin + client, _ = new_listener.accept + client.syswrite('abcde') + exit 0 + rescue => e + warn "#{e.message} (#{e.class})" + exit 1 + end end - s = UNIXSocket.new(@unix_listener_path) + s = unix_socket(@unix_listener_path) IO.select([s]) assert_equal 'abcde', s.sysread(5) pid, status = Process.waitpid2(pid) assert status.success? end - def test_server_cast - test_bind_listen_unix - test_bind_listen_tcp - unix_listener_socket = Socket.for_fd(@unix_listener.fileno) - assert Socket === unix_listener_socket - @unix_server = server_cast(unix_listener_socket) - assert_equal @unix_listener.fileno, @unix_server.fileno - assert UNIXServer === @unix_server - assert_equal(@unix_server.path, @unix_listener.path, - "##{@unix_server.path} != #{@unix_listener.path}") - assert File.socket?(@unix_server.path) - assert_equal @unix_listener_path, sock_name(@unix_server) - - tcp_listener_socket = Socket.for_fd(@tcp_listener.fileno) - assert Socket === tcp_listener_socket - @tcp_server = server_cast(tcp_listener_socket) - assert_equal @tcp_listener.fileno, @tcp_server.fileno - assert TCPServer === @tcp_server - assert_equal @tcp_listener_name, sock_name(@tcp_server) - end - - def test_sock_name - test_server_cast - sock_name(@unix_server) - end - def test_tcp_defer_accept_default return unless defined?(TCP_DEFER_ACCEPT) port = unused_port @test_addr diff --git a/test/unit/test_stream_input.rb b/test/unit/test_stream_input.rb index 1a07ec3..7986ca7 100644 --- a/test/unit/test_stream_input.rb +++ b/test/unit/test_stream_input.rb @@ -6,16 +6,16 @@ require 'unicorn' class TestStreamInput < Test::Unit::TestCase def setup - @rs = $/ + @rs = "\n" + $/ == "\n" or abort %q{test broken if \$/ != "\\n"} @env = {} - @rd, @wr = Kgio::UNIXSocket.pair + @rd, @wr = UNIXSocket.pair @rd.sync = @wr.sync = true @start_pid = $$ end def teardown return if $$ != @start_pid - $/ = @rs @rd.close rescue nil @wr.close rescue nil Process.waitall @@ -54,11 +54,18 @@ class TestStreamInput < Test::Unit::TestCase end def test_gets_empty_rs - $/ = nil r = init_request("a\nb\n\n") si = Unicorn::StreamInput.new(@rd, r) - assert_equal "a\nb\n\n", si.gets - assert_nil si.gets + pid = fork do # to avoid $/ warning (hopefully) + $/ = nil + @rd.close + @wr.write(si.gets) + @wr.close + end + @wr.close + assert_equal "a\nb\n\n", @rd.read + pid, status = Process.waitpid2(pid) + assert_predicate status, :success? end def test_read_with_equal_len @@ -90,21 +97,21 @@ class TestStreamInput < Test::Unit::TestCase end def test_gets_long - r = init_request("hello", 5 + (4096 * 4 * 3) + "#$/foo#$/".size) + r = init_request("hello", 5 + (4096 * 4 * 3) + "#{@rs}foo#{@rs}".size) si = Unicorn::StreamInput.new(@rd, r) status = line = nil pid = fork { @rd.close 3.times { @wr.write("ffff" * 4096) } - @wr.write "#$/foo#$/" + @wr.write "#{@rs}foo#{@rs}" @wr.close } @wr.close line = si.gets assert_equal(4096 * 4 * 3 + 5 + $/.size, line.size) - assert_equal("hello" << ("ffff" * 4096 * 3) << "#$/", line) + assert_equal("hello" << ("ffff" * 4096 * 3) << "#{@rs}", line) line = si.gets - assert_equal "foo#$/", line + assert_equal "foo#{@rs}", line assert_nil si.gets pid, status = Process.waitpid2(pid) assert status.success? diff --git a/test/unit/test_tee_input.rb b/test/unit/test_tee_input.rb index 4647e66..607ce87 100644 --- a/test/unit/test_tee_input.rb +++ b/test/unit/test_tee_input.rb @@ -9,17 +9,16 @@ class TeeInput < Unicorn::TeeInput end class TestTeeInput < Test::Unit::TestCase - def setup - @rs = $/ - @rd, @wr = Kgio::UNIXSocket.pair + @rd, @wr = UNIXSocket.pair @rd.sync = @wr.sync = true @start_pid = $$ + @rs = "\n" + $/ == "\n" or abort %q{test broken if \$/ != "\\n"} end def teardown return if $$ != @start_pid - $/ = @rs @rd.close rescue nil @wr.close rescue nil begin @@ -37,38 +36,38 @@ class TestTeeInput < Test::Unit::TestCase end def test_gets_long - r = init_request("hello", 5 + (4096 * 4 * 3) + "#$/foo#$/".size) + r = init_request("hello", 5 + (4096 * 4 * 3) + "#{@rs}foo#{@rs}".size) ti = TeeInput.new(@rd, r) status = line = nil pid = fork { @rd.close 3.times { @wr.write("ffff" * 4096) } - @wr.write "#$/foo#$/" + @wr.write "#{@rs}foo#{@rs}" @wr.close } @wr.close line = ti.gets assert_equal(4096 * 4 * 3 + 5 + $/.size, line.size) - assert_equal("hello" << ("ffff" * 4096 * 3) << "#$/", line) + assert_equal("hello" << ("ffff" * 4096 * 3) << "#{@rs}", line) line = ti.gets - assert_equal "foo#$/", line + assert_equal "foo#{@rs}", line assert_nil ti.gets pid, status = Process.waitpid2(pid) assert status.success? end def test_gets_short - r = init_request("hello", 5 + "#$/foo".size) + r = init_request("hello", 5 + "#{@rs}foo".size) ti = TeeInput.new(@rd, r) status = line = nil pid = fork { @rd.close - @wr.write "#$/foo" + @wr.write "#{@rs}foo" @wr.close } @wr.close line = ti.gets - assert_equal("hello#$/", line) + assert_equal("hello#{@rs}", line) line = ti.gets assert_equal "foo", line assert_nil ti.gets diff --git a/test/unit/test_upload.rb b/test/unit/test_upload.rb deleted file mode 100644 index 5de02e4..0000000 --- a/test/unit/test_upload.rb +++ /dev/null @@ -1,306 +0,0 @@ -# -*- encoding: binary -*- - -# Copyright (c) 2009 Eric Wong -require './test/test_helper' -require 'digest/md5' - -include Unicorn - -class UploadTest < Test::Unit::TestCase - - def setup - @addr = ENV['UNICORN_TEST_ADDR'] || '127.0.0.1' - @port = unused_port - @hdr = {'Content-Type' => 'text/plain', 'Content-Length' => '0'} - @bs = 4096 - @count = 256 - @server = nil - - # we want random binary data to test 1.9 encoding-aware IO craziness - @random = File.open('/dev/urandom','rb') - @sha1 = Digest::SHA1.new - @sha1_app = lambda do |env| - input = env['rack.input'] - resp = {} - - @sha1.reset - while buf = input.read(@bs) - @sha1.update(buf) - end - resp[:sha1] = @sha1.hexdigest - - # rewind and read again - input.rewind - @sha1.reset - while buf = input.read(@bs) - @sha1.update(buf) - end - - if resp[:sha1] == @sha1.hexdigest - resp[:sysread_read_byte_match] = true - end - - if expect_size = env['HTTP_X_EXPECT_SIZE'] - if expect_size.to_i == input.size - resp[:expect_size_match] = true - end - end - resp[:size] = input.size - resp[:content_md5] = env['HTTP_CONTENT_MD5'] - - [ 200, @hdr.merge({'X-Resp' => resp.inspect}), [] ] - end - end - - def teardown - redirect_test_io { @server.stop(false) } if @server - @random.close - reset_sig_handlers - end - - def test_put - start_server(@sha1_app) - sock = TCPSocket.new(@addr, @port) - sock.syswrite("PUT / HTTP/1.0\r\nContent-Length: #{length}\r\n\r\n") - @count.times do |i| - buf = @random.sysread(@bs) - @sha1.update(buf) - sock.syswrite(buf) - end - read = sock.read.split(/\r\n/) - assert_equal "HTTP/1.1 200 OK", read[0] - resp = eval(read.grep(/^X-Resp: /).first.sub!(/X-Resp: /, '')) - assert_equal length, resp[:size] - assert_equal @sha1.hexdigest, resp[:sha1] - end - - def test_put_content_md5 - md5 = Digest::MD5.new - start_server(@sha1_app) - sock = TCPSocket.new(@addr, @port) - sock.syswrite("PUT / HTTP/1.0\r\nTransfer-Encoding: chunked\r\n" \ - "Trailer: Content-MD5\r\n\r\n") - @count.times do |i| - buf = @random.sysread(@bs) - @sha1.update(buf) - md5.update(buf) - sock.syswrite("#{'%x' % buf.size}\r\n") - sock.syswrite(buf << "\r\n") - end - sock.syswrite("0\r\n") - - content_md5 = [ md5.digest! ].pack('m').strip.freeze - sock.syswrite("Content-MD5: #{content_md5}\r\n\r\n") - read = sock.read.split(/\r\n/) - assert_equal "HTTP/1.1 200 OK", read[0] - resp = eval(read.grep(/^X-Resp: /).first.sub!(/X-Resp: /, '')) - assert_equal length, resp[:size] - assert_equal @sha1.hexdigest, resp[:sha1] - assert_equal content_md5, resp[:content_md5] - end - - def test_put_trickle_small - @count, @bs = 2, 128 - start_server(@sha1_app) - assert_equal 256, length - sock = TCPSocket.new(@addr, @port) - hdr = "PUT / HTTP/1.0\r\nContent-Length: #{length}\r\n\r\n" - @count.times do - buf = @random.sysread(@bs) - @sha1.update(buf) - hdr << buf - sock.syswrite(hdr) - hdr = '' - sleep 0.6 - end - read = sock.read.split(/\r\n/) - assert_equal "HTTP/1.1 200 OK", read[0] - resp = eval(read.grep(/^X-Resp: /).first.sub!(/X-Resp: /, '')) - assert_equal length, resp[:size] - assert_equal @sha1.hexdigest, resp[:sha1] - end - - def test_put_keepalive_truncates_small_overwrite - start_server(@sha1_app) - sock = TCPSocket.new(@addr, @port) - to_upload = length + 1 - sock.syswrite("PUT / HTTP/1.0\r\nContent-Length: #{to_upload}\r\n\r\n") - @count.times do - buf = @random.sysread(@bs) - @sha1.update(buf) - sock.syswrite(buf) - end - sock.syswrite('12345') # write 4 bytes more than we expected - @sha1.update('1') - - buf = sock.readpartial(4096) - while buf !~ /\r\n\r\n/ - buf << sock.readpartial(4096) - end - read = buf.split(/\r\n/) - assert_equal "HTTP/1.1 200 OK", read[0] - resp = eval(read.grep(/^X-Resp: /).first.sub!(/X-Resp: /, '')) - assert_equal to_upload, resp[:size] - assert_equal @sha1.hexdigest, resp[:sha1] - end - - def test_put_excessive_overwrite_closed - tmp = Tempfile.new('overwrite_check') - tmp.sync = true - start_server(lambda { |env| - nr = 0 - while buf = env['rack.input'].read(65536) - nr += buf.size - end - tmp.write(nr.to_s) - [ 200, @hdr, [] ] - }) - sock = TCPSocket.new(@addr, @port) - buf = ' ' * @bs - sock.syswrite("PUT / HTTP/1.0\r\nContent-Length: #{length}\r\n\r\n") - - @count.times { sock.syswrite(buf) } - assert_raise(Errno::ECONNRESET, Errno::EPIPE) do - ::Unicorn::Const::CHUNK_SIZE.times { sock.syswrite(buf) } - end - sock.gets - tmp.rewind - assert_equal length, tmp.read.to_i - end - - # Despite reading numerous articles and inspecting the 1.9.1-p0 C - # source, Eric Wong will never trust that we're always handling - # encoding-aware IO objects correctly. Thus this test uses shell - # utilities that should always operate on files/sockets on a - # byte-level. - def test_uncomfortable_with_onenine_encodings - # POSIX doesn't require all of these to be present on a system - which('curl') or return - which('sha1sum') or return - which('dd') or return - - start_server(@sha1_app) - - tmp = Tempfile.new('dd_dest') - assert(system("dd", "if=#{@random.path}", "of=#{tmp.path}", - "bs=#{@bs}", "count=#{@count}"), - "dd #@random to #{tmp}") - sha1_re = %r!\b([a-f0-9]{40})\b! - sha1_out = `sha1sum #{tmp.path}` - assert $?.success?, 'sha1sum ran OK' - - assert_match(sha1_re, sha1_out) - sha1 = sha1_re.match(sha1_out)[1] - resp = `curl -isSfN -T#{tmp.path} http://#@addr:#@port/` - assert $?.success?, 'curl ran OK' - assert_match(%r!\b#{sha1}\b!, resp) - assert_match(/sysread_read_byte_match/, resp) - - # small StringIO path - assert(system("dd", "if=#{@random.path}", "of=#{tmp.path}", - "bs=1024", "count=1"), - "dd #@random to #{tmp}") - sha1_re = %r!\b([a-f0-9]{40})\b! - sha1_out = `sha1sum #{tmp.path}` - assert $?.success?, 'sha1sum ran OK' - - assert_match(sha1_re, sha1_out) - sha1 = sha1_re.match(sha1_out)[1] - resp = `curl -isSfN -T#{tmp.path} http://#@addr:#@port/` - assert $?.success?, 'curl ran OK' - assert_match(%r!\b#{sha1}\b!, resp) - assert_match(/sysread_read_byte_match/, resp) - end - - def test_chunked_upload_via_curl - # POSIX doesn't require all of these to be present on a system - which('curl') or return - which('sha1sum') or return - which('dd') or return - - start_server(@sha1_app) - - tmp = Tempfile.new('dd_dest') - assert(system("dd", "if=#{@random.path}", "of=#{tmp.path}", - "bs=#{@bs}", "count=#{@count}"), - "dd #@random to #{tmp}") - sha1_re = %r!\b([a-f0-9]{40})\b! - sha1_out = `sha1sum #{tmp.path}` - assert $?.success?, 'sha1sum ran OK' - - assert_match(sha1_re, sha1_out) - sha1 = sha1_re.match(sha1_out)[1] - cmd = "curl -H 'X-Expect-Size: #{tmp.size}' --tcp-nodelay \ - -isSf --no-buffer -T- " \ - "http://#@addr:#@port/" - resp = Tempfile.new('resp') - resp.sync = true - - rd, wr = IO.pipe - wr.sync = rd.sync = true - pid = fork { - STDIN.reopen(rd) - rd.close - wr.close - STDOUT.reopen(resp) - exec cmd - } - rd.close - - tmp.rewind - @count.times { |i| - wr.write(tmp.read(@bs)) - sleep(rand / 10) if 0 == i % 8 - } - wr.close - pid, status = Process.waitpid2(pid) - - resp.rewind - resp = resp.read - assert status.success?, 'curl ran OK' - assert_match(%r!\b#{sha1}\b!, resp) - assert_match(/sysread_read_byte_match/, resp) - assert_match(/expect_size_match/, resp) - end - - def test_curl_chunked_small - # POSIX doesn't require all of these to be present on a system - which('curl') or return - which('sha1sum') or return - which('dd') or return - - start_server(@sha1_app) - - tmp = Tempfile.new('dd_dest') - # small StringIO path - assert(system("dd", "if=#{@random.path}", "of=#{tmp.path}", - "bs=1024", "count=1"), - "dd #@random to #{tmp}") - sha1_re = %r!\b([a-f0-9]{40})\b! - sha1_out = `sha1sum #{tmp.path}` - assert $?.success?, 'sha1sum ran OK' - - assert_match(sha1_re, sha1_out) - sha1 = sha1_re.match(sha1_out)[1] - resp = `curl -H 'X-Expect-Size: #{tmp.size}' --tcp-nodelay \ - -isSf --no-buffer -T- http://#@addr:#@port/ < #{tmp.path}` - assert $?.success?, 'curl ran OK' - assert_match(%r!\b#{sha1}\b!, resp) - assert_match(/sysread_read_byte_match/, resp) - assert_match(/expect_size_match/, resp) - end - - private - - def length - @bs * @count - end - - def start_server(app) - redirect_test_io do - @server = HttpServer.new(app, :listeners => [ "#{@addr}:#{@port}" ] ) - @server.start - end - end - -end diff --git a/test/unit/test_util.rb b/test/unit/test_util.rb index 9d5d4ef..bc7b233 100644 --- a/test/unit/test_util.rb +++ b/test/unit/test_util.rb @@ -51,7 +51,7 @@ class TestUtil < Test::Unit::TestCase def test_reopen_logs_renamed_with_encoding tmp = Tempfile.new('') tmp_path = tmp.path.dup.freeze - Encoding.list.each { |encoding| + Encoding.list.sample(5).each { |encoding| File.open(tmp_path, "a:#{encoding.to_s}") { |fp| fp.sync = true assert_equal encoding, fp.external_encoding @@ -74,8 +74,9 @@ class TestUtil < Test::Unit::TestCase def test_reopen_logs_renamed_with_internal_encoding tmp = Tempfile.new('') tmp_path = tmp.path.dup.freeze - Encoding.list.each { |ext| - Encoding.list.each { |int| + full = Encoding.list + full.sample(2).each { |ext| + full.sample(2).each { |int| next if ext == int File.open(tmp_path, "a:#{ext.to_s}:#{int.to_s}") { |fp| fp.sync = true @@ -114,7 +115,7 @@ class TestUtil < Test::Unit::TestCase f_getpipe_sz = 1032 IO.pipe do |a, b| a_sz = a.fcntl(f_getpipe_sz) - b_sz = b.fcntl(f_getpipe_sz) + b.fcntl(f_getpipe_sz) assert_kind_of Integer, a_sz r_sz = r.fcntl(f_getpipe_sz) assert_equal Raindrops::PAGE_SIZE, r_sz diff --git a/test/unit/test_waiter.rb b/test/unit/test_waiter.rb new file mode 100644 index 0000000..0995de2 --- /dev/null +++ b/test/unit/test_waiter.rb @@ -0,0 +1,34 @@ +require 'test/unit' +require 'unicorn' +require 'unicorn/select_waiter' +class TestSelectWaiter < Test::Unit::TestCase + + def test_select_timeout # n.b. this is level-triggered + sw = Unicorn::SelectWaiter.new + IO.pipe do |r,w| + sw.get_readers(ready = [], [r], 0) + assert_equal [], ready + w.syswrite '.' + sw.get_readers(ready, [r], 1000) + assert_equal [r], ready + sw.get_readers(ready, [r], 0) + assert_equal [r], ready + end + end + + def test_linux # ugh, also level-triggered, unlikely to change + IO.pipe do |r,w| + wtr = Unicorn::Waiter.prep_readers([r]) + wtr.get_readers(ready = [], [r], 0) + assert_equal [], ready + w.syswrite '.' + wtr.get_readers(ready = [], [r], 1000) + assert_equal [r], ready + wtr.get_readers(ready = [], [r], 1000) + assert_equal [r], ready, 'still ready (level-triggered :<)' + assert_nil wtr.close + end + rescue SystemCallError => e + warn "#{e.message} (#{e.class})" + end if Unicorn.const_defined?(:Waiter) +end diff --git a/unicorn.gemspec b/unicorn.gemspec index ceea831..e7e3ef7 100644 --- a/unicorn.gemspec +++ b/unicorn.gemspec @@ -11,31 +11,31 @@ end.compact Gem::Specification.new do |s| s.name = %q{unicorn} - s.version = (ENV['VERSION'] || '5.5.0').dup + s.version = (ENV['VERSION'] || '6.1.0').dup s.authors = ['unicorn hackers'] s.summary = 'Rack HTTP server for fast clients and Unix' s.description = File.read('README').split("\n\n")[1] - s.email = %q{unicorn-public@bogomips.org} + s.email = %q{unicorn-public@yhbt.net} s.executables = %w(unicorn unicorn_rails) s.extensions = %w(ext/unicorn_http/extconf.rb) s.extra_rdoc_files = IO.readlines('.document').map!(&:chomp!).keep_if do |f| File.exist?(f) end s.files = manifest - s.homepage = 'https://bogomips.org/unicorn/' + s.homepage = 'https://yhbt.net/unicorn/' s.test_files = test_files - # technically we need ">= 1.9.3", too, but avoid the array here since - # old rubygems versions (1.8.23.2 at least) do not support multiple - # version requirements here. - s.required_ruby_version = '< 3.0' + # 2.5.0 is the minimum supported version. We don't specify + # a maximum version to make it easier to test pre-releases, + # but we do warn users if they install unicorn on an untested + # version in extconf.rb + s.required_ruby_version = ">= 2.5.0" # We do not have a hard dependency on rack, it's possible to load # things which respond to #call. HTTP status lines in responses # won't have descriptive text, only the numeric status. s.add_development_dependency(%q<rack>) - s.add_dependency(%q<kgio>, '~> 2.6') s.add_dependency(%q<raindrops>, '~> 0.7') s.add_development_dependency('test-unit', '~> 3.0') |