From cb3b9862a5fef4f3fd197e0319bbea0de562f9da Mon Sep 17 00:00:00 2001 From: Evan Weaver Date: Sat, 31 Jan 2009 14:17:06 -0800 Subject: Merge pivotal code. Breaks world. Added option to throttle number of concurrent threads processing requests. Conflicts: bin/mongrel_rails lib/mongrel.rb lib/mongrel/configurator.rb lib/mongrel/rails.rb test/unit/test_ws.rb --- bin/mongrel_rails | 285 ++++++++++++++++++++++++++++++++ lib/mongrel.rb | 76 +++------ lib/mongrel/configurator.rb | 385 ++++++++++++++++++++++++++++++++++++++++++++ lib/mongrel/rails.rb | 180 +++++++++++++++++++++ lib/mongrel/semaphore.rb | 46 ++++++ test/test_suite.rb | 3 + test/unit/test_semaphore.rb | 118 ++++++++++++++ test/unit/test_threading.rb | 82 ++++++++++ test/unit/test_ws.rb | 7 +- 9 files changed, 1126 insertions(+), 56 deletions(-) create mode 100644 bin/mongrel_rails create mode 100644 lib/mongrel/configurator.rb create mode 100644 lib/mongrel/rails.rb create mode 100644 lib/mongrel/semaphore.rb create mode 100644 test/test_suite.rb create mode 100644 test/unit/test_semaphore.rb create mode 100644 test/unit/test_threading.rb diff --git a/bin/mongrel_rails b/bin/mongrel_rails new file mode 100644 index 0000000..c9eac8f --- /dev/null +++ b/bin/mongrel_rails @@ -0,0 +1,285 @@ +#!/usr/bin/env ruby +# +# Copyright (c) 2005 Zed A. Shaw +# You can redistribute it and/or modify it under the same terms as Ruby. +# +# Additional work donated by contributors. See http://mongrel.rubyforge.org/attributions.html +# for more information. + +require 'yaml' +require 'etc' + +$LOAD_PATH.unshift "#{File.dirname(__FILE__)}/../lib" +require 'mongrel' +require 'mongrel/rails' + +Mongrel::Gems.require 'gem_plugin' + +# require 'ruby-debug' +# Debugger.start + +module Mongrel + class Start < GemPlugin::Plugin "/commands" + include Mongrel::Command::Base + + def configure + options [ + ["-e", "--environment ENV", "Rails environment to run as", :@environment, ENV['RAILS_ENV'] || "development"], + ["-d", "--daemonize", "Run daemonized in the background", :@daemon, false], + ['-p', '--port PORT', "Which port to bind to", :@port, 3000], + ['-a', '--address ADDR', "Address to bind to", :@address, "0.0.0.0"], + ['-l', '--log FILE', "Where to write log messages", :@log_file, "log/mongrel.log"], + ['-P', '--pid FILE', "Where to write the PID", :@pid_file, "log/mongrel.pid"], + ['-n', '--num-processors INT', "Number of processors active before clients denied", :@num_processors, 1024], + ['-N', '--num-threads INT', "Maximum number of requests to process concurrently", :@max_concurrent_threads, 1024], + ['-o', '--timeout TIME', "Time to wait (in seconds) before killing a stalled thread", :@timeout, 60], + ['-t', '--throttle TIME', "Time to pause (in hundredths of a second) between accepting clients", :@throttle, 0], + ['-m', '--mime PATH', "A YAML file that lists additional MIME types", :@mime_map, nil], + ['-c', '--chdir PATH', "Change to dir before starting (will be expanded)", :@cwd, Dir.pwd], + ['-r', '--root PATH', "Set the document root (default 'public')", :@docroot, "public"], + ['-B', '--debug', "Enable debugging mode", :@debug, false], + ['-C', '--config PATH', "Use a config file", :@config_file, nil], + ['-S', '--script PATH', "Load the given file as an extra config script", :@config_script, nil], + ['-G', '--generate PATH', "Generate a config file for use with -C", :@generate, nil], + ['', '--user USER', "User to run as", :@user, nil], + ['', '--group GROUP', "Group to run as", :@group, nil], + ['', '--prefix PATH', "URL prefix for Rails app", :@prefix, nil] + ] + end + + def validate + if @config_file + valid_exists?(@config_file, "Config file not there: #@config_file") + return false unless @valid + @config_file = File.expand_path(@config_file) + load_config + return false unless @valid + end + + @cwd = File.expand_path(@cwd) + valid_dir? @cwd, "Invalid path to change to during daemon mode: #@cwd" + + # Change there to start, then we'll have to come back after daemonize + Dir.chdir(@cwd) + + valid?(@prefix[0] == ?/ && @prefix[-1] != ?/, "Prefix must begin with / and not end in /") if @prefix + valid_dir? File.dirname(@log_file), "Path to log file not valid: #@log_file" + valid_dir? File.dirname(@pid_file), "Path to pid file not valid: #@pid_file" + valid_dir? @docroot, "Path to docroot not valid: #@docroot" + valid_exists? @mime_map, "MIME mapping file does not exist: #@mime_map" if @mime_map + valid_exists? @config_file, "Config file not there: #@config_file" if @config_file + valid_dir? File.dirname(@generate), "Problem accessing directory to #@generate" if @generate + valid_user? @user if @user + valid_group? @group if @group + + return @valid + end + + def run + if @generate + @generate = File.expand_path(@generate) + STDERR.puts "** Writing config to \"#@generate\"." + open(@generate, "w") {|f| f.write(settings.to_yaml) } + STDERR.puts "** Finished. Run \"mongrel_rails start -C #@generate\" to use the config file." + exit 0 + end + + config = Mongrel::Rails::RailsConfigurator.new(settings) do + if defaults[:daemon] + if File.exist? defaults[:pid_file] + log "!!! PID file #{defaults[:pid_file]} already exists. Mongrel could be running already. Check your #{defaults[:log_file]} for errors." + log "!!! Exiting with error. You must stop mongrel and clear the .pid before I'll attempt a start." + exit 1 + end + + daemonize + write_pid_file + log "Daemonized, any open files are closed. Look at #{defaults[:pid_file]} and #{defaults[:log_file]} for info." + log "Settings loaded from #{@config_file} (they override command line)." if @config_file + end + + log "Starting Mongrel listening at #{defaults[:host]}:#{defaults[:port]}" + + listener do + mime = {} + if defaults[:mime_map] + log "Loading additional MIME types from #{defaults[:mime_map]}" + mime = load_mime_map(defaults[:mime_map], mime) + end + + if defaults[:debug] + log "Installing debugging prefixed filters. Look in log/mongrel_debug for the files." + debug "/" + end + + log "Starting Rails with #{defaults[:environment]} environment..." + log "Mounting Rails at #{defaults[:prefix]}..." if defaults[:prefix] + uri defaults[:prefix] || "/", :handler => rails(:mime => mime, :prefix => defaults[:prefix]) + log "Rails loaded." + + log "Loading any Rails specific GemPlugins" + load_plugins + + if defaults[:config_script] + log "Loading #{defaults[:config_script]} external config script" + run_config(defaults[:config_script]) + end + + setup_rails_signals + end + end + + config.run + config.log "Mongrel #{Mongrel::Const::MONGREL_VERSION} available at #{@address}:#{@port}" + + unless config.defaults[:daemon] + config.log "Use CTRL-C to stop." + end + + config.join + + if config.needs_restart + if RUBY_PLATFORM !~ /djgpp|(cyg|ms|bcc)win|mingw/ + cmd = "ruby #{__FILE__} start #{original_args.join(' ')}" + config.log "Restarting with arguments: #{cmd}" + config.stop(false, true) + config.remove_pid_file + + if config.defaults[:daemon] + system cmd + else + STDERR.puts "Can't restart unless in daemon mode." + exit 1 + end + else + config.log "Win32 does not support restarts. Exiting." + end + end + end + + def load_config + settings = {} + begin + settings = YAML.load_file(@config_file) + ensure + STDERR.puts "** Loading settings from #{@config_file} (they override command line)." unless @daemon || settings[:daemon] + end + + settings[:includes] ||= ["mongrel"] + + # Config file settings will override command line settings + settings.each do |key, value| + key = key.to_s + if config_keys.include?(key) + key = 'address' if key == 'host' + self.instance_variable_set("@#{key}", value) + else + failure "Unknown configuration setting: #{key}" + @valid = false + end + end + end + + def config_keys + @config_keys ||= + %w(address host port cwd log_file pid_file environment docroot mime_map daemon debug includes config_script num_processors timeout throttle user group prefix max_concurrent_threads) + end + + def settings + config_keys.inject({}) do |hash, key| + value = self.instance_variable_get("@#{key}") + key = 'host' if key == 'address' + hash[key.to_sym] ||= value + hash + end + end + end + + def Mongrel::send_signal(signal, pid_file) + pid = File.read(pid_file).to_i + print "Sending #{signal} to Mongrel at PID #{pid}..." + begin + Process.kill(signal, pid) + rescue Errno::ESRCH + puts "Process does not exist. Not running." + end + + puts "Done." + end + + + class Stop < GemPlugin::Plugin "/commands" + include Mongrel::Command::Base + + def configure + options [ + ['-c', '--chdir PATH', "Change to dir before starting (will be expanded).", :@cwd, "."], + ['-f', '--force', "Force the shutdown (kill -9).", :@force, false], + ['-w', '--wait SECONDS', "Wait SECONDS before forcing shutdown", :@wait, "0"], + ['-P', '--pid FILE', "Where the PID file is located (cannot be changed via soft restart).", :@pid_file, "log/mongrel.pid"] + ] + end + + def validate + @cwd = File.expand_path(@cwd) + valid_dir? @cwd, "Invalid path to change to during daemon mode: #@cwd" + + Dir.chdir @cwd + + valid_exists? @pid_file, "PID file #@pid_file does not exist. Not running?" + return @valid + end + + def run + if @force + @wait.to_i.times do |waiting| + exit(0) if not File.exist? @pid_file + sleep 1 + end + + Mongrel::send_signal("KILL", @pid_file) if File.exist? @pid_file + else + Mongrel::send_signal("TERM", @pid_file) + end + end + end + + + class Restart < GemPlugin::Plugin "/commands" + include Mongrel::Command::Base + + def configure + options [ + ['-c', '--chdir PATH', "Change to dir before starting (will be expanded)", :@cwd, '.'], + ['-s', '--soft', "Do a soft restart rather than a process exit restart", :@soft, false], + ['-P', '--pid FILE', "Where the PID file is located", :@pid_file, "log/mongrel.pid"] + ] + end + + def validate + @cwd = File.expand_path(@cwd) + valid_dir? @cwd, "Invalid path to change to during daemon mode: #@cwd" + + Dir.chdir @cwd + + valid_exists? @pid_file, "PID file #@pid_file does not exist. Not running?" + return @valid + end + + def run + if @soft + Mongrel::send_signal("HUP", @pid_file) + else + Mongrel::send_signal("USR2", @pid_file) + end + end + end +end + + +GemPlugin::Manager.instance.load "mongrel" => GemPlugin::INCLUDE, "rails" => GemPlugin::EXCLUDE + + +if not Mongrel::Command::Registry.instance.run ARGV + exit 1 +end diff --git a/lib/mongrel.rb b/lib/mongrel.rb index a97972b..424b7f0 100644 --- a/lib/mongrel.rb +++ b/lib/mongrel.rb @@ -26,6 +26,7 @@ require 'mongrel/const' require 'mongrel/http_request' require 'mongrel/header_out' require 'mongrel/http_response' +require 'mongrel/semaphore' # Mongrel module containing all of the classes (include C extensions) for running # a Mongrel web server. It contains a minimalist HTTP server with just enough @@ -79,13 +80,20 @@ module Mongrel attr_reader :port attr_reader :throttle attr_reader :timeout - attr_reader :num_processors + attr_reader :max_queued_threads + + DEFAULTS = { + :max_queued_threads => 20, + :max_concurrent_threads => 20, + :throttle => 0, + :timeout => 60 + } # Creates a working server on host:port (strange things happen if port isn't a Number). # Use HttpServer::run to start the server and HttpServer.acceptor.join to # join the thread that's processing incoming requests on the socket. # - # The num_processors optional argument is the maximum number of concurrent + # The max_queued_threads optional argument is the maximum number of concurrent # processors to accept, anything over this is closed immediately to maintain # server processing performance. This may seem mean but it is the most efficient # way to deal with overload. Other schemes involve still parsing the client's request @@ -94,20 +102,21 @@ module Mongrel # The throttle parameter is a sleep timeout (in hundredths of a second) that is placed between # socket.accept calls in order to give the server a cheap throttle time. It defaults to 0 and # actually if it is 0 then the sleep is not done at all. - def initialize(host, port, app, opts = {}) + def initialize(host, port, app, options = {}) + options = DEFAULTS.merge(options) + tries = 0 @socket = TCPServer.new(host, port) if defined?(Fcntl::FD_CLOEXEC) @socket.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC) end - @host = host - @port = port + @host, @port, @app = host, port, app @workers = ThreadGroup.new - # Set default opts - @app = app - @num_processors = opts.delete(:num_processors) || 950 - @throttle = (opts.delete(:throttle) || 0) / 100 - @timeout = opts.delete(:timeout) || 60 + + @throttle = options[:throttle] / 100.0 + @timeout = options[:timeout] + @max_queued_threads = options[:max_queued_threads] + @max_concurrent_threads = options[:max_concurrent_threads] end # Does the majority of the IO processing. It has been written in Ruby using @@ -245,6 +254,7 @@ module Mongrel # Runs the thing. It returns the thread used so you can "join" it. You can also # access the HttpServer::acceptor attribute to get the thread later. def start! + semaphore = Semaphore.new(@max_concurrent_threads) BasicSocket.do_not_reverse_lookup=true configure_socket_options @@ -264,12 +274,12 @@ module Mongrel end worker_list = @workers.list - if worker_list.length >= @num_processors - Mongrel.logger.error "Server overloaded with #{worker_list.length} processors (#@num_processors max). Dropping connection." + if worker_list.length >= @max_queued_threads + Mongrel.logger.error "Server overloaded with #{worker_list.length} processors (#@max_queued_threads max). Dropping connection." client.close rescue nil reap_dead_workers("max processors") else - thread = Thread.new(client) {|c| process_client(c) } + thread = Thread.new(client) {|c| semaphore.synchronize { process_client(c) } } thread[:started_on] = Time.now @workers.add(thread) @@ -298,49 +308,11 @@ module Mongrel return @acceptor end - # Simply registers a handler with the internal URIClassifier. When the URI is - # found in the prefix of a request then your handler's HttpHandler::process method - # is called. See Mongrel::URIClassifier#register for more information. - # - # If you set in_front=true then the passed in handler will be put in the front of the list - # for that particular URI. Otherwise it's placed at the end of the list. - def register(uri, handler, in_front=false) - begin - @classifier.register(uri, [handler]) - rescue URIClassifier::RegistrationError => e - handlers = @classifier.resolve(uri)[2] - if handlers - # Already registered - method_name = in_front ? 'unshift' : 'push' - handlers.send(method_name, handler) - else - raise - end - end - handler.listener = self - end - - # Removes any handlers registered at the given URI. See Mongrel::URIClassifier#unregister - # for more information. Remember this removes them *all* so the entire - # processing chain goes away. - def unregister(uri) - @classifier.unregister(uri) - end - # Stops the acceptor thread and then causes the worker threads to finish # off the request queue before finally exiting. def stop(synchronous=false) @acceptor.raise(StopServer.new) - - if synchronous - sleep(0.5) while @acceptor.alive? - end + (sleep(0.5) while @acceptor.alive?) if synchronous end - end end - -# Load experimental library, if present. We put it here so it can override anything -# in regular Mongrel. - -$LOAD_PATH.unshift 'projects/mongrel_experimental/lib/' diff --git a/lib/mongrel/configurator.rb b/lib/mongrel/configurator.rb new file mode 100644 index 0000000..2442152 --- /dev/null +++ b/lib/mongrel/configurator.rb @@ -0,0 +1,385 @@ +require 'yaml' +require 'etc' + +module Mongrel + # Implements a simple DSL for configuring a Mongrel server for your + # purposes. More used by framework implementers to setup Mongrel + # how they like, but could be used by regular folks to add more things + # to an existing mongrel configuration. + # + # It is used like this: + # + # require 'mongrel' + # config = Mongrel::Configurator.new :host => "127.0.0.1" do + # listener :port => 3000 do + # uri "/app", :handler => Mongrel::DirHandler.new(".", load_mime_map("mime.yaml")) + # end + # run + # end + # + # This will setup a simple DirHandler at the current directory and load additional + # mime types from mimy.yaml. The :host => "127.0.0.1" is actually not + # specific to the servers but just a hash of default parameters that all + # server or uri calls receive. + # + # When you are inside the block after Mongrel::Configurator.new you can simply + # call functions that are part of Configurator (like server, uri, daemonize, etc) + # without having to refer to anything else. You can also call these functions on + # the resulting object directly for additional configuration. + # + # A major thing about Configurator is that it actually lets you configure + # multiple listeners for any hosts and ports you want. These are kept in a + # map config.listeners so you can get to them. + # + # * :pid_file => Where to write the process ID. + class Configurator + attr_reader :listeners + attr_reader :defaults + attr_reader :needs_restart + + # You pass in initial defaults and then a block to continue configuring. + def initialize(defaults={}, &block) + @listener = nil + @listener_name = nil + @listeners = {} + @defaults = defaults + @needs_restart = false + @pid_file = defaults[:pid_file] + + if block + cloaker(&block).bind(self).call + end + end + + # Change privileges of the process to specified user and group. + def change_privilege(user, group) + begin + uid, gid = Process.euid, Process.egid + target_uid = Etc.getpwnam(user).uid if user + target_gid = Etc.getgrnam(group).gid if group + + if uid != target_uid or gid != target_gid + log "Initiating groups for #{user.inspect}:#{group.inspect}." + Process.initgroups(user, target_gid) + + log "Changing group to #{group.inspect}." + Process::GID.change_privilege(target_gid) + + log "Changing user to #{user.inspect}." + Process::UID.change_privilege(target_uid) + end + rescue Errno::EPERM => e + log "Couldn't change user and group to #{user.inspect}:#{group.inspect}: #{e.to_s}." + log "Mongrel failed to start." + exit 1 + end + end + + def remove_pid_file + File.unlink(@pid_file) if @pid_file and File.exists?(@pid_file) + end + + # Writes the PID file if we're not on Windows. + def write_pid_file + if RUBY_PLATFORM !~ /djgpp|(cyg|ms|bcc)win|mingw/ + log "Writing PID file to #{@pid_file}" + open(@pid_file,"w") {|f| f.write(Process.pid) } + open(@pid_file,"w") do |f| + f.write(Process.pid) + File.chmod(0644, @pid_file) + end + end + end + + # Generates a class for cloaking the current self and making the DSL nicer. + def cloaking_class + class << self + self + end + end + + # Do not call this. You were warned. + def cloaker(&block) + cloaking_class.class_eval do + define_method :cloaker_, &block + meth = instance_method( :cloaker_ ) + remove_method :cloaker_ + meth + end + end + + # This will resolve the given options against the defaults. + # Normally just used internally. + def resolve_defaults(options) + options.merge(@defaults) + end + + # Starts a listener block. This is the only one that actually takes + # a block and then you make Configurator.uri calls in order to setup + # your URIs and handlers. If you write your Handlers as GemPlugins + # then you can use load_plugins and plugin to load them. + # + # It expects the following options (or defaults): + # + # * :host => Host name to bind. + # * :port => Port to bind. + # * :max_queued_threads => The maximum number of concurrent threads allowed. + # * :throttle => Time to pause (in hundredths of a second) between accepting clients. + # * :timeout => Time to wait (in seconds) before killing a stalled thread. + # * :user => User to change to, must have :group as well. + # * :group => Group to change to, must have :user as well. + # + def listener(options={},&block) + raise "Cannot call listener inside another listener block." if (@listener or @listener_name) + ops = resolve_defaults(options) + + @listener = Mongrel::HttpServer.new(ops[:host], ops[:port].to_i, ops) + @listener_name = "#{ops[:host]}:#{ops[:port]}" + @listeners[@listener_name] = @listener + + if ops[:user] and ops[:group] + change_privilege(ops[:user], ops[:group]) + end + + # Does the actual cloaking operation to give the new implicit self. + if block + cloaker(&block).bind(self).call + end + + # all done processing this listener setup, reset implicit variables + @listener = nil + @listener_name = nil + end + + + # Called inside a Configurator.listener block in order to + # add URI->handler mappings for that listener. Use this as + # many times as you like. It expects the following options + # or defaults: + # + # * :handler => HttpHandler -- Handler to use for this location. + # * :in_front => true/false -- Rather than appending, it prepends this handler. + def uri(location, options={}) + ops = resolve_defaults(options) + @listener.register(location, ops[:handler], ops[:in_front]) + end + + + # Daemonizes the current Ruby script turning all the + # listeners into an actual "server" or detached process. + # You must call this *before* frameworks that open files + # as otherwise the files will be closed by this function. + # + # Does not work for Win32 systems (the call is silently ignored). + # + # Requires the following options or defaults: + # + # * :cwd => Directory to change to. + # * :log_file => Where to write STDOUT and STDERR. + # + # It is safe to call this on win32 as it will only require the daemons + # gem/library if NOT win32. + def daemonize(options={}) + ops = resolve_defaults(options) + # save this for later since daemonize will hose it + if RUBY_PLATFORM !~ /djgpp|(cyg|ms|bcc)win|mingw/ + require 'daemons/daemonize' + + logfile = ops[:log_file] + if logfile[0].chr != "/" + logfile = File.join(ops[:cwd],logfile) + if not File.exist?(File.dirname(logfile)) + log "!!! Log file directory not found at full path #{File.dirname(logfile)}. Update your configuration to use a full path." + exit 1 + end + end + + Daemonize.daemonize(logfile) + + # change back to the original starting directory + Dir.chdir(ops[:cwd]) + + else + log "WARNING: Win32 does not support daemon mode." + end + end + + + # Uses the GemPlugin system to easily load plugins based on their + # gem dependencies. You pass in either an :includes => [] or + # :excludes => [] setting listing the names of plugins to include + # or exclude from the determining the dependencies. + def load_plugins(options={}) + ops = resolve_defaults(options) + + load_settings = {} + if ops[:includes] + ops[:includes].each do |plugin| + load_settings[plugin] = GemPlugin::INCLUDE + end + end + + if ops[:excludes] + ops[:excludes].each do |plugin| + load_settings[plugin] = GemPlugin::EXCLUDE + end + end + + GemPlugin::Manager.instance.load(load_settings) + end + + + # Easy way to load a YAML file and apply default settings. + def load_yaml(file, default={}) + default.merge(YAML.load_file(file)) + end + + + # Loads the MIME map file and checks that it is correct + # on loading. This is commonly passed to Mongrel::DirHandler + # or any framework handler that uses DirHandler to serve files. + # You can also include a set of default MIME types as additional + # settings. See Mongrel::DirHandler for how the MIME types map + # is organized. + def load_mime_map(file, mime={}) + # configure any requested mime map + mime = load_yaml(file, mime) + + # check all the mime types to make sure they are the right format + mime.each {|k,v| log "WARNING: MIME type #{k} must start with '.'" if k.index(".") != 0 } + + return mime + end + + + # Loads and creates a plugin for you based on the given + # name and configured with the selected options. The options + # are merged with the defaults prior to passing them in. + def plugin(name, options={}) + ops = resolve_defaults(options) + GemPlugin::Manager.instance.create(name, ops) + end + + # Lets you do redirects easily as described in Mongrel::RedirectHandler. + # You use it inside the configurator like this: + # + # redirect("/test", "/to/there") # simple + # redirect("/to", /t/, 'w') # regexp + # redirect("/hey", /(w+)/) {|match| ...} # block + # + def redirect(from, pattern, replacement = nil, &block) + uri from, :handler => Mongrel::RedirectHandler.new(pattern, replacement, &block) + end + + # Works like a meta run method which goes through all the + # configured listeners. Use the Configurator.join method + # to prevent Ruby from exiting until each one is done. + def run + @listeners.each {|name,s| + s.run + } + + $mongrel_sleeper_thread = Thread.new { loop { sleep 1 } } + end + + # Calls .stop on all the configured listeners so they + # stop processing requests (gracefully). By default it + # assumes that you don't want to restart. + def stop(needs_restart=false, synchronous=false) + @listeners.each do |name,s| + s.stop(synchronous) + end + @needs_restart = needs_restart + end + + + # This method should actually be called *outside* of the + # Configurator block so that you can control it. In other words + # do it like: config.join. + def join + @listeners.values.each {|s| s.acceptor.join } + end + + + # Calling this before you register your URIs to the given location + # will setup a set of handlers that log open files, objects, and the + # parameters for each request. This helps you track common problems + # found in Rails applications that are either slow or become unresponsive + # after a little while. + # + # You can pass an extra parameter *what* to indicate what you want to + # debug. For example, if you just want to dump rails stuff then do: + # + # debug "/", what = [:rails] + # + # And it will only produce the log/mongrel_debug/rails.log file. + # Available options are: :access, :files, :objects, :threads, :rails + # + # NOTE: Use [:files] to get accesses dumped to stderr like with WEBrick. + def debug(location, what = [:access, :files, :objects, :threads, :rails]) + require 'mongrel/debug' + handlers = { + :access => "/handlers/requestlog::access", + :files => "/handlers/requestlog::files", + :objects => "/handlers/requestlog::objects", + :threads => "/handlers/requestlog::threads", + :rails => "/handlers/requestlog::params" + } + + # turn on the debugging infrastructure, and ObjectTracker is a pig + MongrelDbg.configure + + # now we roll through each requested debug type, turn it on and load that plugin + what.each do |type| + MongrelDbg.begin_trace type + uri location, :handler => plugin(handlers[type]) + end + end + + # Used to allow you to let users specify their own configurations + # inside your Configurator setup. You pass it a script name and + # it reads it in and does an eval on the contents passing in the right + # binding so they can put their own Configurator statements. + def run_config(script) + open(script) {|f| eval(f.read, proc {self}) } + end + + # Sets up the standard signal handlers that are used on most Ruby + # It only configures if the platform is not win32 and doesn't do + # a HUP signal since this is typically framework specific. + # + # Requires a :pid_file option given to Configurator.new to indicate a file to delete. + # It sets the MongrelConfig.needs_restart attribute if + # the start command should reload. It's up to you to detect this + # and do whatever is needed for a "restart". + # + # This command is safely ignored if the platform is win32 (with a warning) + def setup_signals(options={}) + ops = resolve_defaults(options) + + # forced shutdown, even if previously restarted (actually just like TERM but for CTRL-C) + trap("INT") { log "INT signal received."; stop(false) } + + # clean up the pid file always + at_exit { remove_pid_file } + + if RUBY_PLATFORM !~ /djgpp|(cyg|ms|bcc)win|mingw/ + # graceful shutdown + trap("TERM") { log "TERM signal received."; stop } + trap("USR1") { log "USR1 received, toggling $mongrel_debug_client to #{!$mongrel_debug_client}"; $mongrel_debug_client = !$mongrel_debug_client } + # restart + trap("USR2") { log "USR2 signal received."; stop(true) } + + log "Signals ready. TERM => stop. USR2 => restart. INT => stop (no restart)." + else + log "Signals ready. INT => stop (no restart)." + end + end + + # Logs a simple message to STDERR (or the mongrel log if in daemon mode). + def log(msg) + STDERR.print "** ", msg, "\n" + end + + end +end diff --git a/lib/mongrel/rails.rb b/lib/mongrel/rails.rb new file mode 100644 index 0000000..2cc5e41 --- /dev/null +++ b/lib/mongrel/rails.rb @@ -0,0 +1,180 @@ +# Copyright (c) 2005 Zed A. Shaw +# You can redistribute it and/or modify it under the same terms as Ruby. +# +# Additional work donated by contributors. See http://mongrel.rubyforge.org/attributions.html +# for more information. + +require 'mongrel' +require 'cgi' + + +module Mongrel + module Rails + # Implements a handler that can run Rails and serve files out of the + # Rails application's public directory. This lets you run your Rails + # application with Mongrel during development and testing, then use it + # also in production behind a server that's better at serving the + # static files. + # + # The RailsHandler takes a mime_map parameter which is a simple suffix=mimetype + # mapping that it should add to the list of valid mime types. + # + # It also supports page caching directly and will try to resolve a request + # in the following order: + # + # * If the requested exact PATH_INFO exists as a file then serve it. + # * If it exists at PATH_INFO+".html" exists then serve that. + # * Finally, construct a Mongrel::CGIWrapper and run Dispatcher.dispatch to have Rails go. + # + # This means that if you are using page caching it will actually work with Mongrel + # and you should see a decent speed boost (but not as fast as if you use a static + # server like Apache or Litespeed). + class RailsHandler < Mongrel::HttpHandler + attr_reader :files + attr_reader :guard + @@file_only_methods = ["GET","HEAD"] + + def initialize(dir, mime_map = {}) + @files = Mongrel::DirHandler.new(dir,false) + @guard = Mutex.new + + # Register the requested MIME types + mime_map.each {|k,v| Mongrel::DirHandler::add_mime_type(k,v) } + end + + # Attempts to resolve the request as follows: + # + # * If the requested exact PATH_INFO exists as a file then serve it. + # * If it exists at PATH_INFO+".html" exists then serve that. + # * Finally, construct a Mongrel::CGIWrapper and run Dispatcher.dispatch to have Rails go. + def process(request, response) + return if response.socket.closed? + + path_info = request.params[Mongrel::Const::PATH_INFO] + rest_operator = request.params[Mongrel::Const::REQUEST_URI][/^#{Regexp.escape path_info}(;[^\?]+)/, 1].to_s + path_info.chomp!("/") + + page_cached = path_info + rest_operator + ActionController::Base.page_cache_extension + get_or_head = @@file_only_methods.include? request.params[Mongrel::Const::REQUEST_METHOD] + + if get_or_head and @files.can_serve(path_info) + # File exists as-is so serve it up + @files.process(request,response) + elsif get_or_head and @files.can_serve(page_cached) + # Possible cached page, serve it up + request.params[Mongrel::Const::PATH_INFO] = page_cached + @files.process(request,response) + else + begin + cgi = Mongrel::CGIWrapper.new(request, response) + # We don't want the output to be really final until the dispatch returns a response. + cgi.default_really_final = false + + Dispatcher.dispatch(cgi, ActionController::CgiRequest::DEFAULT_SESSION_OPTIONS, response.body) + + # This finalizes the output using the proper HttpResponse way + cgi.out("text/html",true) {""} + rescue Errno::EPIPE + response.socket.close + rescue Object => rails_error + STDERR.puts "#{Time.now}: Error calling Dispatcher.dispatch #{rails_error.inspect}" + STDERR.puts rails_error.backtrace.join("\n") + end + end + end + + # Does the internal reload for Rails. It might work for most cases, but + # sometimes you get exceptions. In that case just do a real restart. + def reload! + begin + @guard.synchronize { + $".replace $orig_dollar_quote + GC.start + Dispatcher.reset_application! + ActionController::Routing::Routes.reload + } + end + end + end + + # Creates Rails specific configuration options for people to use + # instead of the base Configurator. + class RailsConfigurator < Mongrel::Configurator + + # Creates a single rails handler and returns it so you + # can add it to a URI. You can actually attach it to + # as many URIs as you want, but this returns the + # same RailsHandler for each call. + # + # Requires the following options: + # + # * :docroot => The public dir to serve from. + # * :environment => Rails environment to use. + # * :cwd => The change to working directory + # + # And understands the following optional settings: + # + # * :mime => A map of mime types. + # + # Because of how Rails is designed you can only have + # one installed per Ruby interpreter (talk to them + # about thread safety). Because of this the first + # time you call this function it does all the config + # needed to get your Rails working. After that + # it returns the one handler you've configured. + # This lets you attach Rails to any URI(s) you want, + # but it still protects you from threads destroying + # your handler. + def rails(options={}) + + return @rails_handler if @rails_handler + + ops = resolve_defaults(options) + + # fix up some defaults + ops[:environment] ||= "development" + ops[:docroot] ||= "public" + ops[:mime] ||= {} + + $orig_dollar_quote = $".clone + ENV['RAILS_ENV'] = ops[:environment] + env_location = "#{ops[:cwd]}/config/environment" + require env_location + require 'dispatcher' + require 'mongrel/rails' + + ActionController::AbstractRequest.relative_url_root = ops[:prefix] if ops[:prefix] + + @rails_handler = RailsHandler.new(ops[:docroot], ops[:mime]) + end + + # Reloads Rails. This isn't too reliable really, but it + # should work for most minimal reload purposes. The only reliable + # way to reload properly is to stop and then start the process. + def reload! + if not @rails_handler + raise "Rails was not configured. Read the docs for RailsConfigurator." + end + + log "Reloading Rails..." + @rails_handler.reload! + log "Done reloading Rails." + + end + + # Takes the exact same configuration as Mongrel::Configurator (and actually calls that) + # but sets up the additional HUP handler to call reload!. + def setup_rails_signals(options={}) + ops = resolve_defaults(options) + setup_signals(options) + + if RUBY_PLATFORM !~ /djgpp|(cyg|ms|bcc)win|mingw/ + # rails reload + trap("HUP") { log "HUP signal received."; reload! } + + log "Rails signals registered. HUP => reload (without restart). It might not work well." + end + end + end + end +end diff --git a/lib/mongrel/semaphore.rb b/lib/mongrel/semaphore.rb new file mode 100644 index 0000000..1c0b87c --- /dev/null +++ b/lib/mongrel/semaphore.rb @@ -0,0 +1,46 @@ +class Semaphore + def initialize(resource_count = 0) + @available_resource_count = resource_count + @mutex = Mutex.new + @waiting_threads = [] + end + + def wait + make_thread_wait unless resource_is_available + end + + def signal + schedule_waiting_thread if thread_is_waiting + end + + def synchronize + self.wait + yield + ensure + self.signal + end + + private + + def resource_is_available + @mutex.synchronize do + return (@available_resource_count -= 1) >= 0 + end + end + + def make_thread_wait + @waiting_threads << Thread.current + Thread.stop + end + + def thread_is_waiting + @mutex.synchronize do + return (@available_resource_count += 1) <= 0 + end + end + + def schedule_waiting_thread + thread = @waiting_threads.shift + thread.wakeup if thread + end +end diff --git a/test/test_suite.rb b/test/test_suite.rb new file mode 100644 index 0000000..e3bb0dc --- /dev/null +++ b/test/test_suite.rb @@ -0,0 +1,3 @@ +Dir.glob('test/unit/*').select { |path| path =~ /^test\/unit\/test_.*\.rb$/ }.each do |test_path| + require test_path +end diff --git a/test/unit/test_semaphore.rb b/test/unit/test_semaphore.rb new file mode 100644 index 0000000..5ce70f7 --- /dev/null +++ b/test/unit/test_semaphore.rb @@ -0,0 +1,118 @@ +root_dir = File.join(File.dirname(__FILE__), "../..") +require File.join(root_dir, "test/test_helper") +require File.join(root_dir, "lib/mongrel/semaphore") + +class TestSemaphore < Test::Unit::TestCase + def setup + super + + @semaphore = Semaphore.new + end + + def test_wait_prevents_thread_from_running + thread = Thread.new { @semaphore.wait } + give_up_my_time_slice + + assert thread.stop? + end + + def test_signal_allows_waiting_thread_to_run + ran = false + thread = Thread.new { @semaphore.wait; ran = true } + give_up_my_time_slice + + @semaphore.signal + give_up_my_time_slice + + assert ran + end + + def test_wait_allows_only_specified_number_of_resources + @semaphore = Semaphore.new(1) + + run_count = 0 + thread1 = Thread.new { @semaphore.wait; run_count += 1 } + thread2 = Thread.new { @semaphore.wait; run_count += 1 } + give_up_my_time_slice + + assert_equal 1, run_count + end + + def test_semaphore_serializes_threads + @semaphore = Semaphore.new(1) + + result = "" + thread1 = Thread.new do + @semaphore.wait + 4.times do |i| + give_up_my_time_slice + result << i.to_s + end + @semaphore.signal + end + + thread2 = Thread.new do + @semaphore.wait + ("a".."d").each do |char| + give_up_my_time_slice + result << char + end + @semaphore.signal + end + + give_up_my_time_slice + @semaphore.wait + + assert_equal "0123abcd", result + end + + def test_synchronize_many_threads + @semaphore = Semaphore.new(1) + + result = [] + 5.times do |i| + Thread.new do + @semaphore.wait + 2.times { |j| result << [i, j] } + @semaphore.signal + end + end + + give_up_my_time_slice + @semaphore.wait + + 5.times do |i| + 2.times do |j| + assert_equal i, result[2 * i + j][0] + assert_equal j, result[2 * i + j][1] + end + end + end + + def test_synchronize_ensures_signal + @semaphore = Semaphore.new(1) + threads = [] + run_count = 0 + threads << Thread.new do + @semaphore.synchronize { run_count += 1 } + end + threads << Thread.new do + @semaphore.synchronize { run_count += 1; raise "I'm throwing an error." } + end + threads << Thread.new do + @semaphore.synchronize { run_count += 1 } + end + + give_up_my_time_slice + @semaphore.wait + + assert !threads.any? { |thread| thread.alive? } + assert_equal 3, run_count + end + + private + + def give_up_my_time_slice + sleep(0) + end +end \ No newline at end of file diff --git a/test/unit/test_threading.rb b/test/unit/test_threading.rb new file mode 100644 index 0000000..e577bbb --- /dev/null +++ b/test/unit/test_threading.rb @@ -0,0 +1,82 @@ +root_dir = File.join(File.dirname(__FILE__), "../..") +require File.join(root_dir, "test/test_helper") + +include Mongrel + +class FakeHandler < Mongrel::HttpHandler + @@concurrent_threads = 0 + @@max_concurrent_threads = 0 + + def self.max_concurrent_threads + @@max_concurrent_threads ||= 0 + end + + def initialize + super + @@mutex = Mutex.new + end + + def process(request, response) + @@mutex.synchronize do + @@concurrent_threads += 1 # !!! same for += and -= + @@max_concurrent_threads = [@@concurrent_threads, @@max_concurrent_threads].max + end + + sleep(0.1) + response.socket.write("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n\r\nhello!\n") + ensure + @@mutex.synchronize { @@concurrent_threads -= 1 } + end +end + +class ThreadingTest < Test::Unit::TestCase + def setup + @valid_request = "GET / HTTP/1.1\r\nHost: www.google.com\r\nContent-Type: text/plain\r\n\r\n" + @port = process_based_port + + @max_concurrent_threads = 4 + redirect_test_io do + @server = HttpServer.new("127.0.0.1", @port, :max_concurrent_threads => @max_concurrent_threads) + end + + @server.register("/test", FakeHandler.new) + redirect_test_io do + @server.run + end + end + + def teardown + redirect_test_io do + @server.stop(true) + end + end + + def test_server_respects_max_current_threads_option + threads = [] + (@max_concurrent_threads * 3).times do + threads << Thread.new do + send_data_over_socket("GET /test HTTP/1.1\r\nHost: localhost\r\nContent-Type: text/plain\r\n\r\n") + end + end + while threads.any? { |thread| thread.alive? } + sleep(0) + end + assert_equal @max_concurrent_threads, FakeHandler.max_concurrent_threads + end + + private + + def send_data_over_socket(string) + socket = TCPSocket.new("127.0.0.1", @port) + request = StringIO.new(string) + + while data = request.read(8) + socket.write(data) + socket.flush + sleep(0) + end + sleep(0) + socket.write(" ") # Some platforms only raise the exception on attempted write + socket.flush + end +end \ No newline at end of file diff --git a/test/unit/test_ws.rb b/test/unit/test_ws.rb index 7508c7f..2510c3a 100644 --- a/test/unit/test_ws.rb +++ b/test/unit/test_ws.rb @@ -27,9 +27,8 @@ class WebServerTest < Test::Unit::TestCase @tester = TestHandler.new @app = Rack::URLMap.new('/test' => @tester) redirect_test_io do - # We set num_processors=1 so that we can test the reaping code - @server = HttpServer.new("127.0.0.1", @port, @app, :num_processors => 1) - @server.start! + # We set max_queued_threads=1 so that we can test the reaping code + @server = HttpServer.new("127.0.0.1", @port, @app, :max_queued_threads => 1) end end @@ -90,7 +89,7 @@ class WebServerTest < Test::Unit::TestCase end end - def test_num_processors_overload + def test_max_queued_threads_overload redirect_test_io do assert_raises Errno::ECONNRESET, Errno::EPIPE, Errno::ECONNABORTED, Errno::EINVAL, IOError do tests = [ -- cgit v1.2.3-24-ge0c7