Options to aid zero-downtime deployment / process supervision

README.md

@@ -196,6 +196,23 @@ See the `examples` directory for example `chef` cookbook and
 `god` config.  In the `chef` cookbook, you can also find example `init.d` and
 `muninrc` templates (all very out of date, pull requests welcome).
 
+Zero-downtime code deploys
+--------------------------
+
+In a production environment you will likely want to manage the daemon using a
+process supervisor like `runit` or `god` or an init system like `systemd` or
+`upstart`.  Example configurations for some of these are included in the
+`examples` directory.  With these systems, `reload` typically sends a `HUP`
+signal, which will reload the configuration but not application code.  The
+simplest way to make workers pick up new code after a deploy is to stop and
+start the daemon.  This will result in a period where new jobs are not being
+processed.  You can avoid this delay by using the `--no-pidfile` and
+`--kill-others` flags.  After new worker code is deployed, start a second
+instance of `resque-pool`.  The second daemon will detect and gracefully shut
+down the first when it is ready to process jobs.  This process uses more memory
+than a simple restart, since two copies of the application code are loaded at
+once.
+
 TODO
 -----
 

examples/upstart-reload.conf

@@ -0,0 +1,25 @@
+# Seamless reload of resque-pool after a deploy.
+# Assuming resque-pool is already running, invoke with 
+# `sudo initctl emit resque-pool-reload`.
+description "resque-pool-reload"
+
+start on resque-pool-reload
+
+task 
+
+limit nofile 65536 65536
+
+# You may need to set things like PATH here.
+env HOME=/home/your_app_user
+export HOME
+
+script
+  source /home/your_app_user/app_env.sh
+  cd /your/app_root
+  /usr/local/bin/setuidgid your_app_user bundle exec resque-pool \
+    --daemon \
+    --no-pidfile \
+    --kill-others \
+    --lock /path/to/your/lock_file \
+    --config /path/to/resque_pool_config.yml
+end script

examples/upstart.conf

@@ -0,0 +1,44 @@
+# Manages resque-pool.
+# Start the process with `initctl start resque-pool`.
+description "resque-pool"
+
+start on virtual-filesystems
+stop on runlevel [06]
+
+respawn
+kill timeout 30
+limit nofile 65536 65536
+
+# You may need to set things like PATH here.
+env HOME=/home/your_app_user
+export HOME
+
+# Ensure no subsequently deployed instances are running after shutdown.
+post-stop script
+  /usr/bin/pkill -INT -f resque-pool-master
+end script
+
+# This script assumes you will deploy new application code by using
+# something similar to upstart-reload.conf which runs a second copy of
+# the application then shuts down the first when ready to fork.  The
+# --lock argument should point to a file that is accessible across
+# deploys (i.e. not under a capistrano versioned path).  setuidgid is
+# only necessary if you are running an older version of upstart such as
+# the one included with RHEL/Centos 6.  In Upstart 1.4 and above you can
+# use the setuid and setgid directives instead.
+script
+  # Assuming you use environment-based configuration.
+  source /home/your_app_user/app_env.sh
+  cd /your/app_root
+  /usr/local/bin/setuidgid your_app_user bundle exec resque-pool \
+    --daemon \
+    --no-pidfile \
+    --kill-others \
+    --lock /path/to/your/lock_file \
+    --config /path/to/resque_pool_config.yml
+  # The above command will return if resque-pool is restarted using the
+  # --kill-others functionality.  This line will block until all pool
+  # instances are terminated, ensuring that upstart doesn't try to
+  # restart our process unless it is actually dead.
+  flock -x 0 < /path/to/your/lock_file
+end script

lib/resque/pool/cli.rb

@@ -12,6 +12,7 @@ module CLI
 
       def run
         opts = parse_options
+        obtain_shared_lock opts[:lock_file]
         daemonize if opts[:daemon]
         manage_pidfile opts[:pidfile]
         redirect opts
@@ -20,9 +21,9 @@ def run
         start_pool
       end
 
-      def parse_options
+      def parse_options(argv=nil)
         opts = {}
-        OptionParser.new do |opt|
+        parser = OptionParser.new do |opt|
           opt.banner = <<-EOS.gsub(/^            /, '')
             resque-pool is the best way to manage a group (pool) of resque workers
 
@@ -41,6 +42,8 @@ def parse_options
           opt.on('-e', '--stderr FILE', "Redirect stderr to logfile") { |c| opts[:stderr] = c }
           opt.on('--nosync', "Don't sync logfiles on every write") { opts[:nosync] = true }
           opt.on("-p", '--pidfile FILE', "PID file location") { |c| opts[:pidfile] = c }
+          opt.on('--no-pidfile', "Force no pidfile, even if daemonized") { opts[:no_pidfile] = true }
+          opt.on('-l', '--lock FILE' "Open a shared lock on a file") { |c| opts[:lock_file] = c }
           opt.on("-E", '--environment ENVIRONMENT', "Set RAILS_ENV/RACK_ENV/RESQUE_ENV") { |c| opts[:environment] = c }
           opt.on("-s", '--spawn-delay MS', Integer, "Delay in milliseconds between spawning missing workers") { |c| opts[:spawn_delay] = c }
           opt.on('--term-graceful-wait', "On TERM signal, wait for workers to shut down gracefully") { opts[:term_graceful_wait] = true }
@@ -49,12 +52,19 @@ def parse_options
           opt.on('--single-process-group', "Workers remain in the same process group as the master") { opts[:single_process_group] = true }
           opt.on("-h", "--help", "Show this.") { puts opt; exit }
           opt.on("-v", "--version", "Show Version"){ puts "resque-pool #{VERSION} (c) nicholas a. evans"; exit}
-        end.parse!
+        end
+        parser.parse!(argv || parser.default_argv)
+        if opts[:pidfile]
+          opts.delete(:no_pidfile)
+        end
         if opts[:daemon]
           opts[:stdout]  ||= "log/resque-pool.stdout.log"
           opts[:stderr]  ||= "log/resque-pool.stderr.log"
-          opts[:pidfile] ||= "tmp/pids/resque-pool.pid"
+          unless opts[:no_pidfile]
+            opts[:pidfile] ||= "tmp/pids/resque-pool.pid"
+          end
         end
+
         opts
       end
 
@@ -66,6 +76,18 @@ def daemonize
         exit unless pid.nil?
       end
 
+      # Obtain a lock on a file that will be held for the lifetime of
+      # the process.  This aids in concurrent daemonized deployment with
+      # process managers like upstart since multiple pools can share a
+      # lock, but not a pidfile.
+      def obtain_shared_lock(lock_path)
+        return unless lock_path
+        @lock_file = File.open(lock_path, 'w')
+        unless @lock_file.flock(File::LOCK_SH)
+          fail "unable to obtain shared lock on #{@lock_file}"
+        end
+      end
+
       def manage_pidfile(pidfile)
         return unless pidfile
         pid = Process.pid

spec/cli_spec.rb

@@ -0,0 +1,51 @@
+require 'spec_helper'
+require 'resque/pool/cli'
+
+describe Resque::Pool::CLI do
+  subject(:cli) { Resque::Pool::CLI }
+
+  describe "option parsing" do
+    it "`--daemon` sets the 'daemon' flag" do
+      options = cli.parse_options(%w[--daemon])
+      options[:daemon].should be_truthy
+    end
+
+    it "`--daemon` redirects stdout and stderr, when none specified" do
+      options = cli.parse_options(%w[--daemon])
+      options[:stdout].should == "log/resque-pool.stdout.log"
+      options[:stderr].should == "log/resque-pool.stderr.log"
+    end
+
+    it "`--daemon` does not override provided stdout/stderr options" do
+      options = cli.parse_options(%w[--daemon --stdout my.stdout --stderr my.stderr])
+      options[:stdout].should == "my.stdout"
+      options[:stderr].should == "my.stderr"
+    end
+
+    it "`--daemon` sets a default pidfile, when none specified" do
+      options = cli.parse_options(%w[--daemon])
+      options[:pidfile].should == "tmp/pids/resque-pool.pid"
+    end
+
+    it "`--daemon` does not override provided pidfile" do
+      options = cli.parse_options(%w[--daemon --pidfile my.pid])
+      options[:pidfile].should == "my.pid"
+    end
+
+    it "`--no-pidfile sets the 'no-pidfile' flag" do
+      options = cli.parse_options(%w[--no-pidfile])
+      options[:no_pidfile].should be_truthy
+    end
+
+    it "`--no-pidfile prevents `--daemon` from setting a default pidfile" do
+      options = cli.parse_options(%w[--daemon --no-pidfile])
+      options[:pidfile].should be_nil
+    end
+
+    it "--no-pidfile does not prevent explicit --pidfile setting" do
+      options = cli.parse_options(%w[--no-pidfile --pidfile my.pid])
+      options[:pidfile].should == "my.pid"
+      options[:no_pidfile].should be_falsey
+    end
+  end
+end