feat(dirty): add TTIN/TTOU signal support for dynamic worker scaling

Add support for SIGTTIN and SIGTTOU signals to the dirty arbiter,
allowing dynamic scaling of dirty workers at runtime without restarting
gunicorn.

Changes:
- Add TTIN/TTOU to DirtyArbiter.SIGNALS
- Add num_workers instance variable for dynamic count
- Add _get_minimum_workers() to enforce app worker constraints
- Add signal handlers for TTIN (increase) and TTOU (decrease)
- Update manage_workers() to use dynamic count
- Add documentation for dynamic scaling
- Add unit tests for signal handling
- Add Docker integration tests

The minimum worker constraint ensures TTOU cannot reduce workers below
what apps require (e.g., if an app has workers=3, minimum is 3).

Closes #3489

Author: benoitc

docs/content/dirty.md

@@ -912,9 +912,40 @@ Dirty Arbiters integrate with the main arbiter's signal handling. Signals are fo
 | `SIGQUIT` | Immediate exit via `sys.exit(0)` | Killed immediately | Fast shutdown, no cleanup |
 | `SIGHUP` | Kills all workers, spawns new ones | Exits immediately | Hot reload of workers |
 | `SIGUSR1` | Reopens log files, forwards to workers | Reopens log files | Log rotation support |
+| `SIGTTIN` | Increases worker count by 1 | N/A | Dynamic scaling up |
+| `SIGTTOU` | Decreases worker count by 1 | N/A | Dynamic scaling down |
 | `SIGCHLD` | Handled by event loop, triggers reap | N/A | Worker death detection |
 | `SIGINT` | Same as SIGTERM | Same as SIGTERM | Ctrl-C handling |
 
+### Dynamic Scaling with TTIN/TTOU
+
+You can dynamically scale the number of dirty workers at runtime using signals, without restarting gunicorn:
+
+```bash
+# Find the dirty arbiter process
+ps aux | grep dirty-arbiter
+# Or use the PID file (location depends on your app name)
+cat /tmp/gunicorn-dirty-myapp.pid
+
+# Increase dirty workers by 1
+kill -TTIN <dirty-arbiter-pid>
+
+# Decrease dirty workers by 1
+kill -TTOU <dirty-arbiter-pid>
+```
+
+**Minimum Worker Constraint:** The dirty arbiter will not decrease below the minimum number of workers required by your app configurations. For example, if you have an app with `workers = 3`, you cannot scale below 3 dirty workers. When this limit is reached, a warning is logged:
+
+```
+WARNING: SIGTTOU: Cannot decrease below 3 workers (required by app specs)
+```
+
+**Use Cases:**
+
+- **Burst handling** - Scale up when you anticipate heavy load
+- **Cost optimization** - Scale down during low-traffic periods
+- **Recovery** - Scale up if workers are busy with long-running tasks
+
 ### Forwarded Signals
 
 The main arbiter forwards these signals to the dirty arbiter process:

gunicorn/dirty/arbiter.py

@@ -57,7 +57,7 @@ class DirtyArbiter:
     """
 
     SIGNALS = [getattr(signal, "SIG%s" % x) for x in
-               "HUP QUIT INT TERM USR1 USR2 CHLD".split()]
+               "HUP QUIT INT TERM TTIN TTOU USR1 USR2 CHLD".split()]
 
     # Worker boot error code
     WORKER_BOOT_ERROR = 3
@@ -92,6 +92,7 @@ def __init__(self, cfg, log, socket_path=None, pidfile=None):
         self._worker_rr_index = 0  # Round-robin index for worker selection
         self.worker_age = 0
         self.alive = True
+        self.num_workers = self.cfg.dirty_workers  # Dynamic count for TTIN/TTOU
 
         self._server = None
         self._loop = None
@@ -150,6 +151,23 @@ def _parse_app_specs(self):
             # Initialize the app_worker_map for this app
             self.app_worker_map[import_path] = set()
 
+    def _get_minimum_workers(self):
+        """
+        Calculate minimum number of workers required by app specs.
+
+        Returns the maximum worker_count across all apps that have limits.
+        Apps with worker_count=None don't impose a minimum.
+
+        Returns:
+            int: Minimum workers required (at least 1)
+        """
+        min_required = 1
+        for spec in self.app_specs.values():
+            worker_count = spec['worker_count']
+            if worker_count is not None:
+                min_required = max(min_required, worker_count)
+        return min_required
+
     def _get_apps_for_new_worker(self):
         """
         Determine which apps a new worker should load.
@@ -255,6 +273,8 @@ def init_signals(self):
         signal.signal(signal.SIGHUP, self._signal_handler)
         signal.signal(signal.SIGUSR1, self._signal_handler)
         signal.signal(signal.SIGCHLD, self._signal_handler)
+        signal.signal(signal.SIGTTIN, self._signal_handler)
+        signal.signal(signal.SIGTTOU, self._signal_handler)
 
     def _signal_handler(self, sig, frame):
         """Handle signals."""
@@ -279,6 +299,36 @@ def _signal_handler(self, sig, frame):
                 )
             return
 
+        if sig == signal.SIGTTIN:
+            # Increase number of workers
+            self.num_workers += 1
+            self.log.info("SIGTTIN: Increasing dirty workers to %s",
+                          self.num_workers)
+            if self._loop:
+                self._loop.call_soon_threadsafe(
+                    lambda: asyncio.create_task(self.manage_workers())
+                )
+            return
+
+        if sig == signal.SIGTTOU:
+            # Decrease number of workers (respecting minimum)
+            min_workers = self._get_minimum_workers()
+            if self.num_workers <= min_workers:
+                self.log.warning(
+                    "SIGTTOU: Cannot decrease below %s workers "
+                    "(required by app specs)",
+                    min_workers
+                )
+                return
+            self.num_workers -= 1
+            self.log.info("SIGTTOU: Decreasing dirty workers to %s",
+                          self.num_workers)
+            if self._loop:
+                self._loop.call_soon_threadsafe(
+                    lambda: asyncio.create_task(self.manage_workers())
+                )
+            return
+
         # Shutdown signals
         self.alive = False
         if self._loop:
@@ -717,7 +767,7 @@ async def manage_workers(self):
         if not self.alive:
             return
 
-        num_workers = self.cfg.dirty_workers
+        num_workers = self.num_workers
 
         # Spawn workers if needed
         while self.alive and len(self.workers) < num_workers:

tests/dirty/test_arbiter_signals.py

@@ -0,0 +1,234 @@
+#
+# This file is part of gunicorn released under the MIT license.
+# See the NOTICE for more information.
+
+"""Tests for dirty arbiter TTIN/TTOU signal handling."""
+
+import signal
+from unittest.mock import Mock
+
+import pytest
+
+
+class TestDirtyArbiterSignals:
+    """Test TTIN/TTOU signal handling in DirtyArbiter."""
+
+    @pytest.fixture
+    def arbiter(self, tmp_path):
+        """Create a DirtyArbiter for testing."""
+        from gunicorn.dirty.arbiter import DirtyArbiter
+
+        cfg = Mock()
+        cfg.dirty_workers = 2
+        cfg.dirty_apps = []
+        cfg.dirty_timeout = 30
+        cfg.dirty_graceful_timeout = 30
+        cfg.on_dirty_starting = Mock()
+        log = Mock()
+
+        arbiter = DirtyArbiter(cfg, log, socket_path=str(tmp_path / "test.sock"))
+        return arbiter
+
+    def test_initial_num_workers_from_config(self, arbiter):
+        """num_workers should be initialized from config."""
+        assert arbiter.num_workers == 2
+
+    def test_ttin_increases_num_workers(self, arbiter):
+        """SIGTTIN should increase num_workers by 1."""
+        assert arbiter.num_workers == 2
+        arbiter._signal_handler(signal.SIGTTIN, None)
+        assert arbiter.num_workers == 3
+
+    def test_ttin_logs_info(self, arbiter):
+        """SIGTTIN should log info about the change."""
+        arbiter._signal_handler(signal.SIGTTIN, None)
+        arbiter.log.info.assert_called()
+        call_args = arbiter.log.info.call_args[0]
+        assert "SIGTTIN" in call_args[0]
+        assert "3" in str(call_args)
+
+    def test_ttou_decreases_num_workers(self, arbiter):
+        """SIGTTOU should decrease num_workers by 1."""
+        arbiter.num_workers = 3
+        arbiter._signal_handler(signal.SIGTTOU, None)
+        assert arbiter.num_workers == 2
+
+    def test_ttou_logs_info(self, arbiter):
+        """SIGTTOU should log info about the change."""
+        arbiter.num_workers = 3
+        arbiter._signal_handler(signal.SIGTTOU, None)
+        arbiter.log.info.assert_called()
+        call_args = arbiter.log.info.call_args[0]
+        assert "SIGTTOU" in call_args[0]
+        assert "2" in str(call_args)
+
+    def test_ttou_respects_minimum_one_worker(self, arbiter):
+        """SIGTTOU should not go below 1 worker by default."""
+        arbiter.num_workers = 1
+        arbiter._signal_handler(signal.SIGTTOU, None)
+        assert arbiter.num_workers == 1
+
+    def test_ttou_logs_warning_at_minimum(self, arbiter):
+        """SIGTTOU should log warning when at minimum."""
+        arbiter.num_workers = 1
+        arbiter._signal_handler(signal.SIGTTOU, None)
+        arbiter.log.warning.assert_called()
+        call_args = arbiter.log.warning.call_args[0]
+        assert "Cannot decrease below" in call_args[0]
+
+    def test_ttou_respects_app_minimum(self, arbiter):
+        """SIGTTOU should not go below app-required minimum."""
+        # App requires 3 workers
+        arbiter.app_specs = {
+            'myapp:HeavyTask': {
+                'import_path': 'myapp:HeavyTask',
+                'worker_count': 3,
+                'original_spec': 'myapp:HeavyTask:3',
+            }
+        }
+        arbiter.num_workers = 3
+
+        # Should not decrease below 3
+        arbiter._signal_handler(signal.SIGTTOU, None)
+        assert arbiter.num_workers == 3
+        arbiter.log.warning.assert_called()
+
+    def test_ttou_with_unlimited_app(self, arbiter):
+        """Apps with worker_count=None should not impose minimum."""
+        arbiter.app_specs = {
+            'myapp:UnlimitedTask': {
+                'import_path': 'myapp:UnlimitedTask',
+                'worker_count': None,
+                'original_spec': 'myapp:UnlimitedTask',
+            }
+        }
+        arbiter.num_workers = 2
+
+        # Should decrease to 1 (default minimum)
+        arbiter._signal_handler(signal.SIGTTOU, None)
+        assert arbiter.num_workers == 1
+
+    def test_multiple_ttin_signals(self, arbiter):
+        """Multiple TTIN signals should keep incrementing."""
+        assert arbiter.num_workers == 2
+        arbiter._signal_handler(signal.SIGTTIN, None)
+        arbiter._signal_handler(signal.SIGTTIN, None)
+        arbiter._signal_handler(signal.SIGTTIN, None)
+        assert arbiter.num_workers == 5
+
+    def test_multiple_ttou_signals(self, arbiter):
+        """Multiple TTOU signals should decrement until minimum."""
+        arbiter.num_workers = 5
+        arbiter._signal_handler(signal.SIGTTOU, None)
+        arbiter._signal_handler(signal.SIGTTOU, None)
+        arbiter._signal_handler(signal.SIGTTOU, None)
+        arbiter._signal_handler(signal.SIGTTOU, None)
+        # Should stop at 1
+        assert arbiter.num_workers == 1
+
+
+class TestGetMinimumWorkers:
+    """Test _get_minimum_workers calculation."""
+
+    @pytest.fixture
+    def arbiter(self, tmp_path):
+        """Create a DirtyArbiter for testing."""
+        from gunicorn.dirty.arbiter import DirtyArbiter
+
+        cfg = Mock()
+        cfg.dirty_workers = 2
+        cfg.dirty_apps = []
+        cfg.dirty_timeout = 30
+        cfg.dirty_graceful_timeout = 30
+        cfg.on_dirty_starting = Mock()
+        log = Mock()
+
+        arbiter = DirtyArbiter(cfg, log, socket_path=str(tmp_path / "test.sock"))
+        return arbiter
+
+    def test_minimum_workers_no_apps(self, arbiter):
+        """With no apps, minimum should be 1."""
+        arbiter.app_specs = {}
+        assert arbiter._get_minimum_workers() == 1
+
+    def test_minimum_workers_single_app_with_limit(self, arbiter):
+        """Single app with worker_count should set minimum."""
+        arbiter.app_specs = {
+            'app:Task': {
+                'import_path': 'app:Task',
+                'worker_count': 3,
+                'original_spec': 'app:Task:3',
+            }
+        }
+        assert arbiter._get_minimum_workers() == 3
+
+    def test_minimum_workers_single_app_unlimited(self, arbiter):
+        """Single app with worker_count=None should use default minimum."""
+        arbiter.app_specs = {
+            'app:Task': {
+                'import_path': 'app:Task',
+                'worker_count': None,
+                'original_spec': 'app:Task',
+            }
+        }
+        assert arbiter._get_minimum_workers() == 1
+
+    def test_minimum_workers_multiple_apps_with_limits(self, arbiter):
+        """Multiple apps should use the maximum worker_count."""
+        arbiter.app_specs = {
+            'app1:Task1': {
+                'import_path': 'app1:Task1',
+                'worker_count': 2,
+                'original_spec': 'app1:Task1:2',
+            },
+            'app2:Task2': {
+                'import_path': 'app2:Task2',
+                'worker_count': 4,
+                'original_spec': 'app2:Task2:4',
+            },
+            'app3:Task3': {
+                'import_path': 'app3:Task3',
+                'worker_count': 3,
+                'original_spec': 'app3:Task3:3',
+            },
+        }
+        # Maximum of (2, 4, 3) = 4
+        assert arbiter._get_minimum_workers() == 4
+
+    def test_minimum_workers_mixed_limited_and_unlimited(self, arbiter):
+        """Mixed apps should use max of limited apps only."""
+        arbiter.app_specs = {
+            'app1:Task1': {
+                'import_path': 'app1:Task1',
+                'worker_count': 2,
+                'original_spec': 'app1:Task1:2',
+            },
+            'app2:Task2': {
+                'import_path': 'app2:Task2',
+                'worker_count': None,
+                'original_spec': 'app2:Task2',
+            },
+            'app3:Task3': {
+                'import_path': 'app3:Task3',
+                'worker_count': 4,
+                'original_spec': 'app3:Task3:4',
+            },
+        }
+        # Maximum of (2, 4) = 4, None is ignored
+        assert arbiter._get_minimum_workers() == 4
+
+    def test_minimum_workers_all_unlimited(self, arbiter):
+        """All unlimited apps should use default minimum."""
+        arbiter.app_specs = {
+            'app1:Task1': {
+                'import_path': 'app1:Task1',
+                'worker_count': None,
+                'original_spec': 'app1:Task1',
+            },
+            'app2:Task2': {
+                'import_path': 'app2:Task2',
+                'worker_count': None,
+                'original_spec': 'app2:Task2',
+            },
+        }
+        assert arbiter._get_minimum_workers() == 1

tests/docker/dirty_ttin_ttou/Dockerfile

@@ -0,0 +1,17 @@
+FROM python:3.12-slim
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    curl procps \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Install gunicorn from source
+COPY . /gunicorn-src/
+RUN pip install --no-cache-dir /gunicorn-src/
+
+# Copy test app
+COPY tests/docker/dirty_ttin_ttou/app.py /app/
+COPY tests/docker/dirty_ttin_ttou/gunicorn_conf.py /app/
+
+CMD ["gunicorn", "-c", "gunicorn_conf.py", "app:app"]

tests/docker/dirty_ttin_ttou/__init__.py

@@ -0,0 +1,5 @@
+#
+# This file is part of gunicorn released under the MIT license.
+# See the NOTICE for more information.
+
+"""Docker integration tests for dirty arbiter TTIN/TTOU signals."""