Defer SIGHUP handlers to reactor. (#8817)

We can get a SIGHUP at any point, including times where we are not in a
sane state. By deferring calling the handlers until the next reactor
tick we ensure that we don't get unexpected conflicts, e.g. trying to
flush logs from the signal handler while the code was in the process of
writing a log entry.

Fixes #8769.

Author: erikjohnston

changelog.d/8817.bugfix

@@ -0,0 +1 @@
+Fix bug where logging could break after a call to SIGHUP.

synapse/app/_base.py

@@ -32,6 +32,7 @@
 from synapse.config.server import ListenerConfig
 from synapse.crypto import context_factory
 from synapse.logging.context import PreserveLoggingContext
+from synapse.metrics.background_process_metrics import wrap_as_background_process
 from synapse.util.async_helpers import Linearizer
 from synapse.util.daemonize import daemonize_process
 from synapse.util.rlimit import change_resource_limit
@@ -244,6 +245,7 @@ def start(hs: "synapse.server.HomeServer", listeners: Iterable[ListenerConfig]):
         # Set up the SIGHUP machinery.
         if hasattr(signal, "SIGHUP"):
 
+            @wrap_as_background_process("sighup")
             def handle_sighup(*args, **kwargs):
                 # Tell systemd our state, if we're using it. This will silently fail if
                 # we're not using systemd.
@@ -254,7 +256,13 @@ def handle_sighup(*args, **kwargs):
 
                 sdnotify(b"READY=1")
 
-            signal.signal(signal.SIGHUP, handle_sighup)
+            # We defer running the sighup handlers until next reactor tick. This
+            # is so that we're in a sane state, e.g. flushing the logs may fail
+            # if the sighup happens in the middle of writing a log entry.
+            def run_sighup(*args, **kwargs):
+                hs.get_clock().call_later(0, handle_sighup, *args, **kwargs)
+
+            signal.signal(signal.SIGHUP, run_sighup)
 
             register_sighup(refresh_certificate, hs)