clean-up celery worker

Author: Johnetordoff

docker-compose.yml

@@ -414,26 +414,16 @@ services:
       - .docker-compose.sharejs.env
     stdin_open: true
 
-#  beat:
-#    image: quay.io/centerforopenscience/osf:develop
-#    command: invoke celery_beat
-#    depends_on:
-#      - postgres
-#      - rabbitmq
-#    environment:
-#      C_FORCE_ROOT: 1
-#      DJANGO_SETTINGS_MODULE: api.base.settings
-#    env_file:
-#      - .docker-compose.env
-#    volumes:
-#      - ./:/code
-#      - osf_requirements_3_12_vol:/usr/local/lib/python3.12
-#      - osf_bower_components_vol:/code/website/static/vendor/bower_components
-#      - osf_node_modules_vol:/code/node_modules
-
   worker:
     image: quay.io/centerforopenscience/osf:develop
-    command: python3 -m invoke celery-worker
+    # Run worker in background; keep beat in foreground as the main PID
+    command:
+      - /bin/bash
+      - -lc
+      - |
+        set -e
+        python3 -m invoke celery-worker &
+        invoke beat
     restart: unless-stopped
     depends_on:
       - postgres
@@ -442,17 +432,13 @@ services:
     environment:
       C_FORCE_ROOT: 1
       DJANGO_SETTINGS_MODULE: api.base.settings
-#      BROKER_URL: amqp://guest:guest@192.168.168.167:5671/
     env_file:
       - .docker-compose.env
     volumes:
       - ./:/code:cached
       - osf_requirements_3_12_vol:/usr/local/lib/python3.12/
       - osf_bower_components_vol:/code/website/static/vendor/bower_components
       - osf_node_modules_vol:/code/node_modules
-#      - ./ssl/ca-chain.cert.pem:/etc/ssl/certs/ca-chain.cert.pem:ro
-#      - ./ssl/celery-client.cert.pem:/etc/ssl/certs/worker.pem:ro
-#      - ./ssl/celery-client.key.pem:/etc/ssl/private/worker.key:ro
     stdin_open: true
 
   admin:

tasks/__init__.py

@@ -13,7 +13,6 @@
 import invoke
 from invoke import Collection
 
-from website import settings
 from .utils import pip_install, bin_prefix
 
 
@@ -63,6 +62,8 @@ def decorator(f):
 @task
 def server(ctx, host=None, port=5000, debug=True, gitlogs=False):
     """Run the app server."""
+    from website import settings
+
     if os.environ.get('WERKZEUG_RUN_MAIN') == 'true' or not debug:
         if os.environ.get('WEB_REMOTE_DEBUG', None):
             import pydevd
@@ -95,6 +96,8 @@ def git_logs(ctx, branch=None):
 def apiserver(ctx, port=8000, wait=True, autoreload=True, host='127.0.0.1', pty=True):
     """Run the API server."""
     env = os.environ.copy()
+    from website import settings
+
     cmd = 'DJANGO_SETTINGS_MODULE=api.base.settings {} manage.py runserver {}:{} --nothreading'\
         .format(sys.executable, host, port)
     if not autoreload:
@@ -114,6 +117,8 @@ def apiserver(ctx, port=8000, wait=True, autoreload=True, host='127.0.0.1', pty=
 def adminserver(ctx, port=8001, host='127.0.0.1', pty=True):
     """Run the Admin server."""
     env = 'DJANGO_SETTINGS_MODULE="admin.base.settings"'
+    from website import settings
+
     cmd = f'{env} python3 manage.py runserver {host}:{port} --nothreading'
     if settings.SECURE_MODE:
         cmd = cmd.replace('runserver', 'runsslserver')
@@ -141,6 +146,7 @@ def sharejs(ctx, host=None, port=None, db_url=None, cors_allow_origin=None):
         os.environ['SHAREJS_DB_URL'] = db_url
     if cors_allow_origin:
         os.environ['SHAREJS_CORS_ALLOW_ORIGIN'] = cors_allow_origin
+    from website import settings
 
     if settings.SENTRY_DSN:
         os.environ['SHAREJS_SENTRY_DSN'] = settings.SENTRY_DSN
@@ -179,7 +185,7 @@ def celery_beat(ctx, level='debug', schedule=None):
     ctx.run(bin_prefix(cmd), pty=True)
 
 @task
-def migrate_search(ctx, delete=True, remove=False, index=settings.ELASTIC_INDEX):
+def migrate_search(ctx, delete=True, remove=False, index='website'):
     """Migrate the search-enabled models."""
     from website.app import init_app
     init_app(routes=False, set_backends=False)
@@ -546,6 +552,7 @@ def wheelhouse(ctx, addons=False, release=False, dev=False, pty=True):
         inv wheelhouse --addons
         inv wheelhouse --release
     """
+    from website import settings
     if release or addons:
         for directory in os.listdir(settings.ADDON_PATH):
             path = os.path.join(settings.ADDON_PATH, directory)
@@ -567,6 +574,8 @@ def wheelhouse(ctx, addons=False, release=False, dev=False, pty=True):
 @task
 def addon_requirements(ctx):
     """Install all addon requirements."""
+    from website import settings
+
     for directory in os.listdir(settings.ADDON_PATH):
         path = os.path.join(settings.ADDON_PATH, directory)
 
@@ -583,6 +592,8 @@ def addon_requirements(ctx):
 
 @task
 def ci_addon_settings(ctx):
+    from website import settings
+
     for directory in os.listdir(settings.ADDON_PATH):
         path = os.path.join(settings.ADDON_PATH, directory, 'settings')
         if os.path.isdir(path):
@@ -595,6 +606,8 @@ def ci_addon_settings(ctx):
 
 @task
 def copy_addon_settings(ctx):
+    from website import settings
+
     for directory in os.listdir(settings.ADDON_PATH):
         path = os.path.join(settings.ADDON_PATH, directory, 'settings')
         if os.path.isdir(path) and not os.path.isfile(os.path.join(path, 'local.py')):

website/notifications/tasks.py

@@ -164,7 +164,7 @@ def send_moderator_email_task(self, user_id, provider_id, notification_ids, mess
         logger.exception('Retrying send_moderator_email_task due to exception')
         raise self.retry(exc=e)
 
-@celery_app.task
+@celery_app.task(name='website.notifications.tasks.send_users_digest_email')
 def send_users_digest_email():
     today = date.today()
 
@@ -181,7 +181,7 @@ def send_users_digest_email():
             notification_ids = [msg['notification_id'] for msg in group['info']]
             send_user_email_task.delay(user_id, notification_ids, freq)
 
-@celery_app.task
+@celery_app.task(name='website.notifications.tasks.send_moderators_digest_email')
 def send_moderators_digest_email():
     today = date.today()