Merge pull request #49 from Xpirix/celery_scheduled_task

Run auto reject organization with celery scheduled jobs.

Author: Xpirix

deployment/Makefile

@@ -21,7 +21,7 @@ web:
 	@echo "------------------------------------------------------------------"
 	@echo "Running in production mode"
 	@echo "------------------------------------------------------------------"
-	@docker compose up -d web
+	@docker compose up -d web worker beat
 	@# Dont confuse this with the dbbackup make command below
 	@# This one runs the postgis-backup cron container
 	@# We add --no-recreate so that it does not destroy & recreate the db container
@@ -366,7 +366,7 @@ devweb: db
 	@echo "------------------------------------------------------------------"
 	@echo "Running in DEVELOPMENT mode"
 	@echo "------------------------------------------------------------------"
-	@docker compose up --no-deps -d devweb webpack
+	@docker compose up --no-deps -d devweb webpack rabbitmq worker beat
 
 devweb-runserver: devweb
 	@echo

deployment/docker-compose.override.example.yml

@@ -61,3 +61,11 @@ services:
       - ../django_project:/home/web/django_project
       - ${MEDIA_VOLUME}:/home/web/media:rw
       - ./logs:/var/log
+
+  beat:
+    volumes:
+      - ../django_project:/home/web/django_project
+
+  worker:
+    volumes:
+      - ../django_project:/home/web/django_project

deployment/docker-compose.yml

@@ -13,6 +13,8 @@ volumes:
   media-data:
   reports-data:
   nginx-conf:
+  rabbitmq:
+  celerybeat-schedule:
 services:
 
   db:
@@ -73,6 +75,7 @@ services:
       - media-data:/home/web/media:rw
       - reports-data:/home/web/reports
       - ./docker/uwsgi.conf:/uwsgi.conf
+      - celerybeat-schedule:/home/web/celerybeat-schedule:rw
     command: bash -c "npm install && npm run build && uwsgi --ini /uwsgi.conf"
     links:
       - db:db
@@ -136,6 +139,44 @@ services:
     networks:
       internal:
 
+  rabbitmq:
+    image: ${RABBITMQ_IMAGE:-rabbitmq:3.7-alpine}
+    hostname: rabbitmq
+    volumes:
+      - rabbitmq:/var/lib/rabbitmq
+    restart: unless-stopped
+    networks:
+      internal:
+    healthcheck:
+      test: ["CMD", "rabbitmqctl", "status"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  beat:
+    <<: *uwsgi-common
+    depends_on:
+      rabbitmq:
+        condition: service_healthy
+    working_dir: /home/web/django_project
+    entrypoint: [ ]
+    command: celery --app=certification.celery:app beat -s /home/web/celerybeat-schedule/schedule -l INFO
+    networks:
+      internal:
+
+  worker:
+    <<: *uwsgi-common
+    depends_on:
+      db:
+        condition: service_started
+      beat:
+        condition: service_started
+    working_dir: /home/web/django_project
+    entrypoint: []
+    command: celery -A certification worker -l INFO
+    networks:
+      internal:
+
   # This is the entry point for a development server.
   # Run with --no-deps to run attached to the services
   # from prod environment if wanted

deployment/docker/REQUIREMENTS.txt

@@ -60,4 +60,7 @@ django-webpack-loader~=3.1
 sorl-thumbnail~=12.11
 
 # Resend email
-resend~=2.7
+resend~=2.7
+
+# Celery
+celery~=5.3

django_project/certification/__init__.py

@@ -0,0 +1,10 @@
+from __future__ import absolute_import, unicode_literals
+
+# This will make sure the app is always imported when
+# Django starts so that shared_task will use this app.
+from .celery import app as celery_app
+
+default_app_config = "certification.apps.CertificationConfig"
+__all__ = [
+    "celery_app",
+]

django_project/certification/celery.py

@@ -0,0 +1,28 @@
+from __future__ import absolute_import
+
+import os
+
+from celery import Celery
+import logging
+
+logger = logging.getLogger('certification')
+
+
+# set the default Django settings module for the 'celery' program.
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "core.settings.base")
+
+app = Celery("certification")
+
+# Using a string here means the worker doesn't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+#   should have a `CELERY_` prefix.
+app.config_from_object("django.conf:settings", namespace="CELERY")
+
+# Load task modules from all registered Django app configs.
+app.autodiscover_tasks()
+
+
+@app.task(bind=True)
+def debug_task(self):
+    print("Request: {0!r}".format(self.request))

django_project/certification/management/commands/reject_pending_organisations.py

@@ -0,0 +1 @@
+from certification.tasks.reject_pending_organisations import *

django_project/certification/tasks/__init__.py

@@ -0,0 +1,43 @@
+from celery import shared_task
+from django.utils import timezone
+from datetime import timedelta
+from ..models.certifying_organisation import CertifyingOrganisation
+from ..models.status import Status
+from ..views import send_rejection_email
+
+
+@shared_task
+def reject_pending_organisations(days=365):
+    """Celery task to reject pending certifying organisations
+    that were created more than one year ago.
+    """
+    print('Begin process....')
+    print(f'Number of days: {days}')
+    one_year_ago = timezone.now() - timedelta(days=days)
+    old_organisations = CertifyingOrganisation.unapproved_objects.all()
+
+    print('Begin process to reject old certifying organisations.')
+
+    count = 0
+    for organisation in old_organisations:
+        if organisation.update_date < one_year_ago:
+            rejected_status, created = Status.objects.get_or_create(
+                name='Rejected',
+                project=organisation.project
+            )
+            organisation.status = rejected_status
+            organisation.rejected = True
+            organisation.save()
+
+            send_rejection_email(
+                organisation,
+                'certification.qgis.org',
+                'https'
+            )
+            count += 1
+            print(f'{organisation.name} has been set to Rejected')
+
+    result = f'{count} certifying organisations have been set to Rejected'
+    print('------------------------------------------------------------')
+    print('Process finished.')
+    return result

django_project/certification/tasks/reject_pending_organisations.py

@@ -5,6 +5,7 @@
 # Django settings for projecta project.
 import os
 from .utils import absolute_path
+from celery.schedules import crontab
 
 ADMINS = (
     ('Tim Sutton', 'tim@kartoza.com'),
@@ -210,4 +211,16 @@
         'BUNDLE_DIR_NAME': 'bundles',
         'STATS_FILE': os.path.join(SITE_ROOT, 'webpack-stats.json'),
     }
+}
+
+CELERY_RESULT_BACKEND = 'rpc://'
+CELERY_BROKER_URL = os.environ.get('BROKER_URL', 'amqp://rabbitmq:5672')
+CELERY_BEAT_SCHEDULE = {
+    'reject_pending_organisations': {
+        'task': 'certification.tasks.reject_pending_organisations.reject_pending_organisations',
+        'schedule': crontab(minute=0, hour=0),  # Execute every day at midnight.
+        'kwargs': {
+            'days': 365,  # Auto reject pending organisations that wasn't updated for a year.
+        }
+    },
 }