Introduce Celery worker for processing background tasks (2)

backend/backend/__init__.py

@@ -0,0 +1,3 @@
+from .celery import app as celery_app
+
+__all__ = ('celery_app',)

backend/backend/celery.py

@@ -0,0 +1,22 @@
+import os
+
+from celery import Celery
+
+# set the default Django settings module for the 'celery' program.
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'backend.settings.base')
+
+app = Celery('backend')
+
+# Using a string here means the worker doesn't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+#   should have a `CELERY_` prefix.
+app.config_from_object('django.conf:settings', namespace='CELERY')
+
+# Load task modules from all registered Django app configs.
+app.autodiscover_tasks()
+
+
+@app.task(bind=True)
+def debug_task(self):  # TODO: to delete.
+    print('Request: {0!r}'.format(self.request))

backend/backend/settings/base.py

@@ -15,6 +15,7 @@
 import re
 
 from netaddr import IPNetwork, AddrFormatError
+from celery.schedules import crontab
 
 
 def check_ip_range(ipr):
@@ -67,6 +68,7 @@ def check_ip_range(ipr):
     'tagulous',
     'device_registry.apps.DeviceRegistryConfig',
     'profile_page.apps.ProfilePageConfig',
+    'monitoring.apps.MonitoringConfig',
     'bootstrap4'
 ]
 
@@ -218,3 +220,16 @@ def check_ip_range(ipr):
 
 # Retry to connect to DB (after receiving a connection error) within 60 seconds.
 DB_RETRY_TO_CONNECT_SEC = 60
+
+# Celery settings.
+CELERY_BROKER_URL = 'amqp://%s:%s@%s:5672/%s' % (os.getenv('RABBIT_USER', 'user'),
+                                                 os.getenv('RABBIT_PASSWORD', 'SuperSecurePassword'),
+                                                 os.getenv('RABBIT_HOST', 'localhost'),
+                                                 os.getenv('RABBIT_VHOST', 'wott-dash'))
+
+CELERY_BEAT_SCHEDULE = {
+    'update_celery_pulse_timestamp': {
+        'task': 'monitoring.tasks.update_celery_pulse_timestamp',
+        'schedule': crontab()  # Execute once a minute.
+    }
+}

backend/backend/urls.py

@@ -26,5 +26,6 @@
     url(r'^accounts/', include('registration.backends.simple.urls')),
     path('user/', include('profile_page.urls')),
     path('', include('device_registry.urls')),
+    path('monitoring/', include('monitoring.urls')),
     url('', include('django_prometheus.urls'))
 ]

backend/backend/wsgi.py

@@ -14,7 +14,7 @@
 
 from .utils import ensure_connection_with_retries
 
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'backend.settings')
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'backend.settings.base')
 
 # Patch the standard django database connection class' method in order to try to
 # connect to DB multiple times using exponential backoff algorithm until success

backend/device_registry/tasks.py

@@ -0,0 +1,7 @@
+# Create your tasks here
+from celery import shared_task
+
+
+# @shared_task
+# def add(x, y):
+#     return x + y

backend/monitoring/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

backend/monitoring/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class MonitoringConfig(AppConfig):
+    name = 'monitoring'

backend/monitoring/migrations/0001_initial.py

@@ -0,0 +1,21 @@
+# Generated by Django 2.1.10 on 2019-08-28 12:59
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='CeleryPulseTimestamp',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('timestamp', models.DateTimeField(auto_now=True)),
+            ],
+        ),
+    ]

backend/monitoring/models.py

@@ -0,0 +1,5 @@
+from django.db import models
+
+
+class CeleryPulseTimestamp(models.Model):
+    timestamp = models.DateTimeField(auto_now=True)

backend/monitoring/tasks.py

@@ -0,0 +1,12 @@
+from celery import shared_task
+
+from .models import CeleryPulseTimestamp
+
+
+@shared_task
+def update_celery_pulse_timestamp():
+    pulse_obj = CeleryPulseTimestamp.objects.order_by('id').first()
+    if pulse_obj is None:
+        CeleryPulseTimestamp.objects.create()
+    else:
+        pulse_obj.save()

backend/monitoring/urls.py

@@ -0,0 +1,7 @@
+from django.urls import path
+
+from .views import CeleryPulseTimestampView
+
+urlpatterns = [
+    path('celery/', CeleryPulseTimestampView.as_view(), name='celery_pulse')
+]

backend/monitoring/views.py

@@ -0,0 +1,21 @@
+from django.http import HttpResponse
+from django.views.generic import View
+from django.contrib.auth.mixins import AccessMixin, LoginRequiredMixin
+
+from .models import CeleryPulseTimestamp
+
+
+class StaffuserRequiredMixin(AccessMixin):
+    """Verify that the current user has `is_staff` set to True."""
+
+    def dispatch(self, request, *args, **kwargs):
+        if not request.user.is_staff:
+            return self.handle_no_permission()
+        return super().dispatch(request, *args, **kwargs)
+
+
+class CeleryPulseTimestampView(LoginRequiredMixin, StaffuserRequiredMixin, View):
+
+    def get(self, request, *args, **kwargs):
+        pulse_obj = CeleryPulseTimestamp.objects.order_by('id').first()
+        return HttpResponse(str(pulse_obj.timestamp) if pulse_obj else 'None', content_type='text/plain')

docker-compose.yml

@@ -3,42 +3,41 @@ services:
   migrate:
     command: python manage.py migrate --noinput
     environment:
-    - DJANGO_SETTINGS_MODULE=backend.settings.dev
-    - DB_PASSWORD=SuperSecurePassword
-    - DB_USER=postgres
+      - DJANGO_SETTINGS_MODULE=backend.settings.dev
+      - DB_PASSWORD=SuperSecurePassword
+      - DB_USER=postgres
     build:
       context: .
     volumes:
       - ./backend:/usr/src/app
     depends_on:
-    - psql
-    - nginx-static
+      - psql
 
   dash-dev:
     hostname: dash-dev0
     environment:
-    - DJANGO_SETTINGS_MODULE=backend.settings.dev
-    - DB_PASSWORD=SuperSecurePassword
-    - DB_USER=postgres
-    - DATASTORE_KEY_JSON
+      - DJANGO_SETTINGS_MODULE=backend.settings.dev
+      - DB_PASSWORD=SuperSecurePassword
+      - DB_USER=postgres
+      - DATASTORE_KEY_JSON
     build:
       context: .
     volumes:
       - ./backend:/usr/src/app
     ports:
       - '8000:8000'
     depends_on:
-    - psql
-    - migrate
-    - nginx-static
+      - psql
+      - migrate
+      - nginx-static
 
   api-dev:
     hostname: api-dev0
     environment:
-    - DJANGO_SETTINGS_MODULE=backend.settings.dev
-    - DB_PASSWORD=SuperSecurePassword
-    - DB_USER=postgres
-    - DATASTORE_KEY_JSON
+      - DJANGO_SETTINGS_MODULE=backend.settings.dev
+      - DB_PASSWORD=SuperSecurePassword
+      - DB_USER=postgres
+      - DATASTORE_KEY_JSON
     image: api:latest
     build:
       context: .
@@ -47,25 +46,25 @@ services:
     ports:
       - '8001:8000'
     depends_on:
-    - psql
-    - migrate
+      - psql
+      - migrate
 
   mtls-api-dev:
     hostname: mtls-api-dev0
     environment:
-    - DJANGO_SETTINGS_MODULE=backend.settings.dev
-    - DB_PASSWORD=SuperSecurePassword
-    - DB_USER=postgres
-    - DATASTORE_KEY_JSON
+      - DJANGO_SETTINGS_MODULE=backend.settings.dev
+      - DB_PASSWORD=SuperSecurePassword
+      - DB_USER=postgres
+      - DATASTORE_KEY_JSON
     build:
       context: .
     volumes:
       - ./backend:/usr/src/app
     ports:
       - '8002:8000'
     depends_on:
-    - psql
-    - migrate
+      - psql
+      - migrate
 
   nginx-static:
     hostname: wott-static
@@ -80,11 +79,58 @@ services:
 
   psql:
     environment:
-    - POSTGRES_DB=wott-backend
-    - POSTGRES_PASSWORD=SuperSecurePassword
+      - POSTGRES_DB=wott-backend
+      - POSTGRES_PASSWORD=SuperSecurePassword
     image: postgres:alpine
     volumes:
-    - db-data:/var/lib/postgresql/data
+      - db-data:/var/lib/postgresql/data
+
+  rabbit:
+    image: rabbitmq:3-alpine
+    environment:
+      - RABBITMQ_DEFAULT_USER=user
+      - RABBITMQ_DEFAULT_PASS=SuperSecurePassword
+      - RABBITMQ_DEFAULT_VHOST=wott-dash
+    ports:
+      - '5672:5672'
+
+  celery:
+    environment:
+      - DJANGO_SETTINGS_MODULE=backend.settings.dev
+      - DB_PASSWORD=SuperSecurePassword
+      - DB_USER=postgres
+      - RABBIT_HOST=rabbit
+      - RABBIT_USER=user
+      - RABBIT_PASSWORD=SuperSecurePassword
+      - RABBIT_VHOST=wott-dash
+    build:
+      context: .
+    command: celery -A backend worker -l error --time-limit=90 --soft-time-limit=60
+    volumes:
+      - ./backend:/usr/src/app
+    depends_on:
+      - psql
+      - migrate
+      - rabbit
+
+  celery-beat:
+    environment:
+      - DJANGO_SETTINGS_MODULE=backend.settings.dev
+      - DB_PASSWORD=SuperSecurePassword
+      - DB_USER=postgres
+      - RABBIT_HOST=rabbit
+      - RABBIT_USER=user
+      - RABBIT_PASSWORD=SuperSecurePassword
+      - RABBIT_VHOST=wott-dash
+    build:
+      context: .
+    command: celery -A backend beat -l error
+    volumes:
+      - ./backend:/usr/src/app
+    depends_on:
+      - psql
+      - migrate
+      - rabbit
 
 volumes:
   db-data:

helm/api/templates/deployment-celery-beat.yaml

@@ -0,0 +1,74 @@
+apiVersion: apps/v1beta2
+kind: Deployment
+metadata:
+  name: {{ .Values.service.celery-beat.name }}
+  namespace: {{ .Values.namespace }}
+  labels:
+    app: {{ .Values.service.celery-beat.name }}
+    chart: {{ include "api.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: {{ .Values.service.celery-beat.name }}
+      release: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        app: {{ .Values.service.celery-beat.name }}
+        release: {{ .Release.Name }}
+    spec:
+      containers:
+        - name: {{ .Values.service.celery-beat.name }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          command: "celery -A backend beat -l error"
+          env:
+            - name: RELEASE_TRIGGER
+              value: "{{ .Values.releaseTimeStamp }}"
+            - name: DEBUG
+              value: "{{ .Values.service.api.debug }}"
+            - name: SECRET_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.apiSecrets }}
+                  key: djangoSecretKey
+            - name: DB_HOST
+              value: "{{ .Values.databaseHost }}"
+            - name: DB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.apiSecrets }}
+                  key: databasePassword
+            - name: SENTRY_DSN
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.apiSecrets }}
+                  key: sentryDsn
+            - name: RABBIT_HOST
+              value: {{ .Values.rabbitHost }}
+            - name: RABBIT_USER
+              value: {{ .Values.rabbitUser }}
+            - name: RABBIT_VHOST
+              value: {{ .Values.rabbitVhost }}
+            - name: RABBIT_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.apiSecrets }}
+                  key: rabbitPassword
+          resources:
+{{ toYaml .Values.resources.api | indent 12 }}
+    {{- with .Values.nodeSelector }}
+      nodeSelector:
+{{ toYaml . | indent 8 }}
+    {{- end }}
+    {{- with .Values.affinity }}
+      affinity:
+{{ toYaml . | indent 8 }}
+    {{- end }}
+    {{- with .Values.tolerations }}
+      tolerations:
+{{ toYaml . | indent 8 }}
+    {{- end }}