diff --git a/backend/backend/__init__.py b/backend/backend/__init__.py index e69de29bb..fb989c4e6 100644 --- a/backend/backend/__init__.py +++ b/backend/backend/__init__.py @@ -0,0 +1,3 @@ +from .celery import app as celery_app + +__all__ = ('celery_app',) diff --git a/backend/backend/celery.py b/backend/backend/celery.py new file mode 100644 index 000000000..674699ae9 --- /dev/null +++ b/backend/backend/celery.py @@ -0,0 +1,22 @@ +import os + +from celery import Celery + +# set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'backend.settings.base') + +app = Celery('backend') + +# Using a string here means the worker doesn't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +app.config_from_object('django.conf:settings', namespace='CELERY') + +# Load task modules from all registered Django app configs. +app.autodiscover_tasks() + + +@app.task(bind=True) +def debug_task(self): # TODO: to delete. + print('Request: {0!r}'.format(self.request)) diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py index e53624e2f..b9d15ae49 100644 --- a/backend/backend/settings/base.py +++ b/backend/backend/settings/base.py @@ -15,6 +15,7 @@ import re from netaddr import IPNetwork, AddrFormatError +from celery.schedules import crontab def check_ip_range(ipr): @@ -67,6 +68,7 @@ def check_ip_range(ipr): 'tagulous', 'device_registry.apps.DeviceRegistryConfig', 'profile_page.apps.ProfilePageConfig', + 'monitoring.apps.MonitoringConfig', 'bootstrap4' ] @@ -218,3 +220,16 @@ def check_ip_range(ipr): # Retry to connect to DB (after receiving a connection error) within 60 seconds. DB_RETRY_TO_CONNECT_SEC = 60 + +# Celery settings. +CELERY_BROKER_URL = 'amqp://%s:%s@%s:5672/%s' % (os.getenv('RABBIT_USER', 'user'), + os.getenv('RABBIT_PASSWORD', 'SuperSecurePassword'), + os.getenv('RABBIT_HOST', 'localhost'), + os.getenv('RABBIT_VHOST', 'wott-dash')) + +CELERY_BEAT_SCHEDULE = { + 'update_celery_pulse_timestamp': { + 'task': 'monitoring.tasks.update_celery_pulse_timestamp', + 'schedule': crontab() # Execute once a minute. + } +} diff --git a/backend/backend/urls.py b/backend/backend/urls.py index 46710d684..2ee06792a 100644 --- a/backend/backend/urls.py +++ b/backend/backend/urls.py @@ -26,5 +26,6 @@ url(r'^accounts/', include('registration.backends.simple.urls')), path('user/', include('profile_page.urls')), path('', include('device_registry.urls')), + path('monitoring/', include('monitoring.urls')), url('', include('django_prometheus.urls')) ] diff --git a/backend/backend/wsgi.py b/backend/backend/wsgi.py index 442444528..60ea5bd16 100644 --- a/backend/backend/wsgi.py +++ b/backend/backend/wsgi.py @@ -14,7 +14,7 @@ from .utils import ensure_connection_with_retries -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'backend.settings') +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'backend.settings.base') # Patch the standard django database connection class' method in order to try to # connect to DB multiple times using exponential backoff algorithm until success diff --git a/backend/device_registry/tasks.py b/backend/device_registry/tasks.py new file mode 100644 index 000000000..39dca42e0 --- /dev/null +++ b/backend/device_registry/tasks.py @@ -0,0 +1,7 @@ +# Create your tasks here +from celery import shared_task + + +# @shared_task +# def add(x, y): +# return x + y diff --git a/backend/monitoring/__init__.py b/backend/monitoring/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/monitoring/admin.py b/backend/monitoring/admin.py new file mode 100644 index 000000000..8c38f3f3d --- /dev/null +++ b/backend/monitoring/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/backend/monitoring/apps.py b/backend/monitoring/apps.py new file mode 100644 index 000000000..f27ffed8c --- /dev/null +++ b/backend/monitoring/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class MonitoringConfig(AppConfig): + name = 'monitoring' diff --git a/backend/monitoring/migrations/0001_initial.py b/backend/monitoring/migrations/0001_initial.py new file mode 100644 index 000000000..0da7e4dce --- /dev/null +++ b/backend/monitoring/migrations/0001_initial.py @@ -0,0 +1,21 @@ +# Generated by Django 2.1.10 on 2019-08-28 12:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='CeleryPulseTimestamp', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('timestamp', models.DateTimeField(auto_now=True)), + ], + ), + ] diff --git a/backend/monitoring/migrations/__init__.py b/backend/monitoring/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/monitoring/models.py b/backend/monitoring/models.py new file mode 100644 index 000000000..3f58f975b --- /dev/null +++ b/backend/monitoring/models.py @@ -0,0 +1,5 @@ +from django.db import models + + +class CeleryPulseTimestamp(models.Model): + timestamp = models.DateTimeField(auto_now=True) diff --git a/backend/monitoring/tasks.py b/backend/monitoring/tasks.py new file mode 100644 index 000000000..0f43a64ec --- /dev/null +++ b/backend/monitoring/tasks.py @@ -0,0 +1,12 @@ +from celery import shared_task + +from .models import CeleryPulseTimestamp + + +@shared_task +def update_celery_pulse_timestamp(): + pulse_obj = CeleryPulseTimestamp.objects.order_by('id').first() + if pulse_obj is None: + CeleryPulseTimestamp.objects.create() + else: + pulse_obj.save() diff --git a/backend/monitoring/urls.py b/backend/monitoring/urls.py new file mode 100644 index 000000000..81e6a90a8 --- /dev/null +++ b/backend/monitoring/urls.py @@ -0,0 +1,7 @@ +from django.urls import path + +from .views import CeleryPulseTimestampView + +urlpatterns = [ + path('celery/', CeleryPulseTimestampView.as_view(), name='celery_pulse') +] diff --git a/backend/monitoring/views.py b/backend/monitoring/views.py new file mode 100644 index 000000000..fc58b340d --- /dev/null +++ b/backend/monitoring/views.py @@ -0,0 +1,21 @@ +from django.http import HttpResponse +from django.views.generic import View +from django.contrib.auth.mixins import AccessMixin, LoginRequiredMixin + +from .models import CeleryPulseTimestamp + + +class StaffuserRequiredMixin(AccessMixin): + """Verify that the current user has `is_staff` set to True.""" + + def dispatch(self, request, *args, **kwargs): + if not request.user.is_staff: + return self.handle_no_permission() + return super().dispatch(request, *args, **kwargs) + + +class CeleryPulseTimestampView(LoginRequiredMixin, StaffuserRequiredMixin, View): + + def get(self, request, *args, **kwargs): + pulse_obj = CeleryPulseTimestamp.objects.order_by('id').first() + return HttpResponse(str(pulse_obj.timestamp) if pulse_obj else 'None', content_type='text/plain') diff --git a/docker-compose.yml b/docker-compose.yml index 542013ec4..7f6b91723 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,24 +3,23 @@ services: migrate: command: python manage.py migrate --noinput environment: - - DJANGO_SETTINGS_MODULE=backend.settings.dev - - DB_PASSWORD=SuperSecurePassword - - DB_USER=postgres + - DJANGO_SETTINGS_MODULE=backend.settings.dev + - DB_PASSWORD=SuperSecurePassword + - DB_USER=postgres build: context: . volumes: - ./backend:/usr/src/app depends_on: - - psql - - nginx-static + - psql dash-dev: hostname: dash-dev0 environment: - - DJANGO_SETTINGS_MODULE=backend.settings.dev - - DB_PASSWORD=SuperSecurePassword - - DB_USER=postgres - - DATASTORE_KEY_JSON + - DJANGO_SETTINGS_MODULE=backend.settings.dev + - DB_PASSWORD=SuperSecurePassword + - DB_USER=postgres + - DATASTORE_KEY_JSON build: context: . volumes: @@ -28,17 +27,17 @@ services: ports: - '8000:8000' depends_on: - - psql - - migrate - - nginx-static + - psql + - migrate + - nginx-static api-dev: hostname: api-dev0 environment: - - DJANGO_SETTINGS_MODULE=backend.settings.dev - - DB_PASSWORD=SuperSecurePassword - - DB_USER=postgres - - DATASTORE_KEY_JSON + - DJANGO_SETTINGS_MODULE=backend.settings.dev + - DB_PASSWORD=SuperSecurePassword + - DB_USER=postgres + - DATASTORE_KEY_JSON image: api:latest build: context: . @@ -47,16 +46,16 @@ services: ports: - '8001:8000' depends_on: - - psql - - migrate + - psql + - migrate mtls-api-dev: hostname: mtls-api-dev0 environment: - - DJANGO_SETTINGS_MODULE=backend.settings.dev - - DB_PASSWORD=SuperSecurePassword - - DB_USER=postgres - - DATASTORE_KEY_JSON + - DJANGO_SETTINGS_MODULE=backend.settings.dev + - DB_PASSWORD=SuperSecurePassword + - DB_USER=postgres + - DATASTORE_KEY_JSON build: context: . volumes: @@ -64,8 +63,8 @@ services: ports: - '8002:8000' depends_on: - - psql - - migrate + - psql + - migrate nginx-static: hostname: wott-static @@ -80,11 +79,58 @@ services: psql: environment: - - POSTGRES_DB=wott-backend - - POSTGRES_PASSWORD=SuperSecurePassword + - POSTGRES_DB=wott-backend + - POSTGRES_PASSWORD=SuperSecurePassword image: postgres:alpine volumes: - - db-data:/var/lib/postgresql/data + - db-data:/var/lib/postgresql/data + + rabbit: + image: rabbitmq:3-alpine + environment: + - RABBITMQ_DEFAULT_USER=user + - RABBITMQ_DEFAULT_PASS=SuperSecurePassword + - RABBITMQ_DEFAULT_VHOST=wott-dash + ports: + - '5672:5672' + + celery: + environment: + - DJANGO_SETTINGS_MODULE=backend.settings.dev + - DB_PASSWORD=SuperSecurePassword + - DB_USER=postgres + - RABBIT_HOST=rabbit + - RABBIT_USER=user + - RABBIT_PASSWORD=SuperSecurePassword + - RABBIT_VHOST=wott-dash + build: + context: . + command: celery -A backend worker -l error --time-limit=90 --soft-time-limit=60 + volumes: + - ./backend:/usr/src/app + depends_on: + - psql + - migrate + - rabbit + + celery-beat: + environment: + - DJANGO_SETTINGS_MODULE=backend.settings.dev + - DB_PASSWORD=SuperSecurePassword + - DB_USER=postgres + - RABBIT_HOST=rabbit + - RABBIT_USER=user + - RABBIT_PASSWORD=SuperSecurePassword + - RABBIT_VHOST=wott-dash + build: + context: . + command: celery -A backend beat -l error + volumes: + - ./backend:/usr/src/app + depends_on: + - psql + - migrate + - rabbit volumes: db-data: diff --git a/helm/api/templates/deployment-celery-beat.yaml b/helm/api/templates/deployment-celery-beat.yaml new file mode 100644 index 000000000..3deeec553 --- /dev/null +++ b/helm/api/templates/deployment-celery-beat.yaml @@ -0,0 +1,74 @@ +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: {{ .Values.service.celery-beat.name }} + namespace: {{ .Values.namespace }} + labels: + app: {{ .Values.service.celery-beat.name }} + chart: {{ include "api.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Values.service.celery-beat.name }} + release: {{ .Release.Name }} + template: + metadata: + labels: + app: {{ .Values.service.celery-beat.name }} + release: {{ .Release.Name }} + spec: + containers: + - name: {{ .Values.service.celery-beat.name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: "celery -A backend beat -l error" + env: + - name: RELEASE_TRIGGER + value: "{{ .Values.releaseTimeStamp }}" + - name: DEBUG + value: "{{ .Values.service.api.debug }}" + - name: SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.apiSecrets }} + key: djangoSecretKey + - name: DB_HOST + value: "{{ .Values.databaseHost }}" + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.apiSecrets }} + key: databasePassword + - name: SENTRY_DSN + valueFrom: + secretKeyRef: + name: {{ .Values.apiSecrets }} + key: sentryDsn + - name: RABBIT_HOST + value: {{ .Values.rabbitHost }} + - name: RABBIT_USER + value: {{ .Values.rabbitUser }} + - name: RABBIT_VHOST + value: {{ .Values.rabbitVhost }} + - name: RABBIT_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.apiSecrets }} + key: rabbitPassword + resources: +{{ toYaml .Values.resources.api | indent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} + {{- end }} diff --git a/helm/api/templates/deployment-celery.yaml b/helm/api/templates/deployment-celery.yaml new file mode 100644 index 000000000..623a2cb83 --- /dev/null +++ b/helm/api/templates/deployment-celery.yaml @@ -0,0 +1,74 @@ +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: {{ .Values.service.celery.name }} + namespace: {{ .Values.namespace }} + labels: + app: {{ .Values.service.celery.name }} + chart: {{ include "api.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Values.service.celery.name }} + release: {{ .Release.Name }} + template: + metadata: + labels: + app: {{ .Values.service.celery.name }} + release: {{ .Release.Name }} + spec: + containers: + - name: {{ .Values.service.celery.name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: "celery -A backend worker -l error --time-limit=90 --soft-time-limit=60" + env: + - name: RELEASE_TRIGGER + value: "{{ .Values.releaseTimeStamp }}" + - name: DEBUG + value: "{{ .Values.service.api.debug }}" + - name: SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.apiSecrets }} + key: djangoSecretKey + - name: DB_HOST + value: "{{ .Values.databaseHost }}" + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.apiSecrets }} + key: databasePassword + - name: SENTRY_DSN + valueFrom: + secretKeyRef: + name: {{ .Values.apiSecrets }} + key: sentryDsn + - name: RABBIT_HOST + value: {{ .Values.rabbitHost }} + - name: RABBIT_USER + value: {{ .Values.rabbitUser }} + - name: RABBIT_VHOST + value: {{ .Values.rabbitVhost }} + - name: RABBIT_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.apiSecrets }} + key: rabbitPassword + resources: +{{ toYaml .Values.resources.api | indent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} + {{- end }} diff --git a/helm/api/values.yaml b/helm/api/values.yaml index 1e1f1b67a..b0e6eaa70 100644 --- a/helm/api/values.yaml +++ b/helm/api/values.yaml @@ -30,6 +30,12 @@ service: externalPort: 80 internalPort: 8000 debug: 0 + celery: + name: "celery" + debug: 0 + celery-beat: + name: "celery-beat" + debug: 0 static: name: "static" externalPort: 80 @@ -42,6 +48,10 @@ namespace: api databaseHost: psql0-gcloud-sqlproxy.sqlproxy caHost: asgard.us-central1-c.c.wott-prod.internal +rabbitHost: rabbit0-rabbitmq.rabbit +rabbitUser: user +rabbitVhost: wott-dash + ingress: enabled: true annotations: diff --git a/helm/misc/prod-secrets.yaml b/helm/misc/prod-secrets.yaml index d65dced9c..9e5955818 100644 Binary files a/helm/misc/prod-secrets.yaml and b/helm/misc/prod-secrets.yaml differ diff --git a/requirements.txt b/requirements.txt index 99fd3a19e..2febbcc59 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ asn1crypto==0.24.0 cachetools==3.1.1 +celery==4.3.0 certifi==2018.11.29 cffi==1.11.5 cfssl==0.0.3b243