From 2d88f3c0c5280058955639b12dc7338867f80fef Mon Sep 17 00:00:00 2001 From: Pavel Date: Tue, 13 Aug 2024 00:45:14 +0300 Subject: [PATCH] adding: celery --- .env.example | 9 +++- README.md | 7 +++ docker-compose.yaml | 48 +++++++++++++++++++ nginx/nginx.conf | 14 +++++- src/application/apps.py | 2 +- .../management/commands/init_task.py | 15 ++++++ src/application/tasks/__init__.py | 4 ++ src/application/tasks/abstract/__init__.py | 0 src/application/tasks/abstract/base_task.py | 13 +++++ src/application/tasks/task_example.py | 12 +++++ src/core/__init__.py | 3 ++ src/core/celery_app.py | 14 ++++++ src/core/settings.py | 23 +++++++-- src/helpers/const/__init__.py | 0 src/helpers/const/tasks.py | 1 + src/requirements.txt | 5 +- 16 files changed, 161 insertions(+), 9 deletions(-) create mode 100644 src/application/management/commands/init_task.py create mode 100644 src/application/tasks/__init__.py create mode 100644 src/application/tasks/abstract/__init__.py create mode 100644 src/application/tasks/abstract/base_task.py create mode 100644 src/application/tasks/task_example.py create mode 100644 src/core/celery_app.py create mode 100644 src/helpers/const/__init__.py create mode 100644 src/helpers/const/tasks.py diff --git a/.env.example b/.env.example index d6dec62..0720ed0 100644 --- a/.env.example +++ b/.env.example @@ -11,4 +11,11 @@ DB_PORT=5432 REDIS_HOST=redis REDIS_PORT=6379 -REDIS_PASSWORD=redis \ No newline at end of file +REDIS_PASSWORD=redis + +RABBIT_HOST=rabbitmq +RABBIT_VHOST=/ +RABBIT_PORT=5672 +RABBIT_PORT_API=15672 +RABBIT_LOGIN=admin +RABBIT_PASSWORD=admin diff --git a/README.md b/README.md index 6d687eb..e1afff7 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,20 @@ # simpliest django(uvicorn)+postgresql+fastapi+redis+nginx docker-compose (ready for production and dev) + To run: `docker-compose up -d` Site available on 8000 port. You can make any changes in code, they will appear automatically. If you want to execute something with manage.py use: + ``` docker-compose exec app python3 manage.py migrate docker-compose exec app python3 manage.py makemigrations docker-compose exec app python3 manage.py update_admin admin adminpass # create superuser ``` + and so on. + +Example task [task_example.py](src/application/tasks/task_example.py) + +Example register task [__init__.py](src/application/tasks/__init__.py) diff --git a/docker-compose.yaml b/docker-compose.yaml index f53ea85..9475e08 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -59,8 +59,56 @@ services: - redis-data:/data restart: on-failure + rabbitmq: + image: rabbitmq:3.13.2-management + environment: + RABBITMQ_DEFAULT_USER: ${RABBIT_LOGIN} + RABBITMQ_DEFAULT_PASS: ${RABBIT_PASSWORD} + RABBITMQ_DEFAULT_PORT: ${RABBIT_PORT} + RABBITMQ_DEFAULT_VHOST: ${RABBIT_VHOST} + restart: always + volumes: + - rabbitmq_data:/var/lib/rabbitmq + ports: + - "5672:5672" + + flower: + image: mher/flower + hostname: flower + command: [ + "celery", + "--broker=amqp://admin:admin@rabbitmq:5672//", + "flower", + "--broker-api=http://admin:admin@rabbitmq:15672/api/", + "--url_prefix=/flower" + ] + depends_on: + - redis + - app + - celery_worker + - rabbitmq + volumes: + - flower_data:/data + restart: on-failure + + celery_worker: + build: + context: . + command: celery -A core.celery_app worker --loglevel=info + volumes: + - ./src/:/app/ + depends_on: + - app + - redis + - rabbitmq + - db + env_file: + - .env + volumes: + flower_data: postgresql-data: + rabbitmq_data: static: redis-data: external: false \ No newline at end of file diff --git a/nginx/nginx.conf b/nginx/nginx.conf index abd076a..6db1f66 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -27,11 +27,14 @@ http { server api:8000; } + upstream flower { + server flower:5555; + } server { listen 8000; charset utf-8; - + server_name _; location /static/ { @@ -48,6 +51,15 @@ http { proxy_pass http://app; } + location /flower/ { + proxy_redirect off; + proxy_set_header Host app; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $server_name; + proxy_pass http://flower; + } + location / { proxy_redirect off; proxy_set_header Host app; diff --git a/src/application/apps.py b/src/application/apps.py index 748ca6d..30c0916 100644 --- a/src/application/apps.py +++ b/src/application/apps.py @@ -1,6 +1,6 @@ from django.apps import AppConfig -class MainConfig(AppConfig): +class ApplicationConfig(AppConfig): default_auto_field = 'django.db.models.BigAutoField' name = 'application' diff --git a/src/application/management/commands/init_task.py b/src/application/management/commands/init_task.py new file mode 100644 index 0000000..0552a7f --- /dev/null +++ b/src/application/management/commands/init_task.py @@ -0,0 +1,15 @@ +from django.core.management.base import BaseCommand + +from core import celery_app +from helpers.const.tasks import EXAMPLE_TASK_NAME + + +class Command(BaseCommand): + def handle(self, *args, **options): + celery_app.send_task( + name=EXAMPLE_TASK_NAME, + kwargs={ + "x": 2, + "y": 3, + } + ) diff --git a/src/application/tasks/__init__.py b/src/application/tasks/__init__.py new file mode 100644 index 0000000..7032f9d --- /dev/null +++ b/src/application/tasks/__init__.py @@ -0,0 +1,4 @@ +from core import celery_app +from .task_example import Example + +celery_app.register_task(Example()) diff --git a/src/application/tasks/abstract/__init__.py b/src/application/tasks/abstract/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/application/tasks/abstract/base_task.py b/src/application/tasks/abstract/base_task.py new file mode 100644 index 0000000..a4a2ca8 --- /dev/null +++ b/src/application/tasks/abstract/base_task.py @@ -0,0 +1,13 @@ +from abc import ABC, abstractmethod + +from celery import Task + + +class ProcessingQueue(ABC, Task): + + @abstractmethod + def processed_task(self, *args, **kwargs): + pass + + def run(self, *args, **kwargs): + return self.processed_task(*args, **kwargs) diff --git a/src/application/tasks/task_example.py b/src/application/tasks/task_example.py new file mode 100644 index 0000000..d85fab6 --- /dev/null +++ b/src/application/tasks/task_example.py @@ -0,0 +1,12 @@ +from application.tasks.abstract.base_task import ProcessingQueue +from helpers.const.tasks import EXAMPLE_TASK_NAME + + +class Example(ProcessingQueue): + name = EXAMPLE_TASK_NAME + max_retries = 5 + default_retry_delay = 5 + autoretry_for = (Exception,) + + def processed_task(self, x: int | float, y: int | float): + return x + y diff --git a/src/core/__init__.py b/src/core/__init__.py index e69de29..3a074b7 100644 --- a/src/core/__init__.py +++ b/src/core/__init__.py @@ -0,0 +1,3 @@ +from .celery_app import app as celery_app + +__all__ = ('celery_app',) diff --git a/src/core/celery_app.py b/src/core/celery_app.py new file mode 100644 index 0000000..12fbfd2 --- /dev/null +++ b/src/core/celery_app.py @@ -0,0 +1,14 @@ +import os + +from celery import Celery +from django.conf import settings + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings') + +app = Celery( + 'core', + broker=settings.CELERY_BROKER_URL, +) + +app.config_from_object('django.conf:settings', namespace='CELERY') +app.autodiscover_tasks() diff --git a/src/core/settings.py b/src/core/settings.py index 4b5880a..9b8e75a 100644 --- a/src/core/settings.py +++ b/src/core/settings.py @@ -12,10 +12,6 @@ ALLOWED_HOSTS = os.getenv("ALLOWED_HOSTS", "*").split(" ") CSRF_TRUSTED_ORIGINS = os.getenv("CSRF_TRUSTED_ORIGINS", "http://* https://*").split(" ") -REDIS_HOST = os.getenv("REDIS_HOST") -REDIS_PORT = int(os.getenv("REDIS_PORT")) -REDIS_PASSWORD = os.getenv("REDIS_PASSWORD") - INSTALLED_APPS = [ 'django.contrib.admin', 'django.contrib.auth', @@ -23,7 +19,9 @@ INSTALLED_APPS = [ 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', - 'application' + 'application.apps.ApplicationConfig', + 'djangoql', + 'django_celery_results' ] MIDDLEWARE = [ @@ -98,3 +96,18 @@ STATIC_ROOT = os.path.join(BASE_DIR, 'static') DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' os.environ["DJANGO_ALLOW_ASYNC_UNSAFE"] = "true" + +REDIS_HOST = os.getenv("REDIS_HOST") +REDIS_PORT = int(os.getenv("REDIS_PORT")) +REDIS_PASSWORD = os.getenv("REDIS_PASSWORD") + +RABBIT_HOST = os.getenv("RABBIT_HOST") +RABBIT_PORT = int(os.getenv("RABBIT_PORT")) +RABBIT_LOGIN = os.getenv("RABBIT_LOGIN") +RABBIT_PASSWORD = os.getenv("RABBIT_PASSWORD") +RABBIT_VHOST = os.getenv("RABBIT_VHOST") + +CELERY_BROKER_URL = f'amqp://{RABBIT_LOGIN}:{RABBIT_PASSWORD}@{RABBIT_HOST}:{RABBIT_PORT}/{RABBIT_VHOST}' +CELERY_RESULT_BACKEND = 'django-db' +CELERY_CACHE_BACKEND = 'django-cache' +CELERY_RESULT_EXTENDED = True diff --git a/src/helpers/const/__init__.py b/src/helpers/const/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/helpers/const/tasks.py b/src/helpers/const/tasks.py new file mode 100644 index 0000000..e687899 --- /dev/null +++ b/src/helpers/const/tasks.py @@ -0,0 +1 @@ +EXAMPLE_TASK_NAME = "example" diff --git a/src/requirements.txt b/src/requirements.txt index 8bdb446..84910da 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -3,4 +3,7 @@ fastapi==0.109.0 Django==5.0.2 psycopg2 redis==4.6.0 -djangoql==0.18.1 \ No newline at end of file +djangoql==0.18.1 +celery==5.4.0 +django-celery-beat +django-celery-results==2.5.1 \ No newline at end of file