adding: celery

.env.example

@@ -11,4 +11,11 @@ DB_PORT=5432
 
 REDIS_HOST=redis
 REDIS_PORT=6379
-REDIS_PASSWORD=redis
+REDIS_PASSWORD=redis
+
+RABBIT_HOST=rabbitmq
+RABBIT_VHOST=/
+RABBIT_PORT=5672
+RABBIT_PORT_API=15672
+RABBIT_LOGIN=admin
+RABBIT_PASSWORD=admin

README.md

@@ -1,13 +1,20 @@
 # simpliest django(uvicorn)+postgresql+fastapi+redis+nginx docker-compose (ready for production and dev)
+
 To run:
 `docker-compose up -d`
 
 Site available on 8000 port.
 
 You can make any changes in code, they will appear automatically. If you want to execute something with manage.py use:
+
 ```
 docker-compose exec app python3 manage.py migrate
 docker-compose exec app python3 manage.py makemigrations
 docker-compose exec app python3 manage.py update_admin admin adminpass # create superuser
 ```
+
 and so on.
+
+Example task [task_example.py](src/application/tasks/task_example.py)
+
+Example register task [__init__.py](src/application/tasks/__init__.py)

docker-compose.yaml

@@ -59,8 +59,56 @@ services:
       - redis-data:/data
     restart: on-failure
 
+  rabbitmq:
+    image: rabbitmq:3.13.2-management
+    environment:
+      RABBITMQ_DEFAULT_USER: ${RABBIT_LOGIN}
+      RABBITMQ_DEFAULT_PASS: ${RABBIT_PASSWORD}
+      RABBITMQ_DEFAULT_PORT: ${RABBIT_PORT}
+      RABBITMQ_DEFAULT_VHOST: ${RABBIT_VHOST}
+    restart: always
+    volumes:
+      - rabbitmq_data:/var/lib/rabbitmq
+    ports:
+      - "5672:5672"
+
+  flower:
+    image: mher/flower
+    hostname: flower
+    command: [
+      "celery",
+      "--broker=amqp://admin:admin@rabbitmq:5672//",
+      "flower",
+      "--broker-api=http://admin:admin@rabbitmq:15672/api/",
+      "--url_prefix=/flower"
+    ]
+    depends_on:
+      - redis
+      - app
+      - celery_worker
+      - rabbitmq
+    volumes:
+      - flower_data:/data
+    restart: on-failure
+
+  celery_worker:
+    build:
+      context: .
+    command: celery -A core.celery_app worker --loglevel=info
+    volumes:
+      - ./src/:/app/
+    depends_on:
+      - app
+      - redis
+      - rabbitmq
+      - db
+    env_file:
+      - .env
+
 volumes:
+  flower_data:
   postgresql-data:
+  rabbitmq_data:
   static:
   redis-data:
     external: false

nginx/nginx.conf

@@ -27,11 +27,14 @@ http {
         server api:8000;
     }
 
+    upstream flower {
+        server flower:5555;
+    }
 
     server {
         listen 8000;
         charset utf-8;
-        
+
         server_name _;
 
         location /static/ {
@@ -48,6 +51,15 @@ http {
             proxy_pass http://app;
         }
 
+        location /flower/ {
+            proxy_redirect     off;
+            proxy_set_header   Host app;
+            proxy_set_header   X-Real-IP $remote_addr;
+            proxy_set_header   X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header   X-Forwarded-Host $server_name;
+            proxy_pass http://flower;
+        }
+
         location / {
             proxy_redirect     off;
             proxy_set_header   Host app;

src/application/apps.py

@@ -1,6 +1,6 @@
 from django.apps import AppConfig
 
 
-class MainConfig(AppConfig):
+class ApplicationConfig(AppConfig):
     default_auto_field = 'django.db.models.BigAutoField'
     name = 'application'

src/application/management/commands/init_task.py

@@ -0,0 +1,15 @@
+from django.core.management.base import BaseCommand
+
+from core import celery_app
+from helpers.const.tasks import EXAMPLE_TASK_NAME
+
+
+class Command(BaseCommand):
+    def handle(self, *args, **options):
+        celery_app.send_task(
+            name=EXAMPLE_TASK_NAME,
+            kwargs={
+                "x": 2,
+                "y": 3,
+            }
+        )

src/application/tasks/__init__.py

@@ -0,0 +1,4 @@
+from core import celery_app
+from .task_example import Example
+
+celery_app.register_task(Example())

src/application/tasks/abstract/base_task.py

@@ -0,0 +1,13 @@
+from abc import ABC, abstractmethod
+
+from celery import Task
+
+
+class ProcessingQueue(ABC, Task):
+
+    @abstractmethod
+    def processed_task(self, *args, **kwargs):
+        pass
+
+    def run(self, *args, **kwargs):
+        return self.processed_task(*args, **kwargs)

src/application/tasks/task_example.py

@@ -0,0 +1,12 @@
+from application.tasks.abstract.base_task import ProcessingQueue
+from helpers.const.tasks import EXAMPLE_TASK_NAME
+
+
+class Example(ProcessingQueue):
+    name = EXAMPLE_TASK_NAME
+    max_retries = 5
+    default_retry_delay = 5
+    autoretry_for = (Exception,)
+
+    def processed_task(self, x: int | float, y: int | float):
+        return x + y

src/core/__init__.py

@@ -0,0 +1,3 @@
+from .celery_app import app as celery_app
+
+__all__ = ('celery_app',)

src/core/celery_app.py

@@ -0,0 +1,14 @@
+import os
+
+from celery import Celery
+from django.conf import settings
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings')
+
+app = Celery(
+    'core',
+    broker=settings.CELERY_BROKER_URL,
+)
+
+app.config_from_object('django.conf:settings', namespace='CELERY')
+app.autodiscover_tasks()

src/core/settings.py

@@ -12,10 +12,6 @@ ALLOWED_HOSTS = os.getenv("ALLOWED_HOSTS", "*").split(" ")
 
 CSRF_TRUSTED_ORIGINS = os.getenv("CSRF_TRUSTED_ORIGINS", "http://* https://*").split(" ")
 
-REDIS_HOST = os.getenv("REDIS_HOST")
-REDIS_PORT = int(os.getenv("REDIS_PORT"))
-REDIS_PASSWORD = os.getenv("REDIS_PASSWORD")
-
 INSTALLED_APPS = [
     'django.contrib.admin',
     'django.contrib.auth',
@@ -23,7 +19,9 @@ INSTALLED_APPS = [
     'django.contrib.sessions',
     'django.contrib.messages',
     'django.contrib.staticfiles',
-    'application'
+    'application.apps.ApplicationConfig',
+    'djangoql',
+    'django_celery_results'
 ]
 
 MIDDLEWARE = [
@@ -98,3 +96,18 @@ STATIC_ROOT = os.path.join(BASE_DIR, 'static')
 
 DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
 os.environ["DJANGO_ALLOW_ASYNC_UNSAFE"] = "true"
+
+REDIS_HOST = os.getenv("REDIS_HOST")
+REDIS_PORT = int(os.getenv("REDIS_PORT"))
+REDIS_PASSWORD = os.getenv("REDIS_PASSWORD")
+
+RABBIT_HOST = os.getenv("RABBIT_HOST")
+RABBIT_PORT = int(os.getenv("RABBIT_PORT"))
+RABBIT_LOGIN = os.getenv("RABBIT_LOGIN")
+RABBIT_PASSWORD = os.getenv("RABBIT_PASSWORD")
+RABBIT_VHOST = os.getenv("RABBIT_VHOST")
+
+CELERY_BROKER_URL = f'amqp://{RABBIT_LOGIN}:{RABBIT_PASSWORD}@{RABBIT_HOST}:{RABBIT_PORT}/{RABBIT_VHOST}'
+CELERY_RESULT_BACKEND = 'django-db'
+CELERY_CACHE_BACKEND = 'django-cache'
+CELERY_RESULT_EXTENDED = True

src/helpers/const/tasks.py

@@ -0,0 +1 @@
+EXAMPLE_TASK_NAME = "example"

src/requirements.txt

@@ -3,4 +3,7 @@ fastapi==0.109.0
 Django==5.0.2
 psycopg2
 redis==4.6.0
-djangoql==0.18.1
+djangoql==0.18.1
+celery==5.4.0
+django-celery-beat
+django-celery-results==2.5.1