adding: celery

This commit is contained in:
Павел Лопатин 2024-08-13 00:45:14 +03:00
parent d2da446009
commit 2d88f3c0c5
16 changed files with 161 additions and 9 deletions

View File

@ -12,3 +12,10 @@ DB_PORT=5432
REDIS_HOST=redis
REDIS_PORT=6379
REDIS_PASSWORD=redis
RABBIT_HOST=rabbitmq
RABBIT_VHOST=/
RABBIT_PORT=5672
RABBIT_PORT_API=15672
RABBIT_LOGIN=admin
RABBIT_PASSWORD=admin

View File

@ -1,13 +1,20 @@
# simpliest django(uvicorn)+postgresql+fastapi+redis+nginx docker-compose (ready for production and dev)
To run:
`docker-compose up -d`
Site available on 8000 port.
You can make any changes in code, they will appear automatically. If you want to execute something with manage.py use:
```
docker-compose exec app python3 manage.py migrate
docker-compose exec app python3 manage.py makemigrations
docker-compose exec app python3 manage.py update_admin admin adminpass # create superuser
```
and so on.
Example task [task_example.py](src/application/tasks/task_example.py)
Example register task [__init__.py](src/application/tasks/__init__.py)

View File

@ -59,8 +59,56 @@ services:
- redis-data:/data
restart: on-failure
rabbitmq:
image: rabbitmq:3.13.2-management
environment:
RABBITMQ_DEFAULT_USER: ${RABBIT_LOGIN}
RABBITMQ_DEFAULT_PASS: ${RABBIT_PASSWORD}
RABBITMQ_DEFAULT_PORT: ${RABBIT_PORT}
RABBITMQ_DEFAULT_VHOST: ${RABBIT_VHOST}
restart: always
volumes:
- rabbitmq_data:/var/lib/rabbitmq
ports:
- "5672:5672"
flower:
image: mher/flower
hostname: flower
command: [
"celery",
"--broker=amqp://admin:admin@rabbitmq:5672//",
"flower",
"--broker-api=http://admin:admin@rabbitmq:15672/api/",
"--url_prefix=/flower"
]
depends_on:
- redis
- app
- celery_worker
- rabbitmq
volumes:
- flower_data:/data
restart: on-failure
celery_worker:
build:
context: .
command: celery -A core.celery_app worker --loglevel=info
volumes:
- ./src/:/app/
depends_on:
- app
- redis
- rabbitmq
- db
env_file:
- .env
volumes:
flower_data:
postgresql-data:
rabbitmq_data:
static:
redis-data:
external: false

View File

@ -27,6 +27,9 @@ http {
server api:8000;
}
upstream flower {
server flower:5555;
}
server {
listen 8000;
@ -48,6 +51,15 @@ http {
proxy_pass http://app;
}
location /flower/ {
proxy_redirect off;
proxy_set_header Host app;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $server_name;
proxy_pass http://flower;
}
location / {
proxy_redirect off;
proxy_set_header Host app;

View File

@ -1,6 +1,6 @@
from django.apps import AppConfig
class MainConfig(AppConfig):
class ApplicationConfig(AppConfig):
default_auto_field = 'django.db.models.BigAutoField'
name = 'application'

View File

@ -0,0 +1,15 @@
from django.core.management.base import BaseCommand
from core import celery_app
from helpers.const.tasks import EXAMPLE_TASK_NAME
class Command(BaseCommand):
def handle(self, *args, **options):
celery_app.send_task(
name=EXAMPLE_TASK_NAME,
kwargs={
"x": 2,
"y": 3,
}
)

View File

@ -0,0 +1,4 @@
from core import celery_app
from .task_example import Example
celery_app.register_task(Example())

View File

@ -0,0 +1,13 @@
from abc import ABC, abstractmethod
from celery import Task
class ProcessingQueue(ABC, Task):
@abstractmethod
def processed_task(self, *args, **kwargs):
pass
def run(self, *args, **kwargs):
return self.processed_task(*args, **kwargs)

View File

@ -0,0 +1,12 @@
from application.tasks.abstract.base_task import ProcessingQueue
from helpers.const.tasks import EXAMPLE_TASK_NAME
class Example(ProcessingQueue):
name = EXAMPLE_TASK_NAME
max_retries = 5
default_retry_delay = 5
autoretry_for = (Exception,)
def processed_task(self, x: int | float, y: int | float):
return x + y

View File

@ -0,0 +1,3 @@
from .celery_app import app as celery_app
__all__ = ('celery_app',)

14
src/core/celery_app.py Normal file
View File

@ -0,0 +1,14 @@
import os
from celery import Celery
from django.conf import settings
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings')
app = Celery(
'core',
broker=settings.CELERY_BROKER_URL,
)
app.config_from_object('django.conf:settings', namespace='CELERY')
app.autodiscover_tasks()

View File

@ -12,10 +12,6 @@ ALLOWED_HOSTS = os.getenv("ALLOWED_HOSTS", "*").split(" ")
CSRF_TRUSTED_ORIGINS = os.getenv("CSRF_TRUSTED_ORIGINS", "http://* https://*").split(" ")
REDIS_HOST = os.getenv("REDIS_HOST")
REDIS_PORT = int(os.getenv("REDIS_PORT"))
REDIS_PASSWORD = os.getenv("REDIS_PASSWORD")
INSTALLED_APPS = [
'django.contrib.admin',
'django.contrib.auth',
@ -23,7 +19,9 @@ INSTALLED_APPS = [
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'application'
'application.apps.ApplicationConfig',
'djangoql',
'django_celery_results'
]
MIDDLEWARE = [
@ -98,3 +96,18 @@ STATIC_ROOT = os.path.join(BASE_DIR, 'static')
DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
os.environ["DJANGO_ALLOW_ASYNC_UNSAFE"] = "true"
REDIS_HOST = os.getenv("REDIS_HOST")
REDIS_PORT = int(os.getenv("REDIS_PORT"))
REDIS_PASSWORD = os.getenv("REDIS_PASSWORD")
RABBIT_HOST = os.getenv("RABBIT_HOST")
RABBIT_PORT = int(os.getenv("RABBIT_PORT"))
RABBIT_LOGIN = os.getenv("RABBIT_LOGIN")
RABBIT_PASSWORD = os.getenv("RABBIT_PASSWORD")
RABBIT_VHOST = os.getenv("RABBIT_VHOST")
CELERY_BROKER_URL = f'amqp://{RABBIT_LOGIN}:{RABBIT_PASSWORD}@{RABBIT_HOST}:{RABBIT_PORT}/{RABBIT_VHOST}'
CELERY_RESULT_BACKEND = 'django-db'
CELERY_CACHE_BACKEND = 'django-cache'
CELERY_RESULT_EXTENDED = True

View File

View File

@ -0,0 +1 @@
EXAMPLE_TASK_NAME = "example"

View File

@ -4,3 +4,6 @@ Django==5.0.2
psycopg2
redis==4.6.0
djangoql==0.18.1
celery==5.4.0
django-celery-beat
django-celery-results==2.5.1