I am trying to write a background task to count the total number of posts. It works fine when I start redis server and celery without using docker but using docker, the tasks state is always pending.

To confirm if the task is working, I tried to do it without using docker and it worked fine but as soon I can run it using docker the task stops working.

Below is the code I tried:

Dockerfile

FROM python:3.8-slim-buster
ENV PYTHONUNBUFFERED=1
WORKDIR /code
COPY requirements.txt requirements.txt
RUN pip3 install -r requirements.txt
RUN pip3 install ipdb

COPY . /code/

docker-compose.yml

version: '3'

services:
  db:
    image: postgres:13
    volumes:
      - postgres_data:/var/lib/postgresql/data
    environment:
      POSTGRES_DB: myproject
      POSTGRES_USER: myprojectuser
      POSTGRES_PASSWORD: password
    networks:
      - mynetwork  

  web:
    build: .
    command: bash -c "
          python manage.py migrate
          && python manage.py runserver 0.0.0.0:8000
        "
    volumes:
      - .:/code
    ports:
      - "8000:8000"
    depends_on:
      - db
      - es
      - redis
      - celery
    environment:
      - DATABASE_URL=postgres://myprojectuser:password@db:5432/myproject
    stdin_open: true  # Keep stdin open
    tty: true  # Enable TTY
    networks:
      - mynetwork

  es:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.13.0
    volumes:
      - esdata01:/usr/share/elasticsearch/data
    ports:
      - 9200:9200
    environment:
      - discovery.type=single-node
      - xpack.security.enabled=false
    networks:
      - mynetwork  

  redis:
    image: redis:latest
    container_name: rd01
    command: redis-server
    ports:
      - '6379:6379'
    networks:
      - mynetwork  

  celery:
    build: .
    command: celery -A core worker -l info
    volumes:
      - .:/code
    depends_on:
      - redis
    networks:
      - mynetwork  
    environment:
      - CELERY_BROKER=redis://redis:6379/0
      - CELERY_RESULT_BACKEND=redis://redis:6379/0

  celery-beat:
    build: .
    command: celery -A core beat --loglevel=info
    volumes:
      - .:/code
    depends_on:
      - redis
      - celery
    networks:
      - mynetwork

volumes:
  postgres_data:
  esdata01:
networks:
  mynetwork:
    driver: bridge

tasks.py

# myapp/tasks.py
import logging
from celery import shared_task
from .models import Post  # Ensure you import the Post model
import pdb

print("Tasks module loaded")

# logger = logging.getLogger(__name__)

@shared_task
def count_posts():
    print("count_posts task registered") 
    count_posts = Post.objects.count()
    return count_posts

views.py

def home(request):
    result = count_posts.delay()
    time.sleep(1)
    async_result = AsyncResult(result.id)
    
    if async_result.ready():
        total_posts = async_result.get()
        return JsonResponse({'total_posts': total_posts})
    else:
        return JsonResponse({'status': 'pending'})

settings.py

CELERY_BROKER_URL = 'redis://redis:6379/0'
CELERY_RESULT_BACKEND = 'redis://redis:6379/0'
# CELERY_CACHE_BACKEND = 'django-cache'
CELERY_ACCEPT_CONTENT = ['pickle']
CELERY_TASK_SERIALIZER = 'pickle'
CELERY_RESULT_SERIALIZER = 'pickle'
CELERY_TIMEZONE = "UTC"
# CELERY_TASK_TRACK_STARTED = True