I am using docker to setup spark-master, spark-worker and spark-history-server.

For some reason, I am not able to see any logs on history-server although server is running and log directory is mentioned.

/tmp/spark-events is created and rwx permissions are given to the user while building the image.

Below is my docker compose for spark services.

services:
  spark-master:
    <<: *spark-common
    command: bin/spark-class org.apache.spark.deploy.master.Master
    ports:
      - "8081:8080"
      - "7077:7077"
  spark-worker-1:
    <<: *spark-common
    command: bin/spark-class org.apache.spark.deploy.worker.Worker spark://spark-master:7077
    depends_on:
      - spark-master
    environment:
      SPARK_MODE: worker
      SPARK_WORKER_CORES: 1
      SPARK_WORKER_MEMORY: 1g
      SPARK_MASTER_URL: spark://spark-master:7077
      SPARK_LOCAL_IP: "0.0.0.0"
      SPARK_WORKER_PORT: 8081
      SPARK_WORKER_WEBUI_PORT: 8090
    ports:
        - "8082:8081"
        - "8090:8090"

  spark-history-server:
    <<: *spark-common
    command: bin/spark-class org.apache.spark.deploy.history.HistoryServer
    environment:
      - SPARK_EVENTLOG_ENABLED=true
      - SPARK_HISTORY_FS_LOGDIRECTORY=/tmp/spark-events 
      - SPARK_EVENTLOG_DIR=/tmp/spark-events
      - SPARK_DAEMON_MEMORY=1g
      - SPARK_HISTORY_RETAINEDAPPLICATIONS=100
      - SPARK_HISTORY_UI_MAXAPPLICATIONS=50
      - SPARK_HISTORY_STORE_MAXDISKUSAGE=1g
      - SPARK_HISTORY_UI_PORT=18080
    env_file:
      - .env.spark
    volumes:
      - spark-events:/tmp/spark-events
    expose:
      - 18080
      - 4040      
    ports:
      - "18080:18080"
      - 4040:4040

volumes:
  spark-events:
    driver: local