name: bigdata-week02

x-spark: &spark
  image: spark:4.1.3-scala2.13-java21-python3-ubuntu
  environment:
    PYSPARK_PYTHON: python3
  init: true
  stop_grace_period: 30s

services:
  spark-master:
    <<: *spark
    hostname: spark-master
    entrypoint: ["/opt/spark/bin/spark-class"]
    command: ["org.apache.spark.deploy.master.Master", "--host", "spark-master", "--port", "7077", "--webui-port", "8080"]
    ports:
      - "127.0.0.1:8080:8080"
    healthcheck:
      test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080', timeout=3)"]
      interval: 5s
      timeout: 5s
      retries: 20
      start_period: 20s

  spark-worker-1:
    <<: *spark
    hostname: spark-worker-1
    entrypoint: ["/opt/spark/bin/spark-class"]
    command: ["org.apache.spark.deploy.worker.Worker", "--host", "spark-worker-1", "--cores", "1", "--memory", "1g", "--webui-port", "8081", "--work-dir", "/tmp/spark-work", "spark://spark-master:7077"]
    ports:
      - "127.0.0.1:8081:8081"
    depends_on:
      spark-master:
        condition: service_healthy

  spark-worker-2:
    <<: *spark
    hostname: spark-worker-2
    entrypoint: ["/opt/spark/bin/spark-class"]
    command: ["org.apache.spark.deploy.worker.Worker", "--host", "spark-worker-2", "--cores", "1", "--memory", "1g", "--webui-port", "8081", "--work-dir", "/tmp/spark-work", "spark://spark-master:7077"]
    ports:
      - "127.0.0.1:8082:8081"
    depends_on:
      spark-master:
        condition: service_healthy

  spark-client:
    <<: *spark
    hostname: spark-client
    entrypoint: ["/bin/bash", "-lc"]
    command: ["exec sleep infinity"]
    ports:
      - "127.0.0.1:4040:4040"
    volumes:
      - ./apps:/opt/spark/apps:ro
    depends_on:
      spark-master:
        condition: service_healthy