name: bigdata-week03-scale

x-spark: &spark
  image: spark:4.1.3-scala2.13-java21-python3-ubuntu
  environment:
    PYSPARK_PYTHON: python3
  init: true
  stop_grace_period: 30s

services:
  spark-master:
    <<: *spark
    hostname: spark-master
    entrypoint: ["/opt/spark/bin/spark-class"]
    command: ["org.apache.spark.deploy.master.Master", "--host", "spark-master", "--port", "7077", "--webui-port", "8080"]
    ports:
      - "127.0.0.1:18080:8080"
    healthcheck:
      test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080', timeout=3)"]
      interval: 5s
      timeout: 5s
      retries: 20
      start_period: 20s

  spark-worker:
    <<: *spark
    entrypoint: ["/bin/bash", "-lc"]
    command:
      - >-
        exec /opt/spark/bin/spark-class
        org.apache.spark.deploy.worker.Worker
        --host "$$HOSTNAME"
        --cores 1
        --memory 1g
        --webui-port 8081
        --work-dir /tmp/spark-work
        spark://spark-master:7077
    depends_on:
      spark-master:
        condition: service_healthy

  spark-client:
    <<: *spark
    hostname: spark-client
    entrypoint: ["/bin/bash", "-lc"]
    command: ["exec sleep infinity"]
    ports:
      - "127.0.0.1:24040:4040"
    volumes:
      - ./apps:/opt/spark/apps:ro
    depends_on:
      spark-master:
        condition: service_healthy