Wiring the Mini-Platform

# Defines the whole local data platform. Bring it up with: docker compose up -d
services:

  # ── SOURCE: operational database we ingest from ───────────────────
  postgres:
    image: postgres:16
    env_file: .env                      # POSTGRES_* read from the environment, not hardcoded
    ports:
      - "5432:5432"                     # publish to your laptop for psql access
    volumes:
      - pgdata:/var/lib/postgresql/data # persist tables across restarts
    networks: [platform]
    healthcheck:                        # "is it actually ready?" — others wait on this
      test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER}"]
      interval: 5s
      timeout: 3s
      retries: 10

  # ── STREAM: Kafka-compatible broker ───────────────────────────────
  redpanda:
    image: redpandadata/redpanda:v24.2.7
    command:
      - redpanda
      - start
      - --smp=1
      - --overprovisioned
      - --kafka-addr=PLAINTEXT://0.0.0.0:9092
      - --advertise-kafka-addr=PLAINTEXT://redpanda:9092
    ports:
      - "9092:9092"
    volumes:
      - redpandadata:/var/lib/redpanda/data
    networks: [platform]
    healthcheck:
      test: ["CMD-SHELL", "rpk cluster health | grep -q 'Healthy:.*true'"]
      interval: 10s
      timeout: 5s
      retries: 10

  # ── ORCHESTRATION: runs jobs + serves the web UI ──────────────────
  # DuckDB runs IN-PROCESS inside this container — note the shared 'warehouse' volume.
  dagster:
    image: python:3.12-slim
    command: ["sleep", "infinity"]      # placeholder; replace with: dagster dev -h 0.0.0.0 -p 3000
    env_file: .env
    environment:
      DUCKDB_PATH: /warehouse/warehouse.duckdb   # the in-process warehouse file
    ports:
      - "3000:3000"                     # Dagster web UI
    volumes:
      - ./dagster:/opt/dagster          # your Dagster project code
      - ./dbt:/opt/dbt                  # your dbt project (runs against DuckDB)
      - warehouse:/warehouse            # the DuckDB file lives here, shared
    networks: [platform]
    depends_on:
      postgres:
        condition: service_healthy      # don't start jobs before the source is ready
      redpanda:
        condition: service_healthy

  # ── BI: charts & SQL over the warehouse ───────────────────────────
  metabase:
    image: metabase/metabase:v0.50.21
    ports:
      - "3001:3000"                     # Metabase serves on 3000 inside; we publish 3001
    volumes:
      - warehouse:/warehouse            # read the same DuckDB file
      - metabasedata:/metabase-data
    environment:
      MB_DB_FILE: /metabase-data/metabase.db
    networks: [platform]
    depends_on:
      postgres:
        condition: service_healthy

  # ── OBJECT STORAGE (optional): S3-compatible bucket ───────────────
  minio:
    image: minio/minio:RELEASE.2024-09-13T20-26-02Z
    command: server /data --console-address ":9001"
    env_file: .env                      # MINIO_ROOT_USER / MINIO_ROOT_PASSWORD
    ports:
      - "9000:9000"                     # S3 API
      - "9001:9001"                     # web console
    volumes:
      - miniodata:/data
    networks: [platform]

# One private network so services reach each other by name (e.g. host "postgres").
networks:
  platform:
    driver: bridge

# Named volumes survive `docker compose down` — your data isn't lost on restart.
volumes:
  pgdata:
  redpandadata:
  warehouse:
  metabasedata:
  miniodata:

# ── Postgres (the source database) ────────────────────────────────
POSTGRES_USER=platform
POSTGRES_PASSWORD=dev_only_change_me
POSTGRES_DB=appdb

# ── How services address each other on the shared network ─────────
# (use the service NAME as host, not localhost — containers aren't on your laptop's network)
POSTGRES_HOST=postgres
REDPANDA_BROKER=redpanda:9092

# ── DuckDB warehouse file (in-process) ────────────────────────────
DUCKDB_PATH=/warehouse/warehouse.duckdb

# ── MinIO (optional object storage) ───────────────────────────────
MINIO_ROOT_USER=minio
MINIO_ROOT_PASSWORD=dev_only_change_me

# secrets & local config — NEVER commit
.env
*.env.local

# local data & state
*.duckdb
data/
.dagster/

POSTGRES_USER=platform
POSTGRES_PASSWORD=changeme
POSTGRES_DB=appdb
POSTGRES_HOST=postgres
REDPANDA_BROKER=redpanda:9092
DUCKDB_PATH=/warehouse/warehouse.duckdb
MINIO_ROOT_USER=minio
MINIO_ROOT_PASSWORD=changeme

cd ~/mini-platform

# pull images and start everything in the background (-d = detached)
docker compose up -d

# watch the services come up; STATUS should move to "healthy" / "running"
docker compose ps

# tail the logs of one service while it boots (Ctrl-C to stop watching)
docker compose logs -f dagster

# 1) psql INTO Postgres from your laptop (password is in your .env)
docker compose exec postgres psql -U platform -d appdb -c "SELECT 'source db up' AS status;"

# 2) confirm Redpanda's cluster is healthy
docker compose exec redpanda rpk cluster health

# 3) open the web UIs in your browser
open http://localhost:3000    # Dagster  (use xdg-open on Linux)
open http://localhost:3001    # Metabase (first visit walks you through setup)
open http://localhost:9001    # MinIO console (optional)