nooriuta/icebergstremingpoc:compose

# Iceberg Streaming POC — self-contained стенд для шаринга
#
# Образы POC опубликованы на Docker Hub: nooriuta/icebergstremingpoc:{poc1,poc3,poc4s,poc5}
# Инфра (MinIO, Gravitino, Flink, Spark, Trino) — публичные образы.
#
# ── Быстрый старт (инфраструктура):
#   docker compose up -d minio mc gravitino init-tables
#
# ── Batch-бенчмарки POC1 (C) / POC3 (C++):
#   docker compose run --rm -e BENCH_MODE=1 -e N_ROWS=1000 -e N_COLS=5 poc1
#   docker compose run --rm -e BENCH_MODE=1 -e N_ROWS=1000 -e N_COLS=5 poc3
#
# ── POC4s (Flink streaming):
#   docker compose --profile flink up -d flink-jobmanager flink-taskmanager
#   docker compose exec -e EVENT_RATE=1000 -e N_COLS=300 flink-jobmanager \
#     flink run --class aida.iceberg.CdcStreamingBenchmark /opt/flink/usrlib/poc4-flink-iceberg.jar
#
# ── POC5 / POC5ss (Spark):
#   docker compose --profile spark run --rm -e N_ROWS=1000 -e N_COLS=5 poc5
#   docker compose --profile spark run --rm -e EVENT_RATE=1000 -e N_COLS=300 -e TRIGGER_MS=1000 poc5ss
#
# ── UI: MinIO http://localhost:19001 (minioadmin/minioadmin) · Flink http://localhost:28081 · Trino http://localhost:29080

name: icebergpoc

networks:
  icebergpoc:
    driver: bridge

volumes:
  minio-data:

services:

  minio:
    image: minio/minio:latest
    container_name: icebergpoc-minio
    ports:
      - "19000:9000"
      - "19001:9001"
    environment:
      MINIO_ROOT_USER:     minioadmin
      MINIO_ROOT_PASSWORD: minioadmin
    command: server /data --console-address ":9001"
    volumes:
      - minio-data:/data
    networks: [icebergpoc]
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
      interval: 5s
      timeout: 5s
      retries: 10

  mc:
    image: minio/mc:latest
    container_name: icebergpoc-mc
    depends_on:
      minio: { condition: service_healthy }
    entrypoint: >
      /bin/sh -c "
        mc alias set local http://minio:9000 minioadmin minioadmin &&
        mc mb --ignore-existing local/warehouse &&
        echo 'bucket warehouse ready' && exit 0"
    restart: "no"
    networks: [icebergpoc]

  gravitino:
    image: apache/gravitino-iceberg-rest:1.0.1
    container_name: icebergpoc-gravitino
    ports:
      - "29002:9001"   # Iceberg REST → http://localhost:29002/iceberg/v1/config
    depends_on:
      mc: { condition: service_completed_successfully }
    environment:
      GRAVITINO_CATALOG_BACKEND:      memory
      GRAVITINO_WAREHOUSE:            s3://warehouse/gravitino
      GRAVITINO_IO_IMPL:              org.apache.iceberg.aws.s3.S3FileIO
      GRAVITINO_S3_ENDPOINT:          http://minio:9000
      GRAVITINO_S3_ACCESS_KEY:        minioadmin
      GRAVITINO_S3_SECRET_KEY:        minioadmin
      GRAVITINO_S3_REGION:            us-east-1
      GRAVITINO_S3_PATH_STYLE_ACCESS: "true"
    networks: [icebergpoc]
    healthcheck:
      test: ["CMD", "curl", "-sf", "http://localhost:9001/iceberg/v1/config"]
      interval: 10s
      timeout: 5s
      retries: 15
      start_period: 30s

  # Инлайн-инициализация (без bind-mount) — compose остаётся standalone и публикуемым
  init-tables:
    image: curlimages/curl:latest
    container_name: icebergpoc-init
    depends_on:
      gravitino: { condition: service_healthy }
    environment:
      CATALOG_URL: "http://gravitino:9001/iceberg/v1"
    entrypoint:
      - /bin/sh
      - -c
      - |
        B=$$CATALOG_URL
        echo "[init] $$B"
        curl -sf -X POST "$$B/namespaces" -H 'Content-Type: application/json' -d '{"namespace":["db"]}' >/dev/null 2>&1 || true
        curl -sf -X DELETE "$$B/namespaces/db/tables/orders_ext" >/dev/null 2>&1 || true
        curl -sf -X POST "$$B/namespaces/db/tables" -H 'Content-Type: application/json' -d '{"name":"orders_ext","schema":{"type":"struct","schema-id":0,"identifier-field-ids":[1],"fields":[{"id":1,"name":"order_id","required":true,"type":"long"},{"id":2,"name":"customer_id","required":false,"type":"long"},{"id":3,"name":"product_id","required":false,"type":"int"},{"id":4,"name":"amount","required":false,"type":"double"},{"id":5,"name":"status","required":false,"type":"int"}]},"partition-spec":{"spec-id":0,"fields":[]},"write-order":{"order-id":0,"fields":[]},"stage-create":false,"properties":{"format-version":"2","write.delete.mode":"merge-on-read","write.merge.mode":"merge-on-read","write.update.mode":"merge-on-read"}}' && echo "[init] db.orders_ext ready"
        curl -sf -X DELETE "$$B/namespaces/db/tables/orders_ext_poc3" >/dev/null 2>&1 || true
        echo "[init] done"
    restart: "no"
    networks: [icebergpoc]

  trino:
    image: trinodb/trino:467
    container_name: icebergpoc-trino
    profiles: [trino]
    ports:
      - "29080:8080"
    depends_on:
      gravitino: { condition: service_healthy }
    networks: [icebergpoc]

  # ── POC1 — hand-rolled C (Arrow GLib + libcurl) ──────────────────────
  poc1:
    image: nooriuta/icebergstremingpoc:poc1
    container_name: icebergpoc-poc1
    depends_on:
      init-tables: { condition: service_completed_successfully }
    environment:
      CATALOG_URL:           "http://gravitino:9001/iceberg/v1"
      AWS_S3_ENDPOINT:       "http://minio:9000"
      AWS_ACCESS_KEY_ID:     minioadmin
      AWS_SECRET_ACCESS_KEY: minioadmin
      AWS_REGION:            us-east-1
      N_TASKS:               "4"
      WARMUP_ROUNDS:         "2"
    networks: [icebergpoc]
    profiles: [poc]

  # ── POC3 — iceberg-cpp v0.3.0 ────────────────────────────────────────
  poc3:
    image: nooriuta/icebergstremingpoc:poc3
    container_name: icebergpoc-poc3
    depends_on:
      init-tables: { condition: service_completed_successfully }
    environment:
      CATALOG_URL:           "http://gravitino:9001/iceberg/v1"
      AWS_S3_ENDPOINT:       "http://minio:9000"
      AWS_ACCESS_KEY_ID:     minioadmin
      AWS_SECRET_ACCESS_KEY: minioadmin
      AWS_REGION:            us-east-1
      N_TASKS:               "4"
      WARMUP_ROUNDS:         "2"
    networks: [icebergpoc]
    profiles: [poc]

  # ── POC4s — Flink 1.20 streaming (jar вшит в образ) ──────────────────
  flink-jobmanager:
    image: nooriuta/icebergstremingpoc:poc4s
    container_name: icebergpoc-flink-jm
    command: jobmanager
    ports:
      - "28081:8081"
    environment:
      FLINK_PROPERTIES: |
        jobmanager.rpc.address: flink-jobmanager
        execution.checkpointing.interval: 500ms
        execution.checkpointing.mode: EXACTLY_ONCE
        state.backend.type: hashmap
    networks: [icebergpoc]
    profiles: [flink]

  flink-taskmanager:
    image: nooriuta/icebergstremingpoc:poc4s
    container_name: icebergpoc-flink-tm
    command: taskmanager
    depends_on: [flink-jobmanager]
    environment:
      FLINK_PROPERTIES: |
        jobmanager.rpc.address: flink-jobmanager
        taskmanager.numberOfTaskSlots: 4
        taskmanager.memory.process.size: 2560m
        taskmanager.memory.jvm-metaspace.size: 1024m
      JAVA_OPTS_TASKMANAGER: "-XX:MaxMetaspaceSize=1024m -XX:+UseG1GC"
    networks: [icebergpoc]
    profiles: [flink]

  # ── POC5 — Spark batch (writeTo().append()) ──────────────────────────
  poc5:
    image: nooriuta/icebergstremingpoc:poc5
    container_name: icebergpoc-poc5
    user: root
    depends_on:
      gravitino: { condition: service_healthy }
    environment:
      CATALOG_URI:           "http://gravitino:9001/iceberg/"
      S3_ENDPOINT:           "http://minio:9000"
      AWS_ACCESS_KEY_ID:     minioadmin
      AWS_SECRET_ACCESS_KEY: minioadmin
      AWS_REGION:            us-east-1
      N_ROWS:                "1000"
      N_COLS:                "5"
      N_ROUNDS:              "8"
      BATCH_SLEEP_S:         "0"
      PARALLELISM:           "4"
    command: >
      /opt/spark/bin/spark-submit
      --packages org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.6.1,org.apache.hadoop:hadoop-aws:3.3.4
      /opt/spark-job/iceberg_bench.py
    networks: [icebergpoc]
    profiles: [spark]

  # ── POC5ss — Spark Structured Streaming (foreachBatch) ───────────────
  poc5ss:
    image: nooriuta/icebergstremingpoc:poc5
    container_name: icebergpoc-poc5ss
    user: root
    depends_on:
      gravitino: { condition: service_healthy }
    environment:
      CATALOG_URI:           "http://gravitino:9001/iceberg/"
      S3_ENDPOINT:           "http://minio:9000"
      AWS_ACCESS_KEY_ID:     minioadmin
      AWS_SECRET_ACCESS_KEY: minioadmin
      AWS_REGION:            us-east-1
      EVENT_RATE:            "1000"
      N_COLS:                "300"
      N_ROUNDS:              "8"
      TRIGGER_MS:            "1000"
      RUN_ID:                "1"
      PARALLELISM:           "4"
    command: >
      /opt/spark/bin/spark-submit
      --packages org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.6.1,org.apache.hadoop:hadoop-aws:3.3.4
      /opt/spark-job/iceberg_bench_ss.py
    networks: [icebergpoc]
    profiles: [spark]