sbloodys/hadoop repository overview

version: "3"

services:
  namenode:
    image: sbloodys/hadoop:3.3.6
    hostname: namenode
    container_name: namenode
    command: [ "hdfs", "namenode" ]
    ports:
      - 19870:9870
    env_file:
      - docker-compose.config
    environment:
      ENSURE_NAMENODE_DIR: "/tmp/hadoop-root/dfs/name"
    logging:
      driver: "json-file"
      options:
        max-size: "200m"
        max-file: "1"
    tty: true
    stdin_open: true
    restart: always
    healthcheck:
      test: [ "CMD", "curl", "http://namenode:9870" ]
      interval: 5s
      timeout: 5s
      retries: 120
  datanode:
    image: sbloodys/hadoop:3.3.6
    hostname: datanode
    container_name: datanode
    command: [ "hdfs", "datanode" ]
    env_file:
      - docker-compose.config
    logging:
      driver: "json-file"
      options:
        max-size: "200m"
        max-file: "1"
    tty: true
    stdin_open: true
    restart: always
    healthcheck:
      test: [ "CMD", "curl", "http://datanode:9864" ]
      interval: 5s
      timeout: 5s
      retries: 120
    depends_on:
      namenode:
        condition: service_healthy
  resourcemanager:
    image: sbloodys/hadoop:3.3.6
    hostname: resourcemanager
    container_name: resourcemanager
    command: [ "yarn", "resourcemanager" ]
    ports:
      - 18088:8088
    env_file:
      - docker-compose.config
    logging:
      driver: "json-file"
      options:
        max-size: "200m"
        max-file: "1"
    tty: true
    stdin_open: true
    restart: always
    healthcheck:
      test: [ "CMD", "curl", "http://resourcemanager:8088" ]
      interval: 5s
      timeout: 5s
      retries: 120
  nodemanager:
    image: sbloodys/hadoop:3.3.6
    hostname: nodemanager
    container_name: nodemanager
    command: [ "yarn", "nodemanager" ]
    env_file:
      - docker-compose.config
    logging:
      driver: "json-file"
      options:
        max-size: "200m"
        max-file: "1"
    tty: true
    stdin_open: true
    restart: always
    depends_on:
      resourcemanager:
        condition: service_healthy
    healthcheck:
      test: [ "CMD", "curl", "http://nodemanager:8042" ]
      interval: 5s
      timeout: 5s
      retries: 120

HADOOP_HOME=/opt/hadoop
CORE-SITE.XML_fs.default.name=hdfs://namenode
CORE-SITE.XML_fs.defaultFS=hdfs://namenode
CORE-SITE.XML_hadoop.http.staticuser.user=hadoop
HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:8020
HDFS-SITE.XML_dfs.replication=1
MAPRED-SITE.XML_mapreduce.framework.name=yarn
MAPRED-SITE.XML_yarn.app.mapreduce.am.env=HADOOP_MAPRED_HOME=$HADOOP_HOME
MAPRED-SITE.XML_mapreduce.map.env=HADOOP_MAPRED_HOME=$HADOOP_HOME
MAPRED-SITE.XML_mapreduce.reduce.env=HADOOP_MAPRED_HOME=$HADOOP_HOME
YARN-SITE.XML_yarn.resourcemanager.hostname=resourcemanager
YARN-SITE.XML_yarn.nodemanager.pmem-check-enabled=false
YARN-SITE.XML_yarn.nodemanager.delete.debug-delay-sec=600
YARN-SITE.XML_yarn.nodemanager.vmem-check-enabled=false
YARN-SITE.XML_yarn.nodemanager.aux-services=mapreduce_shuffle
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-applications=10000
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-am-resource-percent=0.1
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.resource-calculator=org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.queues=default
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.capacity=100
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.user-limit-factor=1
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.maximum-capacity=100
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.state=RUNNING
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_submit_applications=*
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_administer_queue=*
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.node-locality-delay=40
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings=
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings-override.enable=false

docker-compose -f docker-compose-hadoop.yaml up -d