Sign inSign up

projecteaina/mlops-nemo-harness:latest

Manifest digest

sha256:c6498b3b50f6b04008bfce3d6541268c7bd95efe370aa575532c4caf6c91fbfd

OS/ARCH

linux/amd64

Compressed size

15.16 GB

Last pushed

over 2 years by langtechbsc

Type

Image

Manifest digest

sha256:c6498b3b50f6b04008bfce3d6541268c7bd95efe370aa575532c4caf6c91fbfd

Image Layers

Image Layers

1ARG RELEASE0 B
2ARG LAUNCHPAD_BUILD_ARCH0 B
3LABEL org.opencontainers.image.ref.name=ubuntu0 B
4LABEL org.opencontainers.image.version=22.040 B
5ADD file ... in / 29.03 MB
6CMD ["/bin/bash"]0 B
7RUN /bin/sh -c export DEBIAN_FRONTEND=noninteractive129.91 MB
8ARG CUDA_VERSION0 B
9ARG CUDA_DRIVER_VERSION0 B
10ARG JETPACK_HOST_MOUNTS0 B
11ENV CUDA_VERSION=12.2.2.009 CUDA_DRIVER_VERSION=535.104.05 CUDA_CACHE_DISABLE=1 NVIDIA_REQUIRE_JETPACK_HOST_MOUNTS=0 B
12RUN |3 CUDA_VERSION=12.2.2.009 CUDA_DRIVER_VERSION=535.104.05 JETPACK_HOST_MOUNTS=143.25 MB
13RUN |3 CUDA_VERSION=12.2.2.009 CUDA_DRIVER_VERSION=535.104.05 JETPACK_HOST_MOUNTS=14.74 KB
14ENV _CUDA_COMPAT_PATH=/usr/local/cuda/compat ENV=/etc/shinit_v2 BASH_ENV=/etc/bash.bashrc SHELL=/bin/bash0 B
15LABEL com.nvidia.volumes.needed=nvidia_driver com.nvidia.cuda.version=9.00 B
16ARG NCCL_VERSION0 B
17ARG CUBLAS_VERSION0 B
18ARG CUFFT_VERSION0 B
19ARG CURAND_VERSION0 B
20ARG CUSPARSE_VERSION0 B
21ARG CUSOLVER_VERSION0 B
22ARG CUTENSOR_VERSION0 B
23ARG NPP_VERSION0 B
24ARG NVJPEG_VERSION0 B
25ARG CUDNN_VERSION0 B
26ARG TRT_VERSION0 B
27ARG TRTOSS_VERSION0 B
28ARG NSIGHT_SYSTEMS_VERSION0 B
29ARG NSIGHT_COMPUTE_VERSION0 B
30ENV NCCL_VERSION=2.19.3 CUBLAS_VERSION=12.2.5.6 CUFFT_VERSION=11.0.8.103 CURAND_VERSION=10.3.3.1410 B
31RUN |17 CUDA_VERSION=12.2.2.009 CUDA_DRIVER_VERSION=535.104.05 JETPACK_HOST_MOUNTS=2.75 GB
32LABEL com.nvidia.nccl.version=2.19.3 com.nvidia.cublas.version=12.2.5.6 com.nvidia.cufft.version=11.0.8.103 com.nvidia.curand.version=10.3.3.1410 B
33ARG DALI_VERSION0 B
34ARG DALI_BUILD0 B
35ARG POLYGRAPHY_VERSION0 B
36ARG TRANSFORMER_ENGINE_VERSION0 B
37ENV DALI_VERSION=1.30.0 DALI_BUILD=9783408 POLYGRAPHY_VERSION=0.49.0 TRANSFORMER_ENGINE_VERSION=0.120 B
38ADD docs.tgz / # buildkit11.26 KB
39RUN |21 CUDA_VERSION=12.2.2.009 CUDA_DRIVER_VERSION=535.104.05 JETPACK_HOST_MOUNTS=182 B
40ARG _LIBPATH_SUFFIX0 B
41ENV PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin LD_LIBRARY_PATH=/usr/local/cuda/compat/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64 NVIDIA_VISIBLE_DEVICES=all NVIDIA_DRIVER_CAPABILITIES=compute,utility,video0 B
42COPY entrypoint/ /opt/nvidia/ # buildkit5.59 KB
43ENV NVIDIA_PRODUCT_NAME=CUDA0 B
44ENTRYPOINT ["/opt/nvidia/nvidia_entrypoint.sh"]0 B
45COPY NVIDIA_Deep_Learning_Container_License.pdf /workspace/ # buildkit136.03 KB
46RUN /bin/sh -c export DEBIAN_FRONTEND=noninteractive29.9 MB
47ARG GDRCOPY_VERSION0 B
48ARG HPCX_VERSION0 B
49ARG RDMACORE_VERSION0 B
50ARG MOFED_VERSION=5.4-rdmacore39.00 B
51ARG OPENUCX_VERSION0 B
52ARG OPENMPI_VERSION0 B
53ENV GDRCOPY_VERSION=2.3 HPCX_VERSION=2.16rc4 MOFED_VERSION=5.4-rdmacore39.0 OPENUCX_VERSION=1.15.00 B
54ARG TARGETARCH0 B
55RUN |7 GDRCOPY_VERSION=2.3 HPCX_VERSION=2.16rc4 RDMACORE_VERSION=39.082.88 MB
56ENV OPAL_PREFIX=/opt/hpcx/ompi PATH=/usr/local/mpi/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/ucx/bin0 B
57ENV OMPI_MCA_coll_hcoll_enable=00 B
58COPY cuda-*.patch /tmp # buildkit503 B
59RUN |7 GDRCOPY_VERSION=2.3 HPCX_VERSION=2.16rc4 RDMACORE_VERSION=39.0453.7 MB
60ENV LIBRARY_PATH=/usr/local/cuda/lib64/stubs:0 B
61ENV NVIDIA_PRODUCT_NAME=PyTorch0 B
62ARG NVIDIA_PYTORCH_VERSION0 B
63ARG PYTORCH_BUILD_VERSION0 B
64ENV PYTORCH_BUILD_VERSION=2.1.0a0+32f93b1 PYTORCH_VERSION=2.1.0a0+32f93b1 PYTORCH_BUILD_NUMBER=0 NVIDIA_PYTORCH_VERSION=23.100 B
65LABEL com.nvidia.pytorch.version=2.1.0a0+32f93b10 B
66ARG TARGETARCH0 B
67ARG PYVER=3.100 B
68ARG L4T=00 B
69RUN |5 NVIDIA_PYTORCH_VERSION=23.10 PYTORCH_BUILD_VERSION=2.1.0a0+32f93b1 TARGETARCH=amd6461.57 MB
70ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python0 B
71ENV SETUPTOOLS_USE_DISTUTILS=stdlib0 B
72RUN |5 NVIDIA_PYTORCH_VERSION=23.10 PYTORCH_BUILD_VERSION=2.1.0a0+32f93b1 TARGETARCH=amd648.05 MB
73RUN |5 NVIDIA_PYTORCH_VERSION=23.10 PYTORCH_BUILD_VERSION=2.1.0a0+32f93b1 TARGETARCH=amd6424.27 MB
74WORKDIR /opt32 B
75ENV OPENBLAS_VERSION=0.3.230 B
76RUN |5 NVIDIA_PYTORCH_VERSION=23.10 PYTORCH_BUILD_VERSION=2.1.0a0+32f93b1 TARGETARCH=amd6412.14 MB
77WORKDIR /opt/pytorch116 B
78COPY . . # buildkit486.75 MB
79ENV PYTHONIOENCODING=utf-80 B
80ENV LC_ALL=C.UTF-80 B
81ENV PIP_DEFAULT_TIMEOUT=1000 B
82RUN |5 NVIDIA_PYTORCH_VERSION=23.10 PYTORCH_BUILD_VERSION=2.1.0a0+32f93b1 TARGETARCH=amd64893.56 MB
83RUN |5 NVIDIA_PYTORCH_VERSION=23.10 PYTORCH_BUILD_VERSION=2.1.0a0+32f93b1 TARGETARCH=amd64325 B
84RUN |5 NVIDIA_PYTORCH_VERSION=23.10 PYTORCH_BUILD_VERSION=2.1.0a0+32f93b1 TARGETARCH=amd6452.38 MB
85RUN |5 NVIDIA_PYTORCH_VERSION=23.10 PYTORCH_BUILD_VERSION=2.1.0a0+32f93b1 TARGETARCH=amd648.22 KB
86ENV NVM_DIR=/usr/local/nvm0 B
87RUN |5 NVIDIA_PYTORCH_VERSION=23.10 PYTORCH_BUILD_VERSION=2.1.0a0+32f93b1 TARGETARCH=amd6466.04 MB
88COPY jupyter_notebook_config.py /usr/local/etc/jupyter/ # buildkit471 B
89ENV JUPYTER_PORT=88880 B
90ENV TENSORBOARD_PORT=60060 B
91EXPOSE map[8888/tcp:{}]0 B
92EXPOSE map[6006/tcp:{}]0 B
93RUN |5 NVIDIA_PYTORCH_VERSION=23.10 PYTORCH_BUILD_VERSION=2.1.0a0+32f93b1 TARGETARCH=amd6415.02 MB
94RUN |5 NVIDIA_PYTORCH_VERSION=23.10 PYTORCH_BUILD_VERSION=2.1.0a0+32f93b1 TARGETARCH=amd64152.98 MB
95ENV UCC_CL_BASIC_TLS=^sharp0 B
96ENV TORCH_CUDA_ARCH_LIST=5.2 6.0 6.1 7.00 B
97ENV PYTORCH_HOME=/opt/pytorch/pytorch0 B
98ENV CUDA_HOME=/usr/local/cuda0 B
99ENV TORCH_ALLOW_TF32_CUBLAS_OVERRIDE=10 B
100ENV USE_EXPERIMENTAL_CUDNN_V8_API=10 B
101RUN |5 NVIDIA_PYTORCH_VERSION=23.10 PYTORCH_BUILD_VERSION=2.1.0a0+32f93b1 TARGETARCH=amd641.92 GB
102RUN |5 NVIDIA_PYTORCH_VERSION=23.10 PYTORCH_BUILD_VERSION=2.1.0a0+32f93b1 TARGETARCH=amd642.34 KB
103RUN /bin/sh -c pip install86.42 MB
104RUN /bin/sh -c if [281.29 MB
105ENV COCOAPI_VERSION=2.0+nv0.7.30 B
106RUN /bin/sh -c export COCOAPI_TAG=$(echo26.43 MB
107COPY singularity/ /.singularity.d/ # buildkit1.09 KB
108RUN /bin/sh -c ( cd273.35 MB
109RUN /bin/sh -c pip uninstall1.26 MB
110RUN /bin/sh -c pip install60.24 KB
111RUN /bin/sh -c if [1.62 GB
112WORKDIR /workspace32 B
113COPY NVREADME.md README.md # buildkit1.1 KB
114COPY docker-examples docker-examples # buildkit1.09 KB
115COPY examples examples # buildkit6.45 MB
116COPY tutorials tutorials # buildkit28.73 MB
117RUN /bin/sh -c chmod -R35.32 MB
118RUN /bin/sh -c set -x1.49 MB
119ENV PATH=/usr/local/mpi/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/ucx/bin:/opt/tensorrt/bin0 B
120RUN /bin/sh -c pip --version3.53 MB
121COPY torch_tensorrt/ /opt/pytorch/torch_tensorrt/ # buildkit56.39 MB
122ARG PYVER0 B
123RUN |1 PYVER=3.10 /bin/sh -c14.01 MB
124ENV LD_LIBRARY_PATH=/usr/local/lib/python3.10/dist-packages/torch/lib:/usr/local/lib/python3.10/dist-packages/torch_tensorrt/lib:/usr/local/cuda/compat/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib640 B
125ENV PATH=/usr/local/lib/python3.10/dist-packages/torch_tensorrt/bin:/usr/local/mpi/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/ucx/bin:/opt/tensorrt/bin0 B
126RUN |1 PYVER=3.10 /bin/sh -c115.94 MB
127RUN |1 PYVER=3.10 /bin/sh -c104.06 MB
128ENV TORCH_CUDNN_V8_API_ENABLED=10 B
129ENV CUDA_MODULE_LOADING=LAZY0 B
130RUN |1 PYVER=3.10 /bin/sh -c13.88 KB
131COPY entrypoint.d/ /opt/nvidia/entrypoint.d/ # buildkit504 B
132ARG NVIDIA_BUILD_ID0 B
133ENV NVIDIA_BUILD_ID=714223370 B
134LABEL com.nvidia.build.id=714223370 B
135ARG NVIDIA_BUILD_REF0 B
136LABEL com.nvidia.build.ref=798008b068e6dbd0088bab08098b0fce963b87b30 B
137ENV NVIDIA_PRODUCT_NAME=NeMo Megatron0 B
138ARG NVIDIA_BIGNLP_VERSION0 B
139ENV NVIDIA_BIGNLP_VERSION=0 B
140LABEL com.nvidia.bignlp.version=0 B
141ENV DEBIAN_FRONTEND=noninteractive0 B
142/bin/sh -c apt-get update &&4.92 MB
143WORKDIR /opt0 B
144/bin/sh -c mkdir s5cmd &&4.09 MB
145ENV PATH=/usr/local/lib/python3.10/dist-packages/torch_tensorrt/bin:/usr/local/mpi/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/ucx/bin:/opt/tensorrt/bin:/opt/s5cmd0 B
146/bin/sh -c git clone https://github.com/google/sentencepiece.git46.13 MB
147/bin/sh -c pip install packaging92 B
148ARG APEX_COMMIT0 B
149|1 APEX_COMMIT=52e18c894223800cb611682dce27d88050edf1de /bin/sh -c pip192.82 MB
150ARG NEMO_COMMIT0 B
151|2 APEX_COMMIT=52e18c894223800cb611682dce27d88050edf1de NEMO_COMMIT=15e0f8c1676e35d6d3c9e61441bf3cf216884d7a /bin/sh -c428.39 MB
152ARG ALIGNER_COMMIT0 B
153|3 ALIGNER_COMMIT=b6ce38ed2a578432797811f0faf5250a742fcd32 APEX_COMMIT=52e18c894223800cb611682dce27d88050edf1de NEMO_COMMIT=15e0f8c1676e35d6d3c9e61441bf3cf216884d7a /bin/sh2.06 MB
154|3 ALIGNER_COMMIT=b6ce38ed2a578432797811f0faf5250a742fcd32 APEX_COMMIT=52e18c894223800cb611682dce27d88050edf1de NEMO_COMMIT=15e0f8c1676e35d6d3c9e61441bf3cf216884d7a /bin/sh1.92 MB
155ARG TE_COMMIT0 B
156|4 ALIGNER_COMMIT=b6ce38ed2a578432797811f0faf5250a742fcd32 APEX_COMMIT=52e18c894223800cb611682dce27d88050edf1de NEMO_COMMIT=15e0f8c1676e35d6d3c9e61441bf3cf216884d7a TE_COMMIT=cf6fc898286e4ad347ff88925c88663324e2b87d292.79 MB
157COPY dir:235d465f2c76d903e430187baa1e7f681600b91550c3bc3a9df8f06815bbab7d in megatron-lm 1.08 MB
158|4 ALIGNER_COMMIT=b6ce38ed2a578432797811f0faf5250a742fcd32 APEX_COMMIT=52e18c894223800cb611682dce27d88050edf1de NEMO_COMMIT=15e0f8c1676e35d6d3c9e61441bf3cf216884d7a TE_COMMIT=cf6fc898286e4ad347ff88925c88663324e2b87d11.61 KB
159ARG LAUNCHER_COMMIT0 B
160|5 ALIGNER_COMMIT=b6ce38ed2a578432797811f0faf5250a742fcd32 APEX_COMMIT=52e18c894223800cb611682dce27d88050edf1de LAUNCHER_COMMIT= NEMO_COMMIT=15e0f8c1676e35d6d3c9e61441bf3cf216884d7a14.64 MB
161ENV LAUNCHER_SCRIPTS_PATH=/opt/NeMo-Megatron-Launcher/launcher_scripts0 B
162ENV PYTHONPATH=/opt/NeMo-Megatron-Launcher/launcher_scripts:0 B
163|5 ALIGNER_COMMIT=b6ce38ed2a578432797811f0faf5250a742fcd32 APEX_COMMIT=52e18c894223800cb611682dce27d88050edf1de LAUNCHER_COMMIT= NEMO_COMMIT=15e0f8c1676e35d6d3c9e61441bf3cf216884d7a1.65 MB
164|5 ALIGNER_COMMIT=b6ce38ed2a578432797811f0faf5250a742fcd32 APEX_COMMIT=52e18c894223800cb611682dce27d88050edf1de LAUNCHER_COMMIT= NEMO_COMMIT=15e0f8c1676e35d6d3c9e61441bf3cf216884d7a83.25 MB
165|5 ALIGNER_COMMIT=b6ce38ed2a578432797811f0faf5250a742fcd32 APEX_COMMIT=52e18c894223800cb611682dce27d88050edf1de LAUNCHER_COMMIT= NEMO_COMMIT=15e0f8c1676e35d6d3c9e61441bf3cf216884d7a114.07 MB
166|5 ALIGNER_COMMIT=b6ce38ed2a578432797811f0faf5250a742fcd32 APEX_COMMIT=52e18c894223800cb611682dce27d88050edf1de LAUNCHER_COMMIT= NEMO_COMMIT=15e0f8c1676e35d6d3c9e61441bf3cf216884d7a92 B
167|5 ALIGNER_COMMIT=b6ce38ed2a578432797811f0faf5250a742fcd32 APEX_COMMIT=52e18c894223800cb611682dce27d88050edf1de LAUNCHER_COMMIT= NEMO_COMMIT=15e0f8c1676e35d6d3c9e61441bf3cf216884d7a2.72 MB
168COPY dir:4dd099ec572a14294670643839aea26b8403d92532b3bd8ae1c0bc70db46b5be in nemo-data-curator 287.86 KB
169|5 ALIGNER_COMMIT=b6ce38ed2a578432797811f0faf5250a742fcd32 APEX_COMMIT=52e18c894223800cb611682dce27d88050edf1de LAUNCHER_COMMIT= NEMO_COMMIT=15e0f8c1676e35d6d3c9e61441bf3cf216884d7a3.55 GB
170COPY dir:b66a1b818651ea57638e932ad28877344c6351fcd398f0f08e245fa77fb5fbb8 in /opt/ammo/dist 748.72 KB
171/bin/sh -c pip install --no-deps876.58 KB
172/bin/sh -c echo " 2.2 KB
173/bin/sh -c cd /opt/hpcx &&300.62 MB
174ENV GDRCOPY_VERSION=2.3 HPCX_VERSION=2.14 MOFED_VERSION=5.4-rdmacore36.00 B
175WORKDIR /workspace0 B
176/bin/sh -c chmod -R a+w35.32 MB
177ARG NVIDIA_BUILD_ID0 B
178ENV NVIDIA_BUILD_ID=714223370 B
179LABEL com.nvidia.build.id=714223370 B
180ARG NVIDIA_BUILD_REF0 B
181LABEL com.nvidia.build.ref=0 B
182WORKDIR /lm-evaluation-harness111 B
183RUN /bin/sh -c apt-get update7.06 MB
184COPY ./lm-evaluation-harness ./ # buildkit212.77 MB
185RUN /bin/sh -c pip install35.11 MB

Command

 ARG RELEASE