From 148ae9f2ef43b319817dbdc236a8409105f684b2 Mon Sep 17 00:00:00 2001 From: Illia Silin <98187287+illsilin@users.noreply.github.com> Date: Fri, 3 Apr 2026 10:04:12 -0700 Subject: [PATCH] [CK] Replace daily CI builds with mainline compiler with TheRock compiler. (#6147) ## Motivation Since the compiler team has deprecated the amd-mainline branch and switched to TheRock, we'll start building a docker image with TheRock artifacts and building/testing Ck with that. ## Technical Details ## Test Plan ## Test Result ## Submission Checklist - [ ] Look over the contributing guidelines at https://github.com/ROCm/ROCm/blob/develop/CONTRIBUTING.md#pull-requests. --- Dockerfile | 51 +++++++++++----------------- Dockerfile.compiler | 4 +-- Dockerfile.manylinux | 80 +++++++++++++++++++++++++++++++++++--------- Jenkinsfile | 15 +++++---- 4 files changed, 95 insertions(+), 55 deletions(-) diff --git a/Dockerfile b/Dockerfile index f19bc69362..de129d0703 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,22 +2,33 @@ FROM ubuntu:24.04 ARG DEBIAN_FRONTEND=noninteractive ARG ROCMVERSION=7.1.1 +ARG DEB_ROCM_REPO=http://repo.radeon.com/rocm/apt/.apt_$ROCMVERSION/ +ARG TARBALL_URL=https://rocm.nightlies.amd.com/tarball/therock-dist-linux-gfx90X-dcgpu-7.12.0a20260218.tar.gz ARG compiler_version="" ARG compiler_commit="" -ARG CK_SCCACHE="" -ARG DEB_ROCM_REPO=http://repo.radeon.com/rocm/apt/.apt_$ROCMVERSION/ ENV APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=DontWarn ENV DEBIAN_FRONTEND=noninteractive +ENV PATH=$PATH:/opt/rocm/bin +ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/rocm/lib +ENV HIP_PLATFORM=amd # Add rocm repository RUN set -xe && \ apt-get update && apt-get install -y --allow-unauthenticated apt-utils wget gnupg2 curl -RUN wget https://repo.radeon.com/amdgpu-install/7.1.1/ubuntu/noble/amdgpu-install_7.1.1.70101-1_all.deb && \ - apt install ./amdgpu-install_7.1.1.70101-1_all.deb -y && \ - apt update && \ - apt install python3-setuptools python3-wheel -y && \ - apt install rocm-dev -y +RUN if [ "$compiler_version" = "therock" ]; then \ + rm -rf /opt/rocm && mkdir /opt/rocm && \ + echo "Downloading ROCm tarball from $TARBALL_URL..." && \ + wget -q -O /tmp/rocm.tar.gz "$TARBALL_URL" && \ + echo "Extracting tarball to /opt/rocm..." && \ + tar -xzf /tmp/rocm.tar.gz -C /opt/rocm --strip-components=1 ; \ + else echo "using the release compiler" && \ + wget https://repo.radeon.com/amdgpu-install/7.1.1/ubuntu/noble/amdgpu-install_7.1.1.70101-1_all.deb && \ + apt install ./amdgpu-install_7.1.1.70101-1_all.deb -y && \ + apt update && \ + apt install python3-setuptools python3-wheel -y && \ + apt install rocm-dev -y; \ + fi # Install SCCACHE ENV SCCACHE_VERSION="0.14.0" @@ -34,7 +45,6 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --allow- build-essential \ cmake \ git \ - hip-rocclr \ iputils-ping \ jq \ libelf-dev \ @@ -44,8 +54,8 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --allow- net-tools \ pkg-config \ python3-full \ + python3-pip \ redis \ - rocm-llvm-dev \ sshpass \ stunnel \ software-properties-common \ @@ -88,26 +98,3 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --allow- git clone -b master https://github.com/ROCm/rocm-cmake.git && \ cd rocm-cmake && mkdir build && cd build && \ cmake .. && cmake --build . && cmake --build . --target install - -WORKDIR / -# Add alternative compilers, if necessary -ENV compiler_version=$compiler_version -ENV compiler_commit=$compiler_commit -RUN sh -c "echo compiler version = '$compiler_version'" && \ - sh -c "echo compiler commit = '$compiler_commit'" - -RUN if ( [ "$compiler_version" = "develop" ] || [ "$compiler_version" = "amd-mainline" ] ) && [ "$compiler_commit" = "" ]; then \ - git clone -b "$compiler_version" https://github.com/ROCm/llvm-project.git && \ - cd llvm-project && mkdir build && cd build && \ - cmake -DCMAKE_INSTALL_PREFIX=/opt/rocm/llvm -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=1 -DLLVM_TARGETS_TO_BUILD="AMDGPU;X86" -DLLVM_ENABLE_PROJECTS="clang;lld" -DLLVM_ENABLE_RUNTIMES="compiler-rt" ../llvm && \ - make -j 8 ; \ - else echo "using the release compiler"; \ - fi - -RUN if ( [ "$compiler_version" = "develop" ] || [ "$compiler_version" = "amd-mainline" ] ) && [ "$compiler_commit" != "" ]; then \ - git clone -b "$compiler_version" https://github.com/ROCm/llvm-project.git && \ - cd llvm-project && git checkout "$compiler_commit" && echo "checking out commit $compiler_commit" && mkdir build && cd build && \ - cmake -DCMAKE_INSTALL_PREFIX=/opt/rocm/llvm -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=1 -DLLVM_TARGETS_TO_BUILD="AMDGPU;X86" -DLLVM_ENABLE_PROJECTS="clang;lld" -DLLVM_ENABLE_RUNTIMES="compiler-rt" ../llvm && \ - make -j 8 ; \ - else echo "using the release compiler"; \ - fi diff --git a/Dockerfile.compiler b/Dockerfile.compiler index c27e016903..9d1e54106e 100644 --- a/Dockerfile.compiler +++ b/Dockerfile.compiler @@ -9,7 +9,7 @@ ENV compiler_commit=$compiler_commit RUN sh -c "echo compiler version = '$compiler_version'" && \ sh -c "echo compiler commit = '$compiler_commit'" -RUN if ( [ "$compiler_version" = "develop" ] || [ "$compiler_version" = "amd-staging" ] || [ "$compiler_version" = "amd-mainline" ] ) && [ "$compiler_commit" = "" ]; then \ +RUN if ( [ "$compiler_version" = "develop" ] || [ "$compiler_version" = "amd-staging" ] ) && [ "$compiler_commit" = "" ]; then \ git clone -b "$compiler_version" https://github.com/ROCm/llvm-project.git && \ cd llvm-project && git log -1 && mkdir build && cd build && \ cmake -G Ninja \ @@ -43,7 +43,7 @@ RUN if ( [ "$compiler_version" = "develop" ] || [ "$compiler_version" = "amd-sta else echo "using the release compiler"; \ fi -RUN if ( [ "$compiler_version" = "develop" ] || [ "$compiler_version" = "amd-staging" ] || [ "$compiler_version" = "amd-mainline" ] ) && [ "$compiler_commit" != "" ]; then \ +RUN if ( [ "$compiler_version" = "develop" ] || [ "$compiler_version" = "amd-staging" ] ) && [ "$compiler_commit" != "" ]; then \ git clone -b "$compiler_version" https://github.com/ROCm/llvm-project.git && \ cd llvm-project && git checkout "$compiler_commit" && echo "checking out commit $compiler_commit" && mkdir build && cd build && \ cmake -G Ninja \ diff --git a/Dockerfile.manylinux b/Dockerfile.manylinux index 2c0bec2840..bfbe847b1d 100644 --- a/Dockerfile.manylinux +++ b/Dockerfile.manylinux @@ -3,7 +3,6 @@ ARG DEBIAN_FRONTEND=noninteractive ARG ROCMVERSION=7.2 ARG compiler_version="" ARG compiler_commit="" -ARG CK_SCCACHE="" ARG DEB_ROCM_REPO=http://repo.radeon.com/rocm/apt/.apt_$ROCMVERSION/ ENV APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=DontWarn ENV DEBIAN_FRONTEND=noninteractive @@ -19,16 +18,15 @@ RUN wget https://repo.radeon.com/amdgpu-install/7.2/rhel/8.10/amdgpu-install-7.2 dnf install python3-setuptools python3-wheel -y && \ dnf install rocm-dev -y -## Sccache binary built from source for ROCm, only install if CK_SCCACHE is defined -ARG SCCACHE_REPO_URL=http://compute-artifactory.amd.com/artifactory/rocm-generic-experimental/rocm-sccache +# Install SCCACHE +ENV SCCACHE_VERSION="0.14.0" ENV SCCACHE_INSTALL_LOCATION=/usr/local/.cargo/bin ENV PATH=$PATH:${SCCACHE_INSTALL_LOCATION} -ENV CK_SCCACHE=$CK_SCCACHE -RUN if [ "$CK_SCCACHE" != "" ]; then \ - mkdir -p ${SCCACHE_INSTALL_LOCATION} && \ - curl ${SCCACHE_REPO_URL}/portable/0.2.16/sccache-0.2.16-alpha.1-rocm --output ${SCCACHE_INSTALL_LOCATION}/sccache && \ - chmod +x ${SCCACHE_INSTALL_LOCATION}/sccache; \ - fi +RUN set -x && \ + mkdir -p ${SCCACHE_INSTALL_LOCATION} && \ + wget -qO sccache.tar.gz https://github.com/mozilla/sccache/releases/latest/download/sccache-v$SCCACHE_VERSION-x86_64-unknown-linux-musl.tar.gz && \ + tar -xzf sccache.tar.gz --strip-components=1 -C ${SCCACHE_INSTALL_LOCATION} && \ + chmod +x ${SCCACHE_INSTALL_LOCATION}/sccache # Install dependencies RUN dnf update -y && DEBIAN_FRONTEND=noninteractive dnf install -y \ @@ -83,19 +81,71 @@ ENV compiler_commit=$compiler_commit RUN sh -c "echo compiler version = '$compiler_version'" && \ sh -c "echo compiler commit = '$compiler_commit'" -RUN if ( [ "$compiler_version" = "develop" ] || [ "$compiler_version" = "amd-mainline" ] ) && [ "$compiler_commit" = "" ]; then \ +RUN if ( [ "$compiler_version" = "develop" ] || [ "$compiler_version" = "amd-staging" ] ) && [ "$compiler_commit" = "" ]; then \ git clone -b "$compiler_version" https://github.com/ROCm/llvm-project.git && \ cd llvm-project && mkdir build && cd build && \ - cmake -DCMAKE_INSTALL_PREFIX=/opt/rocm/llvm -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=1 -DLLVM_TARGETS_TO_BUILD="AMDGPU;X86" -DLLVM_ENABLE_PROJECTS="clang;lld" -DLLVM_ENABLE_RUNTIMES="compiler-rt" ../llvm && \ - make -j 8 ; \ + cmake -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_ENABLE_PROJECTS="clang;lld;clang-tools-extra;flang;mlir" \ + -DLLVM_LIT_ARGS="-vv --show-unsupported --show-xfail -j 32" \ + -DPACKAGE_VENDOR="AMD" \ + -DCMAKE_INSTALL_PREFIX=/home/$USER/rocm/pure_llvm_1.0 \ + -DLLVM_ENABLE_ASSERTIONS=ON \ + -DLLVM_BUILD_DOCS=ON \ + -DLLVM_TARGETS_TO_BUILD=all \ + -DLIBOMPTARGET_ENABLE_DEBUG=ON \ + -DOFFLOAD_ENABLE_EMISSARY_APIS=OFF \ + -DCLANG_DEFAULT_LINKER=lld \ + -DCLANG_DEFAULT_PIE_ON_LINUX=0 \ + -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;openmp;compiler-rt;libunwind;flang-rt" \ + -DLIBCXX_ENABLE_SHARED=OFF \ + -DLIBCXX_ENABLE_STATIC=ON \ + -DLIBCXX_INSTALL_LIBRARY=OFF \ + -DLIBCXX_INSTALL_HEADERS=OFF \ + -DLIBCXXABI_ENABLE_SHARED=OFF \ + -DLIBCXXABI_ENABLE_STATIC=ON \ + -DLIBCXXABI_INSTALL_STATIC_LIBRARY=OFF \ + -DLLVM_ENABLE_ASSERTIONS=1 \ + -DLLVM_ENABLE_Z3_SOLVER=OFF \ + -DLLVM_ENABLE_ZLIB=ON \ + -DLLVM_LINK_LLVM_DYLIB=OFF \ + -DCLANG_LINK_CLANG_DYLIB=OFF \ + ../llvm && \ + ninja -j16 ; \ else echo "using the release compiler"; \ fi -RUN if ( [ "$compiler_version" = "develop" ] || [ "$compiler_version" = "amd-mainline" ] ) && [ "$compiler_commit" != "" ]; then \ +RUN if ( [ "$compiler_version" = "develop" ] || [ "$compiler_version" = "amd-staging" ] ) && [ "$compiler_commit" != "" ]; then \ git clone -b "$compiler_version" https://github.com/ROCm/llvm-project.git && \ cd llvm-project && git checkout "$compiler_commit" && echo "checking out commit $compiler_commit" && mkdir build && cd build && \ - cmake -DCMAKE_INSTALL_PREFIX=/opt/rocm/llvm -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=1 -DLLVM_TARGETS_TO_BUILD="AMDGPU;X86" -DLLVM_ENABLE_PROJECTS="clang;lld" -DLLVM_ENABLE_RUNTIMES="compiler-rt" ../llvm && \ - make -j 8 ; \ + cmake -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_ENABLE_PROJECTS="clang;lld;clang-tools-extra;flang;mlir" \ + -DLLVM_LIT_ARGS="-vv --show-unsupported --show-xfail -j 32" \ + -DPACKAGE_VENDOR="AMD" \ + -DCMAKE_INSTALL_PREFIX=/home/$USER/rocm/pure_llvm_1.0 \ + -DLLVM_ENABLE_ASSERTIONS=ON \ + -DLLVM_BUILD_DOCS=ON \ + -DLLVM_TARGETS_TO_BUILD=all \ + -DLIBOMPTARGET_ENABLE_DEBUG=ON \ + -DOFFLOAD_ENABLE_EMISSARY_APIS=OFF \ + -DCLANG_DEFAULT_LINKER=lld \ + -DCLANG_DEFAULT_PIE_ON_LINUX=0 \ + -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;openmp;compiler-rt;libunwind;flang-rt" \ + -DLIBCXX_ENABLE_SHARED=OFF \ + -DLIBCXX_ENABLE_STATIC=ON \ + -DLIBCXX_INSTALL_LIBRARY=OFF \ + -DLIBCXX_INSTALL_HEADERS=OFF \ + -DLIBCXXABI_ENABLE_SHARED=OFF \ + -DLIBCXXABI_ENABLE_STATIC=ON \ + -DLIBCXXABI_INSTALL_STATIC_LIBRARY=OFF \ + -DLLVM_ENABLE_ASSERTIONS=1 \ + -DLLVM_ENABLE_Z3_SOLVER=OFF \ + -DLLVM_ENABLE_ZLIB=ON \ + -DLLVM_LINK_LLVM_DYLIB=OFF \ + -DCLANG_LINK_CLANG_DYLIB=OFF \ + ../llvm && \ + ninja -j16 ; \ else echo "using the release compiler"; \ fi diff --git a/Jenkinsfile b/Jenkinsfile index 069a06bfd9..3569d8b267 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -421,10 +421,13 @@ def buildDocker(install_prefix){ def image_name = getDockerImageName() def base_image_name = getBaseDockerImageName() echo "Building Docker for ${image_name}" - def dockerArgs = "--build-arg PREFIX=${install_prefix} --build-arg CK_SCCACHE='${env.CK_SCCACHE}' --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' " - if(params.COMPILER_VERSION == "develop" || params.COMPILER_VERSION == "amd-staging" || params.COMPILER_VERSION == "amd-mainline" || params.COMPILER_COMMIT != ""){ + def dockerArgs = "--build-arg PREFIX=${install_prefix} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' " + if(params.COMPILER_VERSION == "develop" || params.COMPILER_VERSION == "amd-staging" || params.COMPILER_COMMIT != ""){ dockerArgs = dockerArgs + " --no-cache --build-arg BASE_DOCKER='${base_image_name}' -f projects/composablekernel/Dockerfile.compiler . " } + else if(params.COMPILER_VERSION == "therock"){ + dockerArgs = dockerArgs + " --no-cache -f projects/composablekernel/Dockerfile . " + } else if(params.RUN_AITER_TESTS){ image_name = "${env.CK_DOCKERHUB_PRIVATE}:ck_aiter" dockerArgs = dockerArgs + " --no-cache -f projects/composablekernel/Dockerfile.aiter --build-arg AITER_BRANCH='${params.aiter_branch}' --build-arg CK_AITER_BRANCH='${params.ck_aiter_branch}' . " @@ -470,10 +473,10 @@ def get_docker_options(){ else{ //only add kfd and dri paths if you actually going to run somthing on GPUs dockerOpts = "--network=host --device=/dev/kfd --device=/dev/dri --group-add video --group-add render --cap-add=SYS_PTRACE --security-opt seccomp=unconfined" } - if (params.COMPILER_VERSION == "develop" || params.COMPILER_VERSION == "amd-staging" || params.COMPILER_VERSION == "amd-mainline" || params.COMPILER_COMMIT != ""){ + if (params.COMPILER_VERSION == "develop" || params.COMPILER_VERSION == "amd-staging" || params.COMPILER_VERSION == "therock" || params.COMPILER_COMMIT != ""){ // the --env COMPRESSED_BUNDLE_FORMAT_VERSION=2 env variable is required when building code with offload-compress flag with // newer clang22 compilers and running with older hip runtima libraries - dockerOpts = dockerOpts + " --env HIP_CLANG_PATH='/llvm-project/build/bin' --env COMPRESSED_BUNDLE_FORMAT_VERSION=2 " + dockerOpts = dockerOpts + " --env HIP_CLANG_PATH='/llvm-project/build/bin' --env COMPRESSED_BUNDLE_FORMAT_VERSION=2 --env HIP_PLATFORM=amd " } // on some machines the group ids for video and render groups may not be the same as in the docker image! def video_id = sh(returnStdout: true, script: 'getent group video | cut -d: -f3') @@ -1183,7 +1186,7 @@ CRON_SETTINGS = BRANCH_NAME == "develop" ? '''0 23 * * * % RUN_FULL_QA=true;RUN_ 0 22 * * * % RUN_FULL_QA=true;DISABLE_DL_KERNELS=true;RUN_TILE_ENGINE_BASIC_TESTS=true;RUN_TILE_ENGINE_GEMM_TESTS=true;RUN_PERFORMANCE_TESTS=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true 0 21 * * * % RUN_GROUPED_CONV_LARGE_CASES_TESTS=true;hipTensor_test=true;BUILD_GFX101=false;BUILD_GFX908=false;BUILD_GFX942=true;BUILD_GFX950=true;RUN_PERFORMANCE_TESTS=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true;BUILD_PACKAGES=true 0 19 * * * % BUILD_DOCKER=true;COMPILER_VERSION=develop;BUILD_COMPILER=/llvm-project/build/bin/clang++;USE_SCCACHE=false;NINJA_BUILD_TRACE=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true - 0 17 * * * % BUILD_DOCKER=true;COMPILER_VERSION=amd-mainline;BUILD_COMPILER=/llvm-project/build/bin/clang++;USE_SCCACHE=false;NINJA_BUILD_TRACE=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true + 0 17 * * * % BUILD_DOCKER=true;COMPILER_VERSION=therock;USE_SCCACHE=false;NINJA_BUILD_TRACE=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true 0 15 * * * % BUILD_DOCKER=true;COMPILER_VERSION=amd-staging;BUILD_COMPILER=/llvm-project/build/bin/clang++;USE_SCCACHE=false;NINJA_BUILD_TRACE=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true 0 13 * * * % BUILD_INSTANCES_ONLY=true;USE_SCCACHE=false;NINJA_BUILD_TRACE=true;FORCE_CI=true 0 11 * * * % RUN_FULL_CONV_TILE_TESTS=true;RUN_AITER_TESTS=true;USE_SCCACHE=false;RUN_PERFORMANCE_TESTS=false;FORCE_CI=true @@ -1217,7 +1220,7 @@ pipeline { string( name: 'COMPILER_VERSION', defaultValue: '', - description: 'Specify which version of compiler to use: release, develop, amd-staging, amd-mainline, or leave blank (default).') + description: 'Specify which version of compiler to use: develop, amd-staging, therock, or leave blank (default).') string( name: 'COMPILER_COMMIT', defaultValue: '',