From 072fc27075bf47202cc81c6ff2ceef42de88ac12 Mon Sep 17 00:00:00 2001 From: Illia Silin <98187287+illsilin@users.noreply.github.com> Date: Sat, 1 Oct 2022 16:48:19 -0700 Subject: [PATCH] Allow setting ROCM version, activate cchache, etc. (#462) * enable ccache and decouple it from MIOpen ccache use * fix the ccache check script * use another method to get server name * fix syntax * add quotes around the server name variable * use check_host as function * change syntax * fix syntax * test if server name is parsed correctly * try different syntax * check the env var value * test new check node function * add ROCMVERSION parameter and fix script syntax * fix script syntax * add missing instances of rocm version * install ccache in the docker image * do not check GPU in clang format stage, clean up old code * update defaults and clean up [ROCm/composable_kernel commit: 7fc3ed761aa35709d87c8fbbe41dd368648b3541] --- Dockerfile | 7 +-- Jenkinsfile | 153 ++++++++++++++++------------------------------------ 2 files changed, 49 insertions(+), 111 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2c9ec2742e..d024f966c5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ FROM ubuntu:20.04 -ARG ROCMVERSION=5.2.3 -ARG compiler_version -ARG compiler_commit +ARG ROCMVERSION=5.3 +ARG compiler_version="release" +ARG compiler_commit="" RUN set -xe @@ -19,6 +19,7 @@ RUN sh -c "echo deb http://mirrors.kernel.org/ubuntu focal main universe | tee - RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated \ apt-utils \ build-essential \ + ccache \ cmake-data \ cmake \ curl \ diff --git a/Jenkinsfile b/Jenkinsfile index 2a7a582e62..6d9ebc90c3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -19,10 +19,24 @@ def runShell(String command){ } def getDockerImageName(){ - def img = "${env.CK_DOCKERHUB}:ck_ub20.04_rocm5.2.3_${params.COMPILER_VERSION}" + def img = "${env.CK_DOCKERHUB}:ck_ub20.04_rocm${params.ROCMVERSION}_${params.COMPILER_VERSION}" return img } +def check_host() { + if ("${env.CK_CCACHE}" != "null"){ + def CCACHE_SERVER="${env.CK_CCACHE.split(':')[0]}" + echo "ccache server: ${CCACHE_SERVER}" + sh '''ping -c 1 -p 6379 "${CCACHE_SERVER}" | echo $? > tmp.txt''' + def output = readFile(file: "tmp.txt") + echo "tmp.txt contents: \$output" + return (output != "0") + } + else{ + return 1 + } +} + def build_compiler(){ def compiler if (params.BUILD_COMPILER == "hipcc"){ @@ -43,21 +57,21 @@ def getDockerImage(Map conf=[:]){ env.DOCKER_BUILDKIT=1 def prefixpath = conf.get("prefixpath", "/opt/rocm") // prefix:/opt/rocm def no_cache = conf.get("no_cache", false) - def dockerArgs = "--build-arg BUILDKIT_INLINE_CACHE=1 --build-arg PREFIX=${prefixpath} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' " - if(env.CCACHE_HOST) + def dockerArgs = "--build-arg BUILDKIT_INLINE_CACHE=1 --build-arg PREFIX=${prefixpath} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' " + echo "ccache server: ${env.CK_CCACHE}" + if(env.CK_CCACHE) { - def check_host = sh(script:"""(printf "PING\r\n";) | nc -N ${env.CCACHE_HOST} 6379 """, returnStdout: true).trim() - if(check_host == "+PONG") + if(check_host()) { - echo "FOUND CCACHE SERVER: ${CCACHE_HOST}" + echo "FOUND CCACHE SERVER: ${env.CK_CCACHE}" } else { - echo "CCACHE SERVER: ${CCACHE_HOST} NOT FOUND, got ${check_host} response" + echo "CCACHE SERVER: ${env.CK_CCACHE} NOT FOUND, got ${check_host} response" } - dockerArgs = dockerArgs + " --build-arg CCACHE_SECONDARY_STORAGE='redis://${env.CCACHE_HOST}' --build-arg COMPILER_LAUNCHER='ccache' " + dockerArgs = dockerArgs + " --build-arg CCACHE_SECONDARY_STORAGE='redis://${env.CK_CCACHE}' --build-arg COMPILER_LAUNCHER='ccache' " env.CCACHE_DIR = """/tmp/ccache_store""" - env.CCACHE_SECONDARY_STORAGE="""redis://${env.CCACHE_HOST}""" + env.CCACHE_SECONDARY_STORAGE="""redis://${env.CK_CCACHE}""" } if(no_cache) { @@ -86,21 +100,21 @@ def buildDocker(install_prefix){ checkout scm def image_name = getDockerImageName() echo "Building Docker for ${image_name}" - def dockerArgs = "--build-arg BUILDKIT_INLINE_CACHE=1 --build-arg PREFIX=${install_prefix} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}'" - if(env.CCACHE_HOST) + def dockerArgs = "--build-arg BUILDKIT_INLINE_CACHE=1 --build-arg PREFIX=${install_prefix} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' " + echo "ccache server: ${env.CK_CCACHE}" + if(env.CK_CCACHE) { - def check_host = sh(script:"""(printf "PING\\r\\n";) | nc -N ${env.CCACHE_HOST} 6379 """, returnStdout: true).trim() - if(check_host == "+PONG") + if(check_host()) { - echo "FOUND CCACHE SERVER: ${CCACHE_HOST}" + echo "FOUND CCACHE SERVER: ${env.CK_CCACHE}" } else { - echo "CCACHE SERVER: ${CCACHE_HOST} NOT FOUND, got ${check_host} response" + echo "CCACHE SERVER: ${env.CK_CCACHE} NOT FOUND, got ${check_host} response" } - dockerArgs = dockerArgs + " --build-arg CCACHE_SECONDARY_STORAGE='redis://${env.CCACHE_HOST}' --build-arg COMPILER_LAUNCHER='ccache' " + dockerArgs = dockerArgs + " --build-arg CCACHE_SECONDARY_STORAGE='redis://${env.CK_CCACHE}' --build-arg COMPILER_LAUNCHER='ccache' " env.CCACHE_DIR = """/tmp/ccache_store""" - env.CCACHE_SECONDARY_STORAGE="""redis://${env.CCACHE_HOST}""" + env.CCACHE_SECONDARY_STORAGE="""redis://${env.CK_CCACHE}""" } echo "Build Args: ${dockerArgs}" @@ -161,10 +175,11 @@ def cmake_build(Map conf=[:]){ }else{ setup_args = " -DCMAKE_BUILD_TYPE=release" + setup_args } - if(env.CCACHE_HOST) + if(env.CK_CCACHE) { setup_args = " -DCMAKE_CXX_COMPILER_LAUNCHER='ccache' -DCMAKE_C_COMPILER_LAUNCHER='ccache' " + setup_args } + echo "ccache server: ${env.CK_CCACHE}" def pre_setup_cmd = """ echo \$HSA_ENABLE_SDMA @@ -210,7 +225,7 @@ def buildHipClangJob(Map conf=[:]){ if (conf.get("enforce_xnack_on", false)) { dockerOpts = dockerOpts + " --env HSA_XNACK=1 " } - def dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' " + def dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' " if (params.COMPILER_VERSION != "release"){ dockerOpts = dockerOpts + " --env HIP_CLANG_PATH='/llvm-project/build/bin' " } @@ -220,39 +235,6 @@ def buildHipClangJob(Map conf=[:]){ def retimage gitStatusWrapper(credentialsId: "${status_wrapper_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCmSoftwarePlatform', repo: 'composable_kernel') { - try { - (retimage, image) = getDockerImage(conf) - withDockerContainer(image: image, args: dockerOpts) { - timeout(time: 5, unit: 'MINUTES'){ - sh 'PATH="/opt/rocm/opencl/bin:/opt/rocm/opencl/bin/x86_64:$PATH" clinfo | tee clinfo.log' - if ( runShell('grep -n "Number of devices:.*. 0" clinfo.log') ){ - throw new Exception ("GPU not found") - } - else{ - echo "GPU is OK" - } - } - } - } - catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e){ - echo "The job was cancelled or aborted" - throw e - } - catch(Exception ex) { - retimage = docker.build("${image}", dockerArgs + " --no-cache .") - withDockerContainer(image: image, args: dockerOpts) { - timeout(time: 5, unit: 'MINUTES'){ - sh 'PATH="/opt/rocm/opencl/bin:/opt/rocm/opencl/bin/x86_64:$PATH" clinfo |tee clinfo.log' - if ( runShell('grep -n "Number of devices:.*. 0" clinfo.log') ){ - throw new Exception ("GPU not found") - } - else{ - echo "GPU is OK" - } - } - } - } - withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') { timeout(time: 5, unit: 'HOURS') { @@ -297,7 +279,7 @@ def runCKProfiler(Map conf=[:]){ if (conf.get("enforce_xnack_on", false)) { dockerOpts = dockerOpts + " --env HSA_XNACK=1 " } - def dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' " + def dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' " if (params.COMPILER_VERSION != "release"){ dockerOpts = dockerOpts + " --env HIP_CLANG_PATH='/llvm-project/build/bin' " } @@ -430,7 +412,7 @@ def Build_CK(Map conf=[:]){ if (conf.get("enforce_xnack_on", false)) { dockerOpts = dockerOpts + " --env HSA_XNACK=1 " } - def dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' " + def dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' " if (params.COMPILER_VERSION != "release"){ dockerOpts = dockerOpts + " --env HIP_CLANG_PATH='/llvm-project/build/bin' " } @@ -581,15 +563,19 @@ pipeline { booleanParam( name: "BUILD_DOCKER", defaultValue: false, - description: "Force building docker image (default: false)") + description: "Force building docker image (default: false), set to true if docker image needs to be updated.") + string( + name: 'ROCMVERSION', + defaultValue: '5.3', + description: 'Specify which ROCM version to use: 5.2.3, or 5.3 (default), etc.') string( name: 'COMPILER_VERSION', - defaultValue: 'amd-stg-open', - description: 'Specify which version of compiler to use: ck-9110, release, or amd-stg-open (default).') + defaultValue: 'release', + description: 'Specify which version of compiler to use: ck-9110, release (default), or amd-stg-open.') string( name: 'COMPILER_COMMIT', - defaultValue: '8a82e4eb7ba28521ba9a9424a0315a8a16590424', - description: 'Specify which commit of compiler branch to use: leave empty to use the latest commit, or use 10738 commit (default).') + defaultValue: '', + description: 'Specify which commit of compiler branch to use: leave empty to use the latest commit (default), or use 8a82e4eb7ba28521ba9a9424a0315a8a16590424 commit of amd-stg-open branch.') string( name: 'BUILD_COMPILER', defaultValue: 'hipcc', @@ -627,17 +613,6 @@ pipeline { } stage("Static checks") { parallel{ - // enable after we move from hipcc to hip-clang - // stage('Tidy') { - // agent{ label rocmnode("nogpu") } - // environment{ - // // setup_cmd = "CXX='/opt/rocm/bin/hipcc' cmake -DBUILD_DEV=On .. " - // build_cmd = "make -j\$(nproc) -k analyze" - // } - // steps{ - // buildHipClangJobAndReboot(build_cmd: build_cmd, no_reboot:true, prefixpath: '/opt/rocm', build_type: 'debug') - // } - // } stage('Clang Format') { agent{ label rocmnode("nogpu") } environment{ @@ -676,28 +651,6 @@ pipeline { } } - /* - //at present this stage only builds binaries. - //we will now build all binaries in a separate stage. - //once we have some tests to run in this stage, we can enable it again. - stage("Client App") - { - parallel - { - stage("Run Client App") - { - agent{ label rocmnode("gfx908")} - environment{ - setup_args = "${params.COMPILER_VERSION == "ck-9110" ? """ -DBUILD_DEV=Off -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx908;gfx90a" -DCMAKE_CXX_FLAGS="-O3 -Xclang -mlink-builtin-bitcode -Xclang /opt/rocm/amdgcn/bitcode/oclc_abi_version_400.bc" """ : """ -DBUILD_DEV=Off -DCMAKE_INSTALL_PREFIX=../install -DGPU_TARGETS="gfx908;gfx90a" -DCMAKE_CXX_FLAGS="-O3 " """ }" - execute_args = "${params.COMPILER_VERSION == "ck-9110" ? """ cd ../client_example && rm -rf build && mkdir build && cd build && cmake -D CMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" -DGPU_TARGETS="gfx908;gfx90a" -DCMAKE_CXX_FLAGS="-O3 -Xclang -mlink-builtin-bitcode -Xclang /opt/rocm/amdgcn/bitcode/oclc_abi_version_400.bc" -D CMAKE_CXX_COMPILER="${build_compiler()}" .. && make -j """ : """ cd ../client_example && rm -rf build && mkdir build && cd build && cmake -D CMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" -DGPU_TARGETS="gfx908;gfx90a" -DCMAKE_CXX_FLAGS="-O3" -D CMAKE_CXX_COMPILER="${build_compiler()}" .. && make -j """ }" - } - steps{ - buildHipClangJobAndReboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local') - } - } - } - } - */ stage("Performance Tests") { parallel @@ -746,21 +699,5 @@ pipeline { } } } - - /* enable after the cmake file supports packaging - stage("Packages") { - when { - expression { params.BUILD_PACKAGES && params.TARGET_NOGPU && params.DATATYPE_NA } - } - parallel { - stage("Package /opt/rocm") { - agent{ label rocmnode("nogpu") } - steps{ - buildHipClangJobAndReboot( package_build: "true", prefixpath: '/opt/rocm', gpu_arch: "gfx906;gfx908;gfx90a") - } - } - } - } - */ } }