From a225733f11d7d8d1315815a41207a309b12eb113 Mon Sep 17 00:00:00 2001 From: Illia Silin <98187287+illsilin@users.noreply.github.com> Date: Mon, 8 Aug 2022 11:49:14 -0700 Subject: [PATCH] Fix QA, allow switching compiler versions, fix google test compilation error. (#348) * allow selecting compiler version * fix typo * add Wno-deprecated flag for google tests * change git repo, fix qa log files names * change the git clone syntax * use Omkar's git credentials * try to use jenkins as git user * try using illsilin username for gerrit repo with ssh key * try new gerrit authorization * change ssh key syntax * try another way of passing ssh key to docker * add mount ssh in dockerfile * create .ssh folder * move ssh-keyscan to later * get rid of npm call * build first docker image on master * check the contents of the .ssh folder * try replacing omkars creds with gerrit creds * use open repo, clean up changes * get rid of ssh default argument [ROCm/composable_kernel commit: aba7fefce7f7b866e62403c4c4bb1354af32031c] --- Dockerfile | 5 ++- Jenkinsfile | 67 ++++++++++++++----------------------- cmake/googletest.cmake | 1 + script/process_perf_data.sh | 4 +-- script/process_qa_data.sh | 4 +-- 5 files changed, 33 insertions(+), 48 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7c8fb98d95..3d01b36c01 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,6 @@ RUN sh -c "echo deb [arch=amd64] $DEB_ROCM_REPO ubuntu main > /etc/apt/sources.l RUN wget --no-check-certificate -qO - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add - RUN sh -c "echo deb https://apt.kitware.com/ubuntu/ bionic main | tee -a /etc/apt/sources.list" -# ADD requirements.txt requirements.txt # Install dependencies RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated \ apt-utils \ @@ -86,8 +85,8 @@ WORKDIR / ENV compiler_version=$compiler_version RUN sh -c "echo compiler version = '$compiler_version'" -RUN if [ "$compiler_version" = "9110" ]; then \ - git clone -b ck-9110 https://github.com/RadeonOpenCompute/llvm-project.git && \ +RUN --mount=type=ssh if [ "$compiler_version" != "release" ]; then \ + git clone -b "$compiler_version" https://github.com/RadeonOpenCompute/llvm-project.git && \ cd llvm-project && mkdir build && cd build && \ cmake -DCMAKE_INSTALL_PREFIX=/opt/rocm/llvm -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=1 -DLLVM_TARGETS_TO_BUILD="AMDGPU;X86" -DLLVM_ENABLE_PROJECTS="clang;lld;compiler-rt" ../llvm && \ make -j 8 ; \ diff --git a/Jenkinsfile b/Jenkinsfile index 6e890b537a..5b92364322 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -92,7 +92,7 @@ def buildHipClangJob(Map conf=[:]){ env.HSA_ENABLE_SDMA=0 checkout scm - def image = "composable_kernels" + def image = "composable_kernels_${params.COMPILER_VERSION}" def prefixpath = conf.get("prefixpath", "/opt/rocm") def gpu_arch = conf.get("gpu_arch", "gfx908") @@ -102,14 +102,10 @@ def buildHipClangJob(Map conf=[:]){ if (conf.get("enforce_xnack_on", false)) { dockerOpts = dockerOpts + " --env HSA_XNACK=1" } - def dockerArgs - if (params.USE_9110){ - dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg GPU_ARCH='${gpu_arch}' --build-arg compiler_version='9110' " + def dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg GPU_ARCH='${gpu_arch}' --build-arg compiler_version='${params.COMPILER_VERSION}' " + if (params.COMPILER_VERSION != "release"){ dockerOpts = dockerOpts + " --env HIP_CLANG_PATH='/llvm-project/build/bin' " } - else{ - dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg GPU_ARCH='${gpu_arch}' --build-arg compiler_version='release' " - } def variant = env.STAGE_NAME @@ -185,7 +181,8 @@ def runCKProfiler(Map conf=[:]){ env.HSA_ENABLE_SDMA=0 checkout scm - def image = "composable_kernels" + + def image = "composable_kernels_${params.COMPILER_VERSION}" def prefixpath = conf.get("prefixpath", "/opt/rocm") def gpu_arch = conf.get("gpu_arch", "gfx908") @@ -195,14 +192,10 @@ def runCKProfiler(Map conf=[:]){ if (conf.get("enforce_xnack_on", false)) { dockerOpts = dockerOpts + " --env HSA_XNACK=1" } - def dockerArgs - if (params.USE_9110){ - dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg GPU_ARCH='${gpu_arch}' --build-arg compiler_version='9110' " + def dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg GPU_ARCH='${gpu_arch}' --build-arg compiler_version='${params.COMPILER_VERSION}' " + if (params.COMPILER_VERSION != "release"){ dockerOpts = dockerOpts + " --env HIP_CLANG_PATH='/llvm-project/build/bin' " } - else{ - dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg GPU_ARCH='${gpu_arch}' --build-arg compiler_version='release' " - } def variant = env.STAGE_NAME def retimage @@ -248,20 +241,15 @@ def runCKProfiler(Map conf=[:]){ dir("script"){ if (params.RUN_FULL_QA){ def qa_log = "qa_${gpu_arch}.log" - if (params.USE_9110){ - sh "./run_full_performance_tests.sh 1 QA_9110 ${gpu_arch} ${env.BRANCH_NAME} ${NODE_NAME}" - } - else{ - sh "./run_full_performance_tests.sh 1 QA_release ${gpu_arch} ${env.BRANCH_NAME} ${NODE_NAME}" - } + sh "./run_full_performance_tests.sh 1 QA_${params.COMPILER_VERSION} ${gpu_arch} ${env.BRANCH_NAME} ${NODE_NAME}" archiveArtifacts "perf_gemm_${gpu_arch}.log" archiveArtifacts "perf_resnet50_N256_${gpu_arch}.log" archiveArtifacts "perf_resnet50_N4_${gpu_arch}.log" archiveArtifacts "perf_batched_gemm_${gpu_arch}.log" archiveArtifacts "perf_grouped_gemm_${gpu_arch}.log" - archiveArtifacts "perf_fwd_conv_${gpu_arch}.log" - archiveArtifacts "perf_bwd_conv_${gpu_arch}.log" - archiveArtifacts "perf_fusion_${gpu_arch}.log" + archiveArtifacts "perf_conv_fwd_${gpu_arch}.log" + archiveArtifacts "perf_conv_bwd_${gpu_arch}.log" + archiveArtifacts "perf_gemm_bilinear_${gpu_arch}.log" archiveArtifacts "perf_reduction_${gpu_arch}.log" // stash perf files to master stash name: "perf_gemm_${gpu_arch}.log" @@ -269,19 +257,14 @@ def runCKProfiler(Map conf=[:]){ stash name: "perf_resnet50_N4_${gpu_arch}.log" stash name: "perf_batched_gemm_${gpu_arch}.log" stash name: "perf_grouped_gemm_${gpu_arch}.log" - stash name: "perf_fwd_conv_${gpu_arch}.log" - stash name: "perf_bwd_conv_${gpu_arch}.log" - stash name: "perf_fusion_${gpu_arch}.log" + stash name: "perf_conv_fwd_${gpu_arch}.log" + stash name: "perf_conv_bwd_${gpu_arch}.log" + stash name: "perf_gemm_bilinear_${gpu_arch}.log" stash name: "perf_reduction_${gpu_arch}.log" //we will process results on the master node } else{ - if (params.USE_9110){ - sh "./run_performance_tests.sh 0 CI_9110 ${gpu_arch} ${env.BRANCH_NAME} ${NODE_NAME}" - } - else{ - sh "./run_performance_tests.sh 0 CI_release ${gpu_arch} ${env.BRANCH_NAME} ${NODE_NAME}" - } + sh "./run_performance_tests.sh 0 CI_${params.COMPILER_VERSION} ${gpu_arch} ${env.BRANCH_NAME} ${NODE_NAME}" archiveArtifacts "perf_gemm_${gpu_arch}.log" archiveArtifacts "perf_resnet50_N256_${gpu_arch}.log" archiveArtifacts "perf_resnet50_N4_${gpu_arch}.log" @@ -318,7 +301,7 @@ def runPerfTest(Map conf=[:]){ def process_results(Map conf=[:]){ env.HSA_ENABLE_SDMA=0 checkout scm - def image = "composable_kernels" + def image = "composable_kernels_${params.COMPILER_VERSION}" def prefixpath = "/opt/rocm" def gpu_arch = conf.get("gpu_arch", "gfx908") @@ -353,9 +336,9 @@ def process_results(Map conf=[:]){ unstash "perf_resnet50_N4_${gpu_arch}.log" unstash "perf_batched_gemm_${gpu_arch}.log" unstash "perf_grouped_gemm_${gpu_arch}.log" - unstash "perf_fwd_conv_${gpu_arch}.log" - unstash "perf_bwd_conv_${gpu_arch}.log" - unstash "perf_fusion_${gpu_arch}.log" + unstash "perf_conv_fwd_${gpu_arch}.log" + unstash "perf_conv_bwd${gpu_arch}.log" + unstash "perf_gemm_bilinear_${gpu_arch}.log" unstash "perf_reduction_${gpu_arch}.log" sh "./process_qa_data.sh ${gpu_arch}" } @@ -378,7 +361,7 @@ def process_results(Map conf=[:]){ } //launch develop branch daily at 23:00 in FULL_QA mode -CRON_SETTINGS = BRANCH_NAME == "develop" ? '''0 23 * * * % RUN_FULL_QA=true;USE_9110=true''' : "" +CRON_SETTINGS = BRANCH_NAME == "develop" ? '''0 23 * * * % RUN_FULL_QA=true''' : "" pipeline { agent none @@ -389,10 +372,10 @@ pipeline { parallelsAlwaysFailFast() } parameters { - booleanParam( - name: "USE_9110", - defaultValue: true, - description: "Select compiler version: 9110 (default) or release") + string( + name: 'COMPILER_VERSION', + defaultValue: 'ck-9110', + description: 'Specify which version of compiler to use: ck-9110 (default), release, or amd-mainline-open.') booleanParam( name: "RUN_FULL_QA", defaultValue: false, @@ -406,6 +389,8 @@ pipeline { dbsshuser = "${dbsshuser}" dbsshpassword = "${dbsshpassword}" status_wrapper_creds = "${status_wrapper_creds}" + gerrit_cred="${gerrit_cred}" + DOCKER_BUILDKIT = "1" } stages{ stage("Static checks") { diff --git a/cmake/googletest.cmake b/cmake/googletest.cmake index 3718b916ff..cf2240ebc5 100644 --- a/cmake/googletest.cmake +++ b/cmake/googletest.cmake @@ -20,6 +20,7 @@ list(APPEND GTEST_CMAKE_CXX_FLAGS -Wno-unused-member-function -Wno-comma -Wno-old-style-cast + -Wno-deprecated ) message(STATUS "Suppressing googltest warnings with flags: ${GTEST_CMAKE_CXX_FLAGS}") diff --git a/script/process_perf_data.sh b/script/process_perf_data.sh index 412f87d0e3..b68a7c1b2f 100755 --- a/script/process_perf_data.sh +++ b/script/process_perf_data.sh @@ -12,5 +12,5 @@ pip3 install sqlalchemy pymysql pandas sshtunnel #process results gpu_arch=$1 python3 process_perf_data.py perf_gemm_"$gpu_arch".log -python3 process_perf_data.py perf_resnet50_N265_"$gpu_arch".log -python3 process_perf_data.py perf_resnet50_N4_"$gpu_arch".log \ No newline at end of file +python3 process_perf_data.py perf_resnet50_N256_"$gpu_arch".log +python3 process_perf_data.py perf_resnet50_N4_"$gpu_arch".log diff --git a/script/process_qa_data.sh b/script/process_qa_data.sh index dbb7c68d87..fb2dbd5bb5 100755 --- a/script/process_qa_data.sh +++ b/script/process_qa_data.sh @@ -12,11 +12,11 @@ pip3 install sqlalchemy pymysql pandas sshtunnel #process results gpu_arch=$1 python3 process_perf_data.py perf_gemm_"$gpu_arch".log -python3 process_perf_data.py perf_resnet50_N265_"$gpu_arch".log +python3 process_perf_data.py perf_resnet50_N256_"$gpu_arch".log python3 process_perf_data.py perf_resnet50_N4_"$gpu_arch".log python3 process_perf_data.py perf_batched_gemm_"$gpu_arch".log python3 process_perf_data.py perf_grouped_gemm_"$gpu_arch".log python3 process_perf_data.py perf_conv_fwd_"$gpu_arch".log python3 process_perf_data.py perf_conv_bwd_data_"$gpu_arch".log python3 process_perf_data.py perf_gemm_bilinear_"$gpu_arch".log -python3 process_perf_data.py perf_reduction_"$gpu_arch".log \ No newline at end of file +python3 process_perf_data.py perf_reduction_"$gpu_arch".log