mirror of
https://github.com/ROCm/composable_kernel.git
synced 2026-04-20 06:49:15 +00:00
Add option to build ckProfiler packages for individual architectures. (#3105)
* refactor package generation, add dedicated switch * allow building packages not only on gfx9 * enable last stage to post packages * stash packages from different arch into separate stashes * build packages daily automatically
This commit is contained in:
Illia Silin
100
Jenkinsfile
vendored
100
Jenkinsfile
vendored
@@ -194,6 +194,33 @@ def check_arch(){
|
||||
return arch_type
|
||||
}
|
||||
|
||||
def check_arch_name(){
|
||||
def arch_name = ""
|
||||
sh 'rocminfo | tee rocminfo.log'
|
||||
if ( runShell('grep -n "gfx90a" rocminfo.log') ){
|
||||
arch_name = "gfx90a"
|
||||
}
|
||||
else if ( runShell('grep -n "gfx942" rocminfo.log') ) {
|
||||
arch_name = "gfx942"
|
||||
}
|
||||
else if ( runShell('grep -n "gfx10" rocminfo.log') ) {
|
||||
arch_name = "gfx10"
|
||||
}
|
||||
else if ( runShell('grep -n "gfx11" rocminfo.log') ) {
|
||||
arch_name = "gfx11"
|
||||
}
|
||||
else if ( runShell('grep -n "gfx12" rocminfo.log') ) {
|
||||
arch_name = "gfx12"
|
||||
}
|
||||
else if ( runShell('grep -n "gfx908" rocminfo.log') ) {
|
||||
arch_name = "gfx908"
|
||||
}
|
||||
else if ( runShell('grep -n "gfx950" rocminfo.log') ) {
|
||||
arch_name = "gfx950"
|
||||
}
|
||||
return arch_name
|
||||
}
|
||||
|
||||
def getDockerImage(Map conf=[:]){
|
||||
env.DOCKER_BUILDKIT=1
|
||||
def prefixpath = conf.get("prefixpath", "/opt/rocm")
|
||||
@@ -302,12 +329,6 @@ def cmake_build(Map conf=[:]){
|
||||
//cmake_env can overwrite default CXX variables.
|
||||
def cmake_envs = "CXX=${compiler} CXXFLAGS='-Werror' " + conf.get("cmake_ex_env","")
|
||||
|
||||
def package_build = (conf.get("package_build","") == "true")
|
||||
|
||||
if (package_build == true) {
|
||||
config_targets = "package"
|
||||
}
|
||||
|
||||
if(conf.get("build_install","") == "true")
|
||||
{
|
||||
config_targets = 'install ' + config_targets
|
||||
@@ -455,15 +476,20 @@ def cmake_build(Map conf=[:]){
|
||||
else{
|
||||
sh "ninja check"
|
||||
}
|
||||
if(params.BUILD_PACKAGES){
|
||||
echo "Build ckProfiler packages"
|
||||
sh 'ninja -j64 package'
|
||||
def arch_name = check_arch_name()
|
||||
sh "mv composablekernel-ckprofiler_*.deb composablekernel-ckprofiler_1.2.0_amd64_${arch_name}.deb"
|
||||
stash includes: "composablekernel-ckprofiler**.deb", name: "profiler_package_${arch_name}"
|
||||
}
|
||||
}
|
||||
if(params.BUILD_INSTANCES_ONLY){
|
||||
// build deb packages
|
||||
echo "Build packages"
|
||||
echo "Build library package"
|
||||
sh 'ninja -j64 package'
|
||||
archiveArtifacts artifacts: 'composablekernel-dev*.deb'
|
||||
sh 'mv composablekernel-dev_*.deb composablekernel-dev_all_targets_1.2.0_amd64.deb'
|
||||
sh 'mv composablekernel-ckprofiler_*.deb composablekernel-ckprofiler_1.2.0_amd64.deb'
|
||||
stash includes: "composablekernel-**.deb", name: "packages"
|
||||
stash includes: "composablekernel-dev**.deb", name: "lib_package"
|
||||
}
|
||||
}
|
||||
else{
|
||||
@@ -475,15 +501,18 @@ def cmake_build(Map conf=[:]){
|
||||
else{
|
||||
sh "ninja check"
|
||||
}
|
||||
if(params.BUILD_PACKAGES){
|
||||
echo "Build ckProfiler packages"
|
||||
sh 'ninja -j64 package'
|
||||
def arch_name = check_arch_name()
|
||||
sh "mv composablekernel-ckprofiler_*.deb composablekernel-ckprofiler_1.2.0_amd64_${arch_name}.deb"
|
||||
stash includes: "composablekernel-ckprofiler**.deb", name: "profiler_package_${arch_name}"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Only archive from develop
|
||||
if (package_build == true && env.BRANCH_NAME == "develop") {
|
||||
archiveArtifacts artifacts: "build/*.deb", allowEmptyArchive: true, fingerprint: true
|
||||
}
|
||||
//check the node gpu architecture
|
||||
def arch = check_arch()
|
||||
if (params.RUN_CK_TILE_FMHA_TESTS){
|
||||
@@ -823,9 +852,42 @@ def process_results(Map conf=[:]){
|
||||
}
|
||||
if (params.BUILD_INSTANCES_ONLY){
|
||||
// unstash deb packages
|
||||
unstash "packages"
|
||||
try{
|
||||
unstash "lib_package"
|
||||
}
|
||||
catch(Exception err){
|
||||
echo "could not locate lib_package."
|
||||
}
|
||||
sh "sshpass -p ${env.ck_deb_pw} scp -o StrictHostKeyChecking=no composablekernel-*.deb ${env.ck_deb_user}@${env.ck_deb_ip}:/var/www/html/composable_kernel/"
|
||||
}
|
||||
if (params.BUILD_PACKAGES){
|
||||
// unstash deb packages
|
||||
try{
|
||||
unstash "profiler_package_gfx90a"
|
||||
}
|
||||
catch(Exception err){
|
||||
echo "could not locate profiler_package_gfx90a."
|
||||
}
|
||||
try{
|
||||
unstash "profiler_package_gfx942"
|
||||
}
|
||||
catch(Exception err){
|
||||
echo "could not locate profiler_package_gfx942."
|
||||
}
|
||||
try{
|
||||
unstash "profiler_package_gfx950"
|
||||
}
|
||||
catch(Exception err){
|
||||
echo "could not locate profiler_package_gfx950."
|
||||
}
|
||||
try{
|
||||
unstash "profiler_package_gfx12"
|
||||
}
|
||||
catch(Exception err){
|
||||
echo "could not locate profiler_package_gfx12."
|
||||
}
|
||||
sh "sshpass -p ${env.ck_deb_pw} scp -o StrictHostKeyChecking=no composablekernel-ckprofiler*.deb ${env.ck_deb_user}@${env.ck_deb_ip}:/var/www/html/composable_kernel/"
|
||||
}
|
||||
else{
|
||||
// unstash perf files to master
|
||||
try{
|
||||
@@ -993,7 +1055,7 @@ def run_pytorch_tests(Map conf=[:]){
|
||||
//launch develop branch daily jobs
|
||||
CRON_SETTINGS = BRANCH_NAME == "develop" ? '''0 23 * * * % RUN_FULL_QA=true;RUN_CK_TILE_FMHA_TESTS=true;RUN_PERFORMANCE_TESTS=true;FORCE_CI=true
|
||||
0 22 * * * % RUN_FULL_QA=true;DISABLE_DL_KERNELS=true;RUN_TILE_ENGINE_GEMM_TESTS=true;RUN_PERFORMANCE_TESTS=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true
|
||||
0 21 * * * % RUN_GROUPED_CONV_LARGE_CASES_TESTS=true;hipTensor_test=true;BUILD_GFX908=true;BUILD_GFX942=true;BUILD_GFX950=true;RUN_PERFORMANCE_TESTS=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true
|
||||
0 21 * * * % RUN_GROUPED_CONV_LARGE_CASES_TESTS=true;hipTensor_test=true;BUILD_GFX908=true;BUILD_GFX942=true;BUILD_GFX950=true;RUN_PERFORMANCE_TESTS=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true;BUILD_PACKAGES=true
|
||||
0 19 * * * % BUILD_DOCKER=true;COMPILER_VERSION=amd-staging;BUILD_COMPILER=/llvm-project/build/bin/clang++;USE_SCCACHE=false;NINJA_BUILD_TRACE=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true
|
||||
0 17 * * * % BUILD_DOCKER=true;COMPILER_VERSION=amd-mainline;BUILD_COMPILER=/llvm-project/build/bin/clang++;USE_SCCACHE=false;NINJA_BUILD_TRACE=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true
|
||||
0 15 * * * % BUILD_INSTANCES_ONLY=true;USE_SCCACHE=false;NINJA_BUILD_TRACE=true;FORCE_CI=true
|
||||
@@ -1085,6 +1147,10 @@ pipeline {
|
||||
name: "BUILD_INSTANCES_ONLY",
|
||||
defaultValue: false,
|
||||
description: "Test building instances for various architectures simultaneously (default: OFF)")
|
||||
booleanParam(
|
||||
name: "BUILD_PACKAGES",
|
||||
defaultValue: false,
|
||||
description: "Build packages for the libraries and/or ckProfiler (default: OFF)")
|
||||
booleanParam(
|
||||
name: "BUILD_GFX908",
|
||||
defaultValue: false,
|
||||
@@ -1829,7 +1895,7 @@ pipeline {
|
||||
stage("Process results"){
|
||||
when {
|
||||
beforeAgent true
|
||||
expression { (params.RUN_PERFORMANCE_TESTS.toBoolean() || params.BUILD_INSTANCES_ONLY.toBoolean() || params.RUN_CK_TILE_FMHA_TESTS.toBoolean()) && !params.BUILD_LEGACY_OS.toBoolean() }
|
||||
expression { (params.RUN_PERFORMANCE_TESTS.toBoolean() || params.BUILD_INSTANCES_ONLY.toBoolean() || params.RUN_CK_TILE_FMHA_TESTS.toBoolean()|| params.BUILD_PACKAGES.toBoolean()) && !params.BUILD_LEGACY_OS.toBoolean() }
|
||||
}
|
||||
agent { label 'mici' }
|
||||
steps{
|
||||
|
||||
Reference in New Issue
Block a user