ROCm/composable_kernel
1
0
Fork 0
mirror of https://github.com/ROCm/composable_kernel.git synced 2026-07-11 17:51:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Sync with the debug branch

Browse Source
This commit is contained in:
Sudhir Kylasa
2025-03-26 04:06:30 -04:00
parent a93030a78d
commit 446d1ebeed
4 changed files with 184 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

92
include/ck_tile/host/check_err.hpp
View File

@@ -186,7 +186,7 @@ check_err(const Range& out,
{
max_err = err > max_err ? err : max_err;
err_count++;
if(err_count < 5)
if(err_count < 20)
{
std::cerr << msg << std::setw(12) << std::setprecision(7) << " out[" << i
<< "] != ref[" << i << "]: " << o << " != " << r << std::endl;
@@ -194,6 +194,34 @@ check_err(const Range& out,
res = false;
}
}
int total_err_count = err_count - 20;
err_count = 0;
for(std::size_t i = 0; i < ref.size(); ++i)
{
const double o = type_convert<float>(*std::next(std::begin(out), i));
const double r = type_convert<float>(*std::next(std::begin(ref), i));
err = std::abs(o - r);
if(err > atol + rtol * std::abs(r) || is_infinity_error(o, r))
{
max_err = err > max_err ? err : max_err;
err_count++;
if(err_count >= total_err_count)
{
std::cerr << msg << std::setw(12) << std::setprecision(7) << " out[" << i
<< "] != ref[" << i << "]: " << o << " != " << r << std::endl;
}
res = false;
}
}
for(std::size_t i = 0; i < ref.size() / 32; i++)
{
for(std::size_t j = 0; j < 32; j++)
{
const double o = type_convert<float>(*std::next(std::begin(out), i * 32 + j));
std::cerr << std::setw(10) << o;
}
std::cerr << std::endl;
}
if(!res)
{
const float error_percent =
@@ -246,7 +274,7 @@ check_err(const Range& out,
{
max_err = err > max_err ? err : max_err;
err_count++;
if(err_count < 5)
if(err_count < 20)
{
std::cerr << msg << std::setw(12) << std::setprecision(7) << " out[" << i
<< "] != ref[" << i << "]: " << o << " != " << r << std::endl;
@@ -254,6 +282,34 @@ check_err(const Range& out,
res = false;
}
}
int total_err_count = err_count - 20;
err_count = 0;
for(std::size_t i = 0; i < ref.size(); ++i)
{
const double o = type_convert<float>(*std::next(std::begin(out), i));
const double r = type_convert<float>(*std::next(std::begin(ref), i));
err = std::abs(o - r);
if(err > atol + rtol * std::abs(r) || is_infinity_error(o, r))
{
max_err = err > max_err ? err : max_err;
err_count++;
if(err_count >= total_err_count)
{
std::cerr << msg << std::setw(12) << std::setprecision(7) << " out[" << i
<< "] != ref[" << i << "]: " << o << " != " << r << std::endl;
}
res = false;
}
}
for(std::size_t i = 0; i < ref.size() / 32; i++)
{
for(std::size_t j = 0; j < 32; j++)
{
const double o = type_convert<float>(*std::next(std::begin(out), i * 32 + j));
std::cerr << std::setw(10) << o;
}
std::cerr << std::endl;
}
if(!res)
{
const float error_percent =
@@ -305,7 +361,7 @@ check_err(const Range& out,
{
max_err = err > max_err ? err : max_err;
err_count++;
if(err_count < 5)
if(err_count < 20)
{
std::cerr << msg << std::setw(12) << std::setprecision(7) << " out[" << i
<< "] != ref[" << i << "]: " << o << " != " << r << std::endl;
@@ -313,6 +369,34 @@ check_err(const Range& out,
res = false;
}
}
int total_err_count = err_count - 20;
err_count = 0;
for(std::size_t i = 0; i < ref.size(); ++i)
{
const double o = type_convert<float>(*std::next(std::begin(out), i));
const double r = type_convert<float>(*std::next(std::begin(ref), i));
err = std::abs(o - r);
if(err > atol + rtol * std::abs(r) || is_infinity_error(o, r))
{
max_err = err > max_err ? err : max_err;
err_count++;
if(err_count >= total_err_count)
{
std::cerr << msg << std::setw(12) << std::setprecision(7) << " out[" << i
<< "] != ref[" << i << "]: " << o << " != " << r << std::endl;
}
res = false;
}
}
for(std::size_t i = 0; i < ref.size() / 32; i++)
{
for(std::size_t j = 0; j < 32; j++)
{
const double o = type_convert<float>(*std::next(std::begin(out), i * 32 + j));
std::cerr << std::setw(10) << o;
}
std::cerr << std::endl;
}
if(!res)
{
const float error_percent =
@@ -367,7 +451,7 @@ std::enable_if_t<(std::is_same_v<ranges::range_value_t<Range>, ranges::range_val
}
res = false;
}
}
}
if(!res)
{
const float error_percent =

28
include/ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_v5.hpp
View File

@@ -225,7 +225,7 @@ struct GemmPipelineAgBgCrCompV5 : public BaseGemmPipelineAgBgCrCompV5<Problem>
tile_elementwise_inout([](auto& c) { c = 0; }, c_block_tile);
// define ping, pong steps here as lambda functions.
auto MemoryOpsStep = [&](auto idx) {
auto MemoryOpsStep = [&]() {
// Memory read half here.
Base::GlobalPrefetch(
a_global_load_tile, a_copy_dram_window, a_dram_tile_window_step);
@@ -255,36 +255,28 @@ struct GemmPipelineAgBgCrCompV5 : public BaseGemmPipelineAgBgCrCompV5<Problem>
{
Base::LocalPrefill(b_copy_lds_window, b_global_load_tile, b_element_func);
}
// transfer from LDS to registers
if(idx == 0)
{
Base::LocalPrefetch(a_tile_0, a_lds_window);
Base::LocalPrefetch(b_tile_0, b_lds_window);
}
else
{
Base::LocalPrefetch(a_tile_1, a_lds_window);
Base::LocalPrefetch(b_tile_1, b_lds_window);
}
};
auto ComputeStep = [&](auto idx) {
if(idx == 0)
{
Base::LocalPrefetch(a_tile_0, a_lds_window);
Base::LocalPrefetch(b_tile_0, b_lds_window);
block_gemm(c_block_tile, a_tile_0, b_tile_0);
// tile_elementwise_inout([](auto& c) { c += 1; }, c_block_tile);
}
else
{
Base::LocalPrefetch(a_tile_1, a_lds_window);
Base::LocalPrefetch(b_tile_1, b_lds_window);
block_gemm(c_block_tile, a_tile_1, b_tile_1);
// tile_elementwise_inout([](auto& c) { c += 1; }, c_block_tile);
}
};
if(op_id == 0)
if(op_id == 1)
{
MemoryOpsStep(group_id);
MemoryOpsStep();
}
// start the main loop.
index_t num_compute_steps = __builtin_amdgcn_readfirstlane(num_loop) * 2 - 1;
@@ -295,9 +287,9 @@ struct GemmPipelineAgBgCrCompV5 : public BaseGemmPipelineAgBgCrCompV5<Problem>
block_sync_lds();
op_id = (op_id + 1) % num_stages_;
if(op_id == 0)
if(op_id == 1)
{
MemoryOpsStep(group_id);
MemoryOpsStep();
}
else
{
@@ -310,7 +302,7 @@ struct GemmPipelineAgBgCrCompV5 : public BaseGemmPipelineAgBgCrCompV5<Problem>
// Handle Tail Number here.
block_sync_lds();
if(op_id == 0)
if(op_id == 1)
{
ComputeStep(group_id);
}