|
|
|
|
@@ -15,19 +15,19 @@ namespace ck_tile {
|
|
|
|
|
// fp16
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfmaF16F16F32M32N32K8 = WarpGemmImpl<
|
|
|
|
|
WarpGemmAtrributeMfma<WarpGemmAttributeMfmaImplF16F16F32M32N32K8<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
WarpGemmAttributeMfma<WarpGemmAttributeMfmaImplF16F16F32M32N32K8<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfmaF16F16F32M16N16K16 = WarpGemmImpl<
|
|
|
|
|
WarpGemmAtrributeMfma<WarpGemmAttributeMfmaImplF16F16F32M16N16K16<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
WarpGemmAttributeMfma<WarpGemmAttributeMfmaImplF16F16F32M16N16K16<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
|
|
|
|
|
#if defined(__gfx950__)
|
|
|
|
|
template <WGAttrNumAccessEnum AttrNumAccess = WGAttrNumAccessEnum::Single>
|
|
|
|
|
using WarpGemmMfmaF16F16F32M32N32K16 = WarpGemmImpl<
|
|
|
|
|
WarpGemmAtrributeMfma<WarpGemmAttributeMfmaImplF16F16F32M32N32K16<WGAttrCtlEnum::Default_>,
|
|
|
|
|
WarpGemmAttributeMfma<WarpGemmAttributeMfmaImplF16F16F32M32N32K16<WGAttrCtlEnum::Default_>,
|
|
|
|
|
AttrNumAccess>>;
|
|
|
|
|
#else
|
|
|
|
|
template <WGAttrNumAccessEnum AttrNumAccess = WGAttrNumAccessEnum::Single>
|
|
|
|
|
using WarpGemmMfmaF16F16F32M32N32K16 = WarpGemmImpl<WarpGemmAtrributeMfmaIterateK<
|
|
|
|
|
using WarpGemmMfmaF16F16F32M32N32K16 = WarpGemmImpl<WarpGemmAttributeMfmaIterateK<
|
|
|
|
|
WarpGemmAttributeMfmaImplF16F16F32M32N32K8<WGAttrCtlEnum::Default_>,
|
|
|
|
|
2,
|
|
|
|
|
AttrNumAccess>>;
|
|
|
|
|
@@ -36,42 +36,42 @@ using WarpGemmMfmaF16F16F32M32N32K16 = WarpGemmImpl<WarpGemmAtrributeMfmaIterate
|
|
|
|
|
#if defined(__gfx950__)
|
|
|
|
|
template <WGAttrNumAccessEnum AttrNumAccess = WGAttrNumAccessEnum::Single>
|
|
|
|
|
using WarpGemmMfmaF16F16F32M16N16K32 = WarpGemmImpl<
|
|
|
|
|
WarpGemmAtrributeMfma<WarpGemmAttributeMfmaImplF16F16F32M16N16K32<WGAttrCtlEnum::Default_>,
|
|
|
|
|
WarpGemmAttributeMfma<WarpGemmAttributeMfmaImplF16F16F32M16N16K32<WGAttrCtlEnum::Default_>,
|
|
|
|
|
AttrNumAccess>>;
|
|
|
|
|
#else
|
|
|
|
|
template <WGAttrNumAccessEnum AttrNumAccess = WGAttrNumAccessEnum::Single>
|
|
|
|
|
using WarpGemmMfmaF16F16F32M16N16K32 = WarpGemmImpl<WarpGemmAtrributeMfmaIterateK<
|
|
|
|
|
using WarpGemmMfmaF16F16F32M16N16K32 = WarpGemmImpl<WarpGemmAttributeMfmaIterateK<
|
|
|
|
|
WarpGemmAttributeMfmaImplF16F16F32M16N16K16<WGAttrCtlEnum::Default_>,
|
|
|
|
|
2,
|
|
|
|
|
AttrNumAccess>>;
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfmaF16F16F32M32N32K8SwizzleA = WarpGemmImpl<WarpGemmAtrributeMfmaIterateK_SwizzleA<
|
|
|
|
|
using WarpGemmMfmaF16F16F32M32N32K8SwizzleA = WarpGemmImpl<WarpGemmAttributeMfmaIterateK_SwizzleA<
|
|
|
|
|
WarpGemmAttributeMfmaImplF16F16F32M32N32K8<WGAttrCtlEnum::Default_>,
|
|
|
|
|
1>>;
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfmaF16F16F32M32N32K16SwizzleA = WarpGemmImpl<WarpGemmAtrributeMfmaIterateK_SwizzleA<
|
|
|
|
|
using WarpGemmMfmaF16F16F32M32N32K16SwizzleA = WarpGemmImpl<WarpGemmAttributeMfmaIterateK_SwizzleA<
|
|
|
|
|
WarpGemmAttributeMfmaImplF16F16F32M32N32K8<WGAttrCtlEnum::Default_>,
|
|
|
|
|
2>>;
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfmaF16F16F32M32N32K8TransposedCDistribution =
|
|
|
|
|
WarpGemmImpl<WarpGemmAtrributeMfmaTransposedCDistribution<
|
|
|
|
|
WarpGemmImpl<WarpGemmAttributeMfmaTransposedCDistribution<
|
|
|
|
|
WarpGemmAttributeMfmaImplF16F16F32M32N32K8<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfmaF16F16F32M16N16K16TransposedCDistribution =
|
|
|
|
|
WarpGemmImpl<WarpGemmAtrributeMfmaTransposedCDistribution<
|
|
|
|
|
WarpGemmImpl<WarpGemmAttributeMfmaTransposedCDistribution<
|
|
|
|
|
WarpGemmAttributeMfmaImplF16F16F32M16N16K16<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
|
|
|
|
|
#if defined(__gfx950__)
|
|
|
|
|
template <WGAttrNumAccessEnum AttrNumAccess = WGAttrNumAccessEnum::Single>
|
|
|
|
|
using WarpGemmMfmaF16F16F32M32N32K16TransposedCDistribution =
|
|
|
|
|
WarpGemmImpl<WarpGemmAtrributeMfmaTransposedCDistribution<
|
|
|
|
|
WarpGemmImpl<WarpGemmAttributeMfmaTransposedCDistribution<
|
|
|
|
|
WarpGemmAttributeMfmaImplF16F16F32M32N32K16<WGAttrCtlEnum::Default_>,
|
|
|
|
|
AttrNumAccess>>;
|
|
|
|
|
#else
|
|
|
|
|
template <WGAttrNumAccessEnum AttrNumAccess = WGAttrNumAccessEnum::Single>
|
|
|
|
|
using WarpGemmMfmaF16F16F32M32N32K16TransposedCDistribution =
|
|
|
|
|
WarpGemmImpl<WarpGemmAtrributeMfmaIterateKAndTransposedCDistribution<
|
|
|
|
|
WarpGemmImpl<WarpGemmAttributeMfmaIterateKAndTransposedCDistribution<
|
|
|
|
|
WarpGemmAttributeMfmaImplF16F16F32M32N32K8<WGAttrCtlEnum::Default_>,
|
|
|
|
|
2,
|
|
|
|
|
AttrNumAccess>>;
|
|
|
|
|
@@ -80,13 +80,13 @@ using WarpGemmMfmaF16F16F32M32N32K16TransposedCDistribution =
|
|
|
|
|
#if defined(__gfx950__)
|
|
|
|
|
template <WGAttrNumAccessEnum AttrNumAccess = WGAttrNumAccessEnum::Single>
|
|
|
|
|
using WarpGemmMfmaF16F16F32M16N16K32TransposedCDistribution =
|
|
|
|
|
WarpGemmImpl<WarpGemmAtrributeMfmaTransposedCDistribution<
|
|
|
|
|
WarpGemmImpl<WarpGemmAttributeMfmaTransposedCDistribution<
|
|
|
|
|
WarpGemmAttributeMfmaImplF16F16F32M16N16K32<WGAttrCtlEnum::Default_>,
|
|
|
|
|
AttrNumAccess>>;
|
|
|
|
|
#else
|
|
|
|
|
template <WGAttrNumAccessEnum AttrNumAccess = WGAttrNumAccessEnum::Single>
|
|
|
|
|
using WarpGemmMfmaF16F16F32M16N16K32TransposedCDistribution =
|
|
|
|
|
WarpGemmImpl<WarpGemmAtrributeMfmaIterateKAndTransposedCDistribution<
|
|
|
|
|
WarpGemmImpl<WarpGemmAttributeMfmaIterateKAndTransposedCDistribution<
|
|
|
|
|
WarpGemmAttributeMfmaImplF16F16F32M16N16K16<WGAttrCtlEnum::Default_>,
|
|
|
|
|
2,
|
|
|
|
|
AttrNumAccess>>;
|
|
|
|
|
@@ -94,32 +94,32 @@ using WarpGemmMfmaF16F16F32M16N16K32TransposedCDistribution =
|
|
|
|
|
|
|
|
|
|
#if defined(__gfx950__)
|
|
|
|
|
using WarpGemmMfmaF16F16F32M16N16K32SwizzleBTransposedCDistribution =
|
|
|
|
|
WarpGemmImpl<WarpGemmAtrributeMfmaTransposedCDistribution_SwizzleB<
|
|
|
|
|
WarpGemmImpl<WarpGemmAttributeMfmaTransposedCDistribution_SwizzleB<
|
|
|
|
|
WarpGemmAttributeMfmaImplF16F16F32M16N16K32<WGAttrCtlEnum::Default_>,
|
|
|
|
|
1>>;
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfmaBf16Bf16F32M16N16K32SwizzleBTransposedCDistribution =
|
|
|
|
|
WarpGemmImpl<WarpGemmAtrributeMfmaTransposedCDistribution_SwizzleB<
|
|
|
|
|
WarpGemmImpl<WarpGemmAttributeMfmaTransposedCDistribution_SwizzleB<
|
|
|
|
|
WarpGemmAttributeMfmaImplBf16Bf16F32M16N16K32<WGAttrCtlEnum::Default_>,
|
|
|
|
|
1>>;
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#if defined(__gfx950__)
|
|
|
|
|
using WarpGemmMfmaF16F16F32M32N32K16SwizzleBTransposedCDistribution =
|
|
|
|
|
WarpGemmImpl<WarpGemmAtrributeMfmaTransposedCDistribution_SwizzleB<
|
|
|
|
|
WarpGemmImpl<WarpGemmAttributeMfmaTransposedCDistribution_SwizzleB<
|
|
|
|
|
WarpGemmAttributeMfmaImplF16F16F32M32N32K16<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
#else
|
|
|
|
|
using WarpGemmMfmaF16F16F32M32N32K16SwizzleBTransposedCDistribution =
|
|
|
|
|
WarpGemmImpl<WarpGemmAtrributeMfmaIterateKAndTransposedCDistribution_SwizzleB<
|
|
|
|
|
WarpGemmImpl<WarpGemmAttributeMfmaIterateKAndTransposedCDistribution_SwizzleB<
|
|
|
|
|
WarpGemmAttributeMfmaImplF16F16F32M32N32K8<WGAttrCtlEnum::Default_>,
|
|
|
|
|
2>>;
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfmaF16F16F32M4N64K16 = WarpGemmImpl<WarpGemmAtrributeMfmaIterateK<
|
|
|
|
|
using WarpGemmMfmaF16F16F32M4N64K16 = WarpGemmImpl<WarpGemmAttributeMfmaIterateK<
|
|
|
|
|
WarpGemmAttributeMfmaImplF16F16F32M4N64K4<WGAttrCtlEnum::Default_>,
|
|
|
|
|
4>>;
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfmaF16F16F32M64N4K16 = WarpGemmImpl<WarpGemmAtrributeMfmaIterateK<
|
|
|
|
|
using WarpGemmMfmaF16F16F32M64N4K16 = WarpGemmImpl<WarpGemmAttributeMfmaIterateK<
|
|
|
|
|
WarpGemmAttributeMfmaImplF16F16F32M64N4K4<WGAttrCtlEnum::Default_>,
|
|
|
|
|
4>>;
|
|
|
|
|
|
|
|
|
|
@@ -132,19 +132,19 @@ using WarpGemmSmfmacF16F16F32M16N16K32 = WarpGemmSmfmacImpl<WarpGemmAttributeSmf
|
|
|
|
|
|
|
|
|
|
// bf16
|
|
|
|
|
using WarpGemmMfmaBf16Bf16F32M32N32K8 = WarpGemmImpl<
|
|
|
|
|
WarpGemmAtrributeMfma<WarpGemmAttributeMfmaImplBf16Bf16F32M32N32K8<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
WarpGemmAttributeMfma<WarpGemmAttributeMfmaImplBf16Bf16F32M32N32K8<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfmaBf16Bf16F32M16N16K16 = WarpGemmImpl<
|
|
|
|
|
WarpGemmAtrributeMfma<WarpGemmAttributeMfmaImplBf16Bf16F32M16N16K16<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
WarpGemmAttributeMfma<WarpGemmAttributeMfmaImplBf16Bf16F32M16N16K16<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
|
|
|
|
|
#if defined(__gfx950__)
|
|
|
|
|
template <WGAttrNumAccessEnum AttrNumAccess = WGAttrNumAccessEnum::Single>
|
|
|
|
|
using WarpGemmMfmaBf16Bf16F32M32N32K16 = WarpGemmImpl<
|
|
|
|
|
WarpGemmAtrributeMfma<WarpGemmAttributeMfmaImplBf16Bf16F32M32N32K16<WGAttrCtlEnum::Default_>,
|
|
|
|
|
WarpGemmAttributeMfma<WarpGemmAttributeMfmaImplBf16Bf16F32M32N32K16<WGAttrCtlEnum::Default_>,
|
|
|
|
|
AttrNumAccess>>;
|
|
|
|
|
#else
|
|
|
|
|
template <WGAttrNumAccessEnum AttrNumAccess = WGAttrNumAccessEnum::Single>
|
|
|
|
|
using WarpGemmMfmaBf16Bf16F32M32N32K16 = WarpGemmImpl<WarpGemmAtrributeMfmaIterateK<
|
|
|
|
|
using WarpGemmMfmaBf16Bf16F32M32N32K16 = WarpGemmImpl<WarpGemmAttributeMfmaIterateK<
|
|
|
|
|
WarpGemmAttributeMfmaImplBf16Bf16F32M32N32K8<WGAttrCtlEnum::Default_>,
|
|
|
|
|
2,
|
|
|
|
|
AttrNumAccess>>;
|
|
|
|
|
@@ -153,43 +153,43 @@ using WarpGemmMfmaBf16Bf16F32M32N32K16 = WarpGemmImpl<WarpGemmAtrributeMfmaItera
|
|
|
|
|
#if defined(__gfx950__)
|
|
|
|
|
template <WGAttrNumAccessEnum AttrNumAccess = WGAttrNumAccessEnum::Single>
|
|
|
|
|
using WarpGemmMfmaBf16Bf16F32M16N16K32 = WarpGemmImpl<
|
|
|
|
|
WarpGemmAtrributeMfma<WarpGemmAttributeMfmaImplBf16Bf16F32M16N16K32<WGAttrCtlEnum::Default_>,
|
|
|
|
|
WarpGemmAttributeMfma<WarpGemmAttributeMfmaImplBf16Bf16F32M16N16K32<WGAttrCtlEnum::Default_>,
|
|
|
|
|
AttrNumAccess>>;
|
|
|
|
|
#else
|
|
|
|
|
template <WGAttrNumAccessEnum AttrNumAccess = WGAttrNumAccessEnum::Single>
|
|
|
|
|
using WarpGemmMfmaBf16Bf16F32M16N16K32 = WarpGemmImpl<WarpGemmAtrributeMfmaIterateK<
|
|
|
|
|
using WarpGemmMfmaBf16Bf16F32M16N16K32 = WarpGemmImpl<WarpGemmAttributeMfmaIterateK<
|
|
|
|
|
WarpGemmAttributeMfmaImplBf16Bf16F32M16N16K16<WGAttrCtlEnum::Default_>,
|
|
|
|
|
2,
|
|
|
|
|
AttrNumAccess>>;
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfmaBf16Bf16F32M32N32K8SwizzleA = WarpGemmImpl<WarpGemmAtrributeMfmaIterateK_SwizzleA<
|
|
|
|
|
using WarpGemmMfmaBf16Bf16F32M32N32K8SwizzleA = WarpGemmImpl<WarpGemmAttributeMfmaIterateK_SwizzleA<
|
|
|
|
|
WarpGemmAttributeMfmaImplBf16Bf16F32M32N32K8<WGAttrCtlEnum::Default_>,
|
|
|
|
|
1>>;
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfmaBf16Bf16F32M32N32K16SwizzleA =
|
|
|
|
|
WarpGemmImpl<WarpGemmAtrributeMfmaIterateK_SwizzleA<
|
|
|
|
|
WarpGemmImpl<WarpGemmAttributeMfmaIterateK_SwizzleA<
|
|
|
|
|
WarpGemmAttributeMfmaImplBf16Bf16F32M32N32K8<WGAttrCtlEnum::Default_>,
|
|
|
|
|
2>>;
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfmaBf16Bf16F32M32N32K8TransposedCDistribution =
|
|
|
|
|
WarpGemmImpl<WarpGemmAtrributeMfmaTransposedCDistribution<
|
|
|
|
|
WarpGemmImpl<WarpGemmAttributeMfmaTransposedCDistribution<
|
|
|
|
|
WarpGemmAttributeMfmaImplBf16Bf16F32M32N32K8<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfmaBf16Bf16F32M16N16K16TransposedCDistribution =
|
|
|
|
|
WarpGemmImpl<WarpGemmAtrributeMfmaTransposedCDistribution<
|
|
|
|
|
WarpGemmImpl<WarpGemmAttributeMfmaTransposedCDistribution<
|
|
|
|
|
WarpGemmAttributeMfmaImplBf16Bf16F32M16N16K16<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
|
|
|
|
|
#if defined(__gfx950__)
|
|
|
|
|
template <WGAttrNumAccessEnum AttrNumAccess = WGAttrNumAccessEnum::Single>
|
|
|
|
|
using WarpGemmMfmaBf16Bf16F32M32N32K16TransposedCDistribution =
|
|
|
|
|
WarpGemmImpl<WarpGemmAtrributeMfmaTransposedCDistribution<
|
|
|
|
|
WarpGemmImpl<WarpGemmAttributeMfmaTransposedCDistribution<
|
|
|
|
|
WarpGemmAttributeMfmaImplBf16Bf16F32M32N32K16<WGAttrCtlEnum::Default_>,
|
|
|
|
|
AttrNumAccess>>;
|
|
|
|
|
#else
|
|
|
|
|
template <WGAttrNumAccessEnum AttrNumAccess = WGAttrNumAccessEnum::Single>
|
|
|
|
|
using WarpGemmMfmaBf16Bf16F32M32N32K16TransposedCDistribution =
|
|
|
|
|
WarpGemmImpl<WarpGemmAtrributeMfmaIterateKAndTransposedCDistribution<
|
|
|
|
|
WarpGemmImpl<WarpGemmAttributeMfmaIterateKAndTransposedCDistribution<
|
|
|
|
|
WarpGemmAttributeMfmaImplBf16Bf16F32M32N32K8<WGAttrCtlEnum::Default_>,
|
|
|
|
|
2,
|
|
|
|
|
AttrNumAccess>>;
|
|
|
|
|
@@ -198,13 +198,13 @@ using WarpGemmMfmaBf16Bf16F32M32N32K16TransposedCDistribution =
|
|
|
|
|
#if defined(__gfx950__)
|
|
|
|
|
template <WGAttrNumAccessEnum AttrNumAccess = WGAttrNumAccessEnum::Single>
|
|
|
|
|
using WarpGemmMfmaBf16Bf16F32M16N16K32TransposedCDistribution =
|
|
|
|
|
WarpGemmImpl<WarpGemmAtrributeMfmaTransposedCDistribution<
|
|
|
|
|
WarpGemmImpl<WarpGemmAttributeMfmaTransposedCDistribution<
|
|
|
|
|
WarpGemmAttributeMfmaImplBf16Bf16F32M16N16K32<WGAttrCtlEnum::Default_>,
|
|
|
|
|
AttrNumAccess>>;
|
|
|
|
|
#else
|
|
|
|
|
template <WGAttrNumAccessEnum AttrNumAccess = WGAttrNumAccessEnum::Single>
|
|
|
|
|
using WarpGemmMfmaBf16Bf16F32M16N16K32TransposedCDistribution =
|
|
|
|
|
WarpGemmImpl<WarpGemmAtrributeMfmaIterateKAndTransposedCDistribution<
|
|
|
|
|
WarpGemmImpl<WarpGemmAttributeMfmaIterateKAndTransposedCDistribution<
|
|
|
|
|
WarpGemmAttributeMfmaImplBf16Bf16F32M16N16K16<WGAttrCtlEnum::Default_>,
|
|
|
|
|
2,
|
|
|
|
|
AttrNumAccess>>;
|
|
|
|
|
@@ -212,127 +212,127 @@ using WarpGemmMfmaBf16Bf16F32M16N16K32TransposedCDistribution =
|
|
|
|
|
|
|
|
|
|
#if defined(__gfx950__)
|
|
|
|
|
using WarpGemmMfmaBf16Bf16F32M32N32K16SwizzleBTransposedCDistribution =
|
|
|
|
|
WarpGemmImpl<WarpGemmAtrributeMfmaTransposedCDistribution_SwizzleB<
|
|
|
|
|
WarpGemmImpl<WarpGemmAttributeMfmaTransposedCDistribution_SwizzleB<
|
|
|
|
|
WarpGemmAttributeMfmaImplBf16Bf16F32M32N32K16<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
#else
|
|
|
|
|
using WarpGemmMfmaBf16Bf16F32M32N32K16SwizzleBTransposedCDistribution =
|
|
|
|
|
WarpGemmImpl<WarpGemmAtrributeMfmaIterateKAndTransposedCDistribution_SwizzleB<
|
|
|
|
|
WarpGemmImpl<WarpGemmAttributeMfmaIterateKAndTransposedCDistribution_SwizzleB<
|
|
|
|
|
WarpGemmAttributeMfmaImplBf16Bf16F32M32N32K8<WGAttrCtlEnum::Default_>,
|
|
|
|
|
2>>;
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfmaBf16Bf16F32M4N64K16 = WarpGemmImpl<WarpGemmAtrributeMfmaIterateK<
|
|
|
|
|
using WarpGemmMfmaBf16Bf16F32M4N64K16 = WarpGemmImpl<WarpGemmAttributeMfmaIterateK<
|
|
|
|
|
WarpGemmAttributeMfmaImplBf16Bf16F32M4N64K4<WGAttrCtlEnum::Default_>,
|
|
|
|
|
4>>;
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfmaBf16Bf16F32M64N4K16 = WarpGemmImpl<WarpGemmAtrributeMfmaIterateK<
|
|
|
|
|
using WarpGemmMfmaBf16Bf16F32M64N4K16 = WarpGemmImpl<WarpGemmAttributeMfmaIterateK<
|
|
|
|
|
WarpGemmAttributeMfmaImplBf16Bf16F32M64N4K4<WGAttrCtlEnum::Default_>,
|
|
|
|
|
4>>;
|
|
|
|
|
|
|
|
|
|
// fp8
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfma_f32_32x32x16_fp8_fp8 = WarpGemmImpl<
|
|
|
|
|
WarpGemmAtrributeMfma<WarpGemmAttributeMfmaImpl_f32_32x32x16_fp8_fp8<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
WarpGemmAttributeMfma<WarpGemmAttributeMfmaImpl_f32_32x32x16_fp8_fp8<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfma_f32_32x32x16_fp8_bf8 = WarpGemmImpl<
|
|
|
|
|
WarpGemmAtrributeMfma<WarpGemmAttributeMfmaImpl_f32_32x32x16_fp8_bf8<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
WarpGemmAttributeMfma<WarpGemmAttributeMfmaImpl_f32_32x32x16_fp8_bf8<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfma_f32_32x32x16_bf8_fp8 = WarpGemmImpl<
|
|
|
|
|
WarpGemmAtrributeMfma<WarpGemmAttributeMfmaImpl_f32_32x32x16_bf8_fp8<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
WarpGemmAttributeMfma<WarpGemmAttributeMfmaImpl_f32_32x32x16_bf8_fp8<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfma_f32_32x32x16_bf8_bf8 = WarpGemmImpl<
|
|
|
|
|
WarpGemmAtrributeMfma<WarpGemmAttributeMfmaImpl_f32_32x32x16_bf8_bf8<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
WarpGemmAttributeMfma<WarpGemmAttributeMfmaImpl_f32_32x32x16_bf8_bf8<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfma_f32_32x32x32_fp8_fp8 = WarpGemmImpl<WarpGemmAtrributeMfmaIterateK<
|
|
|
|
|
using WarpGemmMfma_f32_32x32x32_fp8_fp8 = WarpGemmImpl<WarpGemmAttributeMfmaIterateK<
|
|
|
|
|
WarpGemmAttributeMfmaImpl_f32_32x32x16_fp8_fp8<WGAttrCtlEnum::Default_>,
|
|
|
|
|
2>>;
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfma_f32_32x32x32_bf8_bf8 = WarpGemmImpl<WarpGemmAtrributeMfmaIterateK<
|
|
|
|
|
using WarpGemmMfma_f32_32x32x32_bf8_bf8 = WarpGemmImpl<WarpGemmAttributeMfmaIterateK<
|
|
|
|
|
WarpGemmAttributeMfmaImpl_f32_32x32x16_bf8_bf8<WGAttrCtlEnum::Default_>,
|
|
|
|
|
2>>;
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfma_f32_16x16x32_fp8_fp8 = WarpGemmImpl<
|
|
|
|
|
WarpGemmAtrributeMfma<WarpGemmAttributeMfmaImpl_f32_16x16x32_fp8_fp8<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
WarpGemmAttributeMfma<WarpGemmAttributeMfmaImpl_f32_16x16x32_fp8_fp8<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfma_f32_16x16x32_bf8_bf8 = WarpGemmImpl<
|
|
|
|
|
WarpGemmAtrributeMfma<WarpGemmAttributeMfmaImpl_f32_16x16x32_bf8_bf8<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
WarpGemmAttributeMfma<WarpGemmAttributeMfmaImpl_f32_16x16x32_bf8_bf8<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfma_f32_16x16x64_fp8_fp8 = WarpGemmImpl<WarpGemmAtrributeMfmaIterateK<
|
|
|
|
|
using WarpGemmMfma_f32_16x16x64_fp8_fp8 = WarpGemmImpl<WarpGemmAttributeMfmaIterateK<
|
|
|
|
|
WarpGemmAttributeMfmaImpl_f32_16x16x32_fp8_fp8<WGAttrCtlEnum::Default_>,
|
|
|
|
|
2>>;
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfma_f32_16x16x64_bf8_bf8 = WarpGemmImpl<WarpGemmAtrributeMfmaIterateK<
|
|
|
|
|
using WarpGemmMfma_f32_16x16x64_bf8_bf8 = WarpGemmImpl<WarpGemmAttributeMfmaIterateK<
|
|
|
|
|
WarpGemmAttributeMfmaImpl_f32_16x16x32_bf8_bf8<WGAttrCtlEnum::Default_>,
|
|
|
|
|
2>>;
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfma_f32_16x16x128_fp8_fp8 = WarpGemmImpl<WarpGemmAtrributeMfma<
|
|
|
|
|
using WarpGemmMfma_f32_16x16x128_fp8_fp8 = WarpGemmImpl<WarpGemmAttributeMfma<
|
|
|
|
|
WarpGemmAttributeMfmaImpl_f32_16x16x128_fp8_fp8<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfma_f32_16x16x128_fp8_bf8 = WarpGemmImpl<WarpGemmAtrributeMfma<
|
|
|
|
|
using WarpGemmMfma_f32_16x16x128_fp8_bf8 = WarpGemmImpl<WarpGemmAttributeMfma<
|
|
|
|
|
WarpGemmAttributeMfmaImpl_f32_16x16x128_fp8_bf8<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfma_f32_16x16x128_bf8_fp8 = WarpGemmImpl<WarpGemmAtrributeMfma<
|
|
|
|
|
using WarpGemmMfma_f32_16x16x128_bf8_fp8 = WarpGemmImpl<WarpGemmAttributeMfma<
|
|
|
|
|
WarpGemmAttributeMfmaImpl_f32_16x16x128_bf8_fp8<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfma_f32_16x16x128_bf8_bf8 = WarpGemmImpl<WarpGemmAtrributeMfma<
|
|
|
|
|
using WarpGemmMfma_f32_16x16x128_bf8_bf8 = WarpGemmImpl<WarpGemmAttributeMfma<
|
|
|
|
|
WarpGemmAttributeMfmaImpl_f32_16x16x128_bf8_bf8<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
|
|
|
|
|
template <WGAttrNumAccessEnum AttrNumAccess = WGAttrNumAccessEnum::Single>
|
|
|
|
|
using WarpGemmMfma_f32_32x32x64_fp8_fp8 = WarpGemmImpl<
|
|
|
|
|
WarpGemmAtrributeMfma<WarpGemmAttributeMfmaImpl_f32_32x32x64_fp8_fp8<WGAttrCtlEnum::Default_>,
|
|
|
|
|
WarpGemmAttributeMfma<WarpGemmAttributeMfmaImpl_f32_32x32x64_fp8_fp8<WGAttrCtlEnum::Default_>,
|
|
|
|
|
AttrNumAccess>>;
|
|
|
|
|
|
|
|
|
|
template <WGAttrNumAccessEnum AttrNumAccess = WGAttrNumAccessEnum::Single>
|
|
|
|
|
using WarpGemmMfma_f32_32x32x64_fp8_bf8 = WarpGemmImpl<
|
|
|
|
|
WarpGemmAtrributeMfma<WarpGemmAttributeMfmaImpl_f32_32x32x64_fp8_bf8<WGAttrCtlEnum::Default_>,
|
|
|
|
|
WarpGemmAttributeMfma<WarpGemmAttributeMfmaImpl_f32_32x32x64_fp8_bf8<WGAttrCtlEnum::Default_>,
|
|
|
|
|
AttrNumAccess>>;
|
|
|
|
|
|
|
|
|
|
template <WGAttrNumAccessEnum AttrNumAccess = WGAttrNumAccessEnum::Single>
|
|
|
|
|
using WarpGemmMfma_f32_32x32x64_bf8_fp8 = WarpGemmImpl<
|
|
|
|
|
WarpGemmAtrributeMfma<WarpGemmAttributeMfmaImpl_f32_32x32x64_bf8_fp8<WGAttrCtlEnum::Default_>,
|
|
|
|
|
WarpGemmAttributeMfma<WarpGemmAttributeMfmaImpl_f32_32x32x64_bf8_fp8<WGAttrCtlEnum::Default_>,
|
|
|
|
|
AttrNumAccess>>;
|
|
|
|
|
|
|
|
|
|
template <WGAttrNumAccessEnum AttrNumAccess = WGAttrNumAccessEnum::Single>
|
|
|
|
|
using WarpGemmMfma_f32_32x32x64_bf8_bf8 = WarpGemmImpl<
|
|
|
|
|
WarpGemmAtrributeMfma<WarpGemmAttributeMfmaImpl_f32_32x32x64_bf8_bf8<WGAttrCtlEnum::Default_>,
|
|
|
|
|
WarpGemmAttributeMfma<WarpGemmAttributeMfmaImpl_f32_32x32x64_bf8_bf8<WGAttrCtlEnum::Default_>,
|
|
|
|
|
AttrNumAccess>>;
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfma_f32_32x32x16_fp8_fp8_CTransposed =
|
|
|
|
|
WarpGemmImpl<WarpGemmAtrributeMfmaTransposedCDistribution<
|
|
|
|
|
WarpGemmImpl<WarpGemmAttributeMfmaTransposedCDistribution<
|
|
|
|
|
WarpGemmAttributeMfmaImpl_f32_32x32x16_fp8_fp8<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfma_f32_32x32x16_fp8_bf8_CTransposed =
|
|
|
|
|
WarpGemmImpl<WarpGemmAtrributeMfmaTransposedCDistribution<
|
|
|
|
|
WarpGemmImpl<WarpGemmAttributeMfmaTransposedCDistribution<
|
|
|
|
|
WarpGemmAttributeMfmaImpl_f32_32x32x16_fp8_bf8<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfma_f32_32x32x16_bf8_fp8_CTransposed =
|
|
|
|
|
WarpGemmImpl<WarpGemmAtrributeMfmaTransposedCDistribution<
|
|
|
|
|
WarpGemmImpl<WarpGemmAttributeMfmaTransposedCDistribution<
|
|
|
|
|
WarpGemmAttributeMfmaImpl_f32_32x32x16_bf8_fp8<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfma_f32_32x32x16_bf8_bf8_CTransposed =
|
|
|
|
|
WarpGemmImpl<WarpGemmAtrributeMfmaTransposedCDistribution<
|
|
|
|
|
WarpGemmImpl<WarpGemmAttributeMfmaTransposedCDistribution<
|
|
|
|
|
WarpGemmAttributeMfmaImpl_f32_32x32x16_bf8_bf8<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
|
|
|
|
|
template <index_t swizzle_factor = 2>
|
|
|
|
|
using WarpGemmMfmaFp8Fp8F32M32N32K16SwizzleBTransposedCDistribution =
|
|
|
|
|
WarpGemmImpl<WarpGemmAtrributeMfmaIterateKAndTransposedCDistribution_SwizzleB<
|
|
|
|
|
WarpGemmImpl<WarpGemmAttributeMfmaIterateKAndTransposedCDistribution_SwizzleB<
|
|
|
|
|
WarpGemmAttributeMfmaImpl_f32_32x32x16_f8_base<fp8_t, fp8_t, WGAttrCtlEnum::Default_>,
|
|
|
|
|
2,
|
|
|
|
|
swizzle_factor>>;
|
|
|
|
|
|
|
|
|
|
// int8
|
|
|
|
|
using WarpGemmMfma_i32_32x32x16_i8_i8 = WarpGemmImpl<
|
|
|
|
|
WarpGemmAtrributeMfma<WarpGemmAttributeMfmaImpl_i32_32x32x16_i8<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
WarpGemmAttributeMfma<WarpGemmAttributeMfmaImpl_i32_32x32x16_i8<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfma_i32_32x32x16_i8_i8_CTransposed =
|
|
|
|
|
WarpGemmImpl<WarpGemmAtrributeMfmaTransposedCDistribution<
|
|
|
|
|
WarpGemmImpl<WarpGemmAttributeMfmaTransposedCDistribution<
|
|
|
|
|
WarpGemmAttributeMfmaImpl_i32_32x32x16_i8<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfma_i32_16x16x32_i8_i8 = WarpGemmImpl<
|
|
|
|
|
WarpGemmAtrributeMfma<WarpGemmAttributeMfmaImpl_i32_16x16x32_i8<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
WarpGemmAttributeMfma<WarpGemmAttributeMfmaImpl_i32_16x16x32_i8<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
|
|
|
|
|
using WarpGemmMfma_i32_16x16x32_i8_i8_CTransposed =
|
|
|
|
|
WarpGemmImpl<WarpGemmAtrributeMfmaTransposedCDistribution<
|
|
|
|
|
WarpGemmImpl<WarpGemmAttributeMfmaTransposedCDistribution<
|
|
|
|
|
WarpGemmAttributeMfmaImpl_i32_16x16x32_i8<WGAttrCtlEnum::Default_>>>;
|
|
|
|
|
|
|
|
|
|
} // namespace ck_tile
|
|
|
|
|
|