[Kernel] Initial Machete W4A8 support + Refactors (#9855)
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
This commit is contained in:
@@ -171,6 +171,10 @@ struct MacheteCollectiveMma {
|
||||
make_shape(size<0>(TileShape_MNK{}), size<2>(TileShape_MNK{}),
|
||||
Int<DispatchPolicy::Stages>{})));
|
||||
|
||||
using SmemLayoutACopy = decltype(GmemLayoutA::TVbNbKL_to_offset_copy(
|
||||
make_shape(size<0>(TileShape_MNK{}), size<2>(TileShape_MNK{}),
|
||||
Int<DispatchPolicy::Stages>{})));
|
||||
|
||||
using SmemLayoutAtomARowMajor =
|
||||
decltype(rs_smem_selector<GmmaMajorA, ElementA,
|
||||
decltype(cute::get<0>(TileShape_MNK{})),
|
||||
@@ -288,14 +292,7 @@ struct MacheteCollectiveMma {
|
||||
static_assert((size<2>(TileShape{}) % size<1>(SmemLayoutAtomScale{})) == 0,
|
||||
"SmemLayoutAtomScale must evenly divide tile k shape.");
|
||||
|
||||
// Tile along modes in a way that maximizes the TMA box size.
|
||||
using SmemLayoutACopy = decltype(tile_to_shape(
|
||||
SmemLayoutAtomARowMajor{},
|
||||
make_shape(shape<0>(TileShape{}), shape<2>(TileShape{}),
|
||||
Int<DispatchPolicy::Stages>{}),
|
||||
conditional_t<::cutlass::gemm::detail::is_major<0, StrideA>(),
|
||||
Step<_2, _1, _3>, Step<_1, _2, _3>>{}));
|
||||
|
||||
// Tile along modes in a way that maximizes the TMA box size
|
||||
using SmemLayoutB = decltype(tile_to_shape(
|
||||
SmemLayoutAtomB{},
|
||||
make_shape(shape<1>(TileShape{}), shape<2>(TileShape{}),
|
||||
@@ -428,12 +425,12 @@ struct MacheteCollectiveMma {
|
||||
// clang-format on
|
||||
|
||||
// ((athrid, val), (BlocksM, BlockK), L) -> (storage_idx)
|
||||
using PrepackedStrideA = decltype(stride(GmemLayoutA::TVbNbKL_to_offset(
|
||||
using PrepackedStrideA = decltype(stride(GmemLayoutA::TVbNbKL_to_offset_copy(
|
||||
make_shape(int32_t(0), int32_t(0), int32_t(0)))));
|
||||
|
||||
using ATensor = decltype(make_tensor(
|
||||
get_logical_ptr(static_cast<InternalElementA const*>(nullptr)),
|
||||
shape(GmemLayoutA::TVbNbKL_to_offset(
|
||||
shape(GmemLayoutA::TVbNbKL_to_offset_copy(
|
||||
make_shape(int32_t(0), int32_t(0), int32_t(0)))),
|
||||
PrepackedStrideA{}));
|
||||
|
||||
@@ -450,8 +447,8 @@ struct MacheteCollectiveMma {
|
||||
|
||||
static constexpr auto make_tma_copy_A(ATensor tensor_a = ATensor{}) {
|
||||
return make_tma_copy<TmaElementA>(
|
||||
GmemTiledCopyA{}, tensor_a, SmemLayoutA{}(_, _, cute::Int<0>{}),
|
||||
shape(SmemLayoutA{}(_, _, cute::Int<0>{})),
|
||||
GmemTiledCopyA{}, tensor_a, SmemLayoutACopy{}(_, _, cute::Int<0>{}),
|
||||
shape(SmemLayoutACopy{}(_, _, cute::Int<0>{})),
|
||||
size<1>(ClusterShape{})); // mcast along N mode for this M load, if any
|
||||
}
|
||||
|
||||
@@ -584,7 +581,7 @@ struct MacheteCollectiveMma {
|
||||
typename Params::TMA_Scale tma_load_scale;
|
||||
typename Params::TMA_Zero tma_load_zero;
|
||||
|
||||
auto layout = GmemLayoutA::TVbNbKL_to_offset(make_shape(M, K, L));
|
||||
auto layout = GmemLayoutA::TVbNbKL_to_offset_copy(make_shape(M, K, L));
|
||||
tma_load_a = make_tma_copy_A(
|
||||
make_logical_tensor(ptr_A, shape(layout), stride(layout)));
|
||||
|
||||
@@ -722,7 +719,7 @@ struct MacheteCollectiveMma {
|
||||
// (TILE_V,TILE_B,m,k,l)
|
||||
auto make_gA_mkl = [&]() {
|
||||
// ((athrid, val), (BlocksM, BlockK), L) -> (storage_idx)
|
||||
auto layout = GmemLayoutA::TVbNbKL_to_offset(make_shape(M, K, L));
|
||||
auto layout = GmemLayoutA::TVbNbKL_to_offset_copy(make_shape(M, K, L));
|
||||
Tensor mA_mkl = mainloop_params.tma_load_a.get_tma_tensor(shape(layout));
|
||||
return local_tile(mA_mkl,
|
||||
make_shape(size<0>(layout), PPBlocksPerTile_MK{}),
|
||||
|
||||
Reference in New Issue
Block a user