diff --git a/csrc/cutlass_extensions/epilogue/broadcast_load_epilogue_array_c3x.hpp b/csrc/cutlass_extensions/epilogue/broadcast_load_epilogue_array_c3x.hpp index 8aa99b3e03..0a3c9e9cc7 100644 --- a/csrc/cutlass_extensions/epilogue/broadcast_load_epilogue_array_c3x.hpp +++ b/csrc/cutlass_extensions/epilogue/broadcast_load_epilogue_array_c3x.hpp @@ -389,20 +389,28 @@ struct Sm90ColOrScalarBroadcastArray { CUTLASS_DEVICE void begin() { - cute::Tensor pred = make_tensor(shape(tCgCol)); - CUTLASS_PRAGMA_UNROLL - for (int i = 0; i < size(pred); ++i) { - pred(i) = get<0>(tCcCol(i)) < m; - } - if (!params.col_broadcast) { fill(tCrCol, *(params.ptr_col_array[group])); return; } - // Filter so we don't issue redundant copies over stride-0 modes - // (only works if 0-strides are in same location, which is by construction) - copy_if(pred, filter(tCgCol), filter(tCrCol)); + // tCgCol has layout (CPY,CPY_M,CPY_N,EPI_M,EPI_N) where CPY_N and + // EPI_N are stride-0 for the column broadcast. Slice those modes at + // index 0 to avoid redundant copies AND ensure pred/data consistency + static_assert(decltype(stride<2>(tCgCol))::value == 0, "Expected stride-0 CPY_N for col broadcast"); + static_assert(decltype(stride<4>(tCgCol))::value == 0, "Expected stride-0 EPI_N for col broadcast"); + + auto tCgCol_s = tCgCol(_,_,0,_,0); // (CPY,CPY_M,EPI_M) + auto tCrCol_s = tCrCol(_,_,0,_,0); // (CPY,CPY_M,EPI_M) + auto tCcCol_s = tCcCol(_,_,0,_,0); // (CPY,CPY_M,EPI_M) + + cute::Tensor pred = make_tensor(shape(tCgCol_s)); + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size(pred); ++i) { + pred(i) = get<0>(tCcCol_s(i)) < m; + } + + copy_if(pred, tCgCol_s, tCrCol_s); } template diff --git a/csrc/cutlass_extensions/epilogue/broadcast_load_epilogue_c3x.hpp b/csrc/cutlass_extensions/epilogue/broadcast_load_epilogue_c3x.hpp index 8203d8930c..29e6ec41e2 100644 --- a/csrc/cutlass_extensions/epilogue/broadcast_load_epilogue_c3x.hpp +++ b/csrc/cutlass_extensions/epilogue/broadcast_load_epilogue_c3x.hpp @@ -382,20 +382,28 @@ struct Sm90ColOrScalarBroadcast { CUTLASS_DEVICE void begin() { - cute::Tensor pred = make_tensor(shape(tCgCol)); - CUTLASS_PRAGMA_UNROLL - for (int i = 0; i < size(pred); ++i) { - pred(i) = get<0>(tCcCol(i)) < m; - } - if (!params.col_broadcast) { fill(tCrCol, *(params.ptr_col)); return; } - // Filter so we don't issue redundant copies over stride-0 modes - // (only works if 0-strides are in same location, which is by construction) - copy_if(pred, filter(tCgCol), filter(tCrCol)); + // tCgCol has layout (CPY,CPY_M,CPY_N,EPI_M,EPI_N) where CPY_N and + // EPI_N are stride-0 for the column broadcast. Slice those modes at + // index 0 to avoid redundant copies AND ensure pred/data consistency + static_assert(decltype(stride<2>(tCgCol))::value == 0, "Expected stride-0 CPY_N for col broadcast"); + static_assert(decltype(stride<4>(tCgCol))::value == 0, "Expected stride-0 EPI_N for col broadcast"); + + auto tCgCol_s = tCgCol(_,_,0,_,0); // (CPY,CPY_M,EPI_M) + auto tCrCol_s = tCrCol(_,_,0,_,0); // (CPY,CPY_M,EPI_M) + auto tCcCol_s = tCcCol(_,_,0,_,0); // (CPY,CPY_M,EPI_M) + + cute::Tensor pred = make_tensor(shape(tCgCol_s)); + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size(pred); ++i) { + pred(i) = get<0>(tCcCol_s(i)) < m; + } + + copy_if(pred, tCgCol_s, tCrCol_s); } template