[Kernel][DSV4.1] Fuse MXFP8 wo_b GEMM with sequence-parallel reduce-scatter (#57428)

Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Signed-off-by: Canlin <canlinguosdu@gmail.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
Canlin Guo
2026-09-22 01:07:17 -07:00
committed by GitHub
co-authored by Thien Tran OpenAI Codex Claude Fable 5.1 Jee Jee Li
parent a6c47fbbf4
commit f92b78f6ef
18 changed files with 1037 additions and 624 deletions
+71
View File
@@ -78,6 +78,10 @@ def make_sdesc_128B_swizzle(LBO: int):
return Uint64((LBO >> 4 << 16) | (SBO >> 4 << 32) | (1 << 46) | (2 << 61))
def make_mxfp8_idesc(MMA_M: int, MMA_N: int):
return Uint32((MMA_N >> 3 << 17) | (1 << 23) | (MMA_M >> 7 << 27))
@dsl_user_op
def mma_f16(
d_tmem,
@@ -130,6 +134,73 @@ def mma_ts_f16(
)
@dsl_user_op
def mma_mxfp8(
d_tmem,
a_desc,
b_desc,
idesc,
sfa_tmem,
sfb_tmem,
enable_input_d,
cta_group: int = 1,
*,
loc=None,
ip=None,
) -> None:
with cute.arch.elect_one():
nvvm.tcgen05_mma_block_scale(
nvvm.Tcgen05MMAKind.MXF8F6F4,
NVVM_CTA_GROUP_MAP[cta_group],
_make_tmem_llvm_ptr(d_tmem, loc=loc, ip=ip),
Uint64(a_desc).ir_value(loc=loc, ip=ip),
Uint64(b_desc).ir_value(loc=loc, ip=ip),
Int32(idesc).ir_value(loc=loc, ip=ip),
Boolean(enable_input_d).ir_value(loc=loc, ip=ip),
_make_tmem_llvm_ptr(sfa_tmem, loc=loc, ip=ip),
_make_tmem_llvm_ptr(sfb_tmem, loc=loc, ip=ip),
scale_vec_size=nvvm.Tcgen05MMAScaleVecSize.X1,
loc=loc,
ip=ip,
)
@dsl_user_op
def cp(
tmem,
sdesc,
shape: str,
mcast: str,
cta_group: int = 1,
*,
loc=None,
ip=None,
) -> None:
SHAPE_MAP = {
"128x256b": nvvm.Tcgen05CpShape.SHAPE_128x256b,
"4x256b": nvvm.Tcgen05CpShape.SHAPE_4x256b,
"128x128b": nvvm.Tcgen05CpShape.SHAPE_128x128b,
"64x128b": nvvm.Tcgen05CpShape.SHAPE_64x128b,
"32x128b": nvvm.Tcgen05CpShape.SHAPE_32x128b,
}
MCAST_MAP = {
"none": nvvm.Tcgen05CpMulticast.NONE,
"warpx2::02_13": nvvm.Tcgen05CpMulticast.WARPX2_02_13,
"warpx2::01_23": nvvm.Tcgen05CpMulticast.WARPX2_01_23,
"warpx4": nvvm.Tcgen05CpMulticast.WARPX4,
}
with cute.arch.elect_one():
nvvm.tcgen05_cp(
SHAPE_MAP[shape],
_make_tmem_llvm_ptr(tmem, loc=loc, ip=ip),
Uint64(sdesc).ir_value(loc=loc, ip=ip),
group=NVVM_CTA_GROUP_MAP[cta_group],
multicast=MCAST_MAP[mcast],
loc=loc,
ip=ip,
)
@dsl_user_op
def commit(mbar, cta_mask=None, cta_group: int = 1, *, loc=None, ip=None):
mbar_llvm = mbar.to_llvm_ptr(loc=loc, ip=ip)