[Kernel][DSV4.1] Fuse MXFP8 wo_b GEMM with sequence-parallel reduce-scatter (#57428)
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg> Signed-off-by: Canlin <canlinguosdu@gmail.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
co-authored by
Thien Tran
OpenAI Codex
Claude Fable 5.1
Jee Jee Li
parent
a6c47fbbf4
commit
f92b78f6ef
@@ -78,6 +78,10 @@ def make_sdesc_128B_swizzle(LBO: int):
|
||||
return Uint64((LBO >> 4 << 16) | (SBO >> 4 << 32) | (1 << 46) | (2 << 61))
|
||||
|
||||
|
||||
def make_mxfp8_idesc(MMA_M: int, MMA_N: int):
|
||||
return Uint32((MMA_N >> 3 << 17) | (1 << 23) | (MMA_M >> 7 << 27))
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def mma_f16(
|
||||
d_tmem,
|
||||
@@ -130,6 +134,73 @@ def mma_ts_f16(
|
||||
)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def mma_mxfp8(
|
||||
d_tmem,
|
||||
a_desc,
|
||||
b_desc,
|
||||
idesc,
|
||||
sfa_tmem,
|
||||
sfb_tmem,
|
||||
enable_input_d,
|
||||
cta_group: int = 1,
|
||||
*,
|
||||
loc=None,
|
||||
ip=None,
|
||||
) -> None:
|
||||
with cute.arch.elect_one():
|
||||
nvvm.tcgen05_mma_block_scale(
|
||||
nvvm.Tcgen05MMAKind.MXF8F6F4,
|
||||
NVVM_CTA_GROUP_MAP[cta_group],
|
||||
_make_tmem_llvm_ptr(d_tmem, loc=loc, ip=ip),
|
||||
Uint64(a_desc).ir_value(loc=loc, ip=ip),
|
||||
Uint64(b_desc).ir_value(loc=loc, ip=ip),
|
||||
Int32(idesc).ir_value(loc=loc, ip=ip),
|
||||
Boolean(enable_input_d).ir_value(loc=loc, ip=ip),
|
||||
_make_tmem_llvm_ptr(sfa_tmem, loc=loc, ip=ip),
|
||||
_make_tmem_llvm_ptr(sfb_tmem, loc=loc, ip=ip),
|
||||
scale_vec_size=nvvm.Tcgen05MMAScaleVecSize.X1,
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def cp(
|
||||
tmem,
|
||||
sdesc,
|
||||
shape: str,
|
||||
mcast: str,
|
||||
cta_group: int = 1,
|
||||
*,
|
||||
loc=None,
|
||||
ip=None,
|
||||
) -> None:
|
||||
SHAPE_MAP = {
|
||||
"128x256b": nvvm.Tcgen05CpShape.SHAPE_128x256b,
|
||||
"4x256b": nvvm.Tcgen05CpShape.SHAPE_4x256b,
|
||||
"128x128b": nvvm.Tcgen05CpShape.SHAPE_128x128b,
|
||||
"64x128b": nvvm.Tcgen05CpShape.SHAPE_64x128b,
|
||||
"32x128b": nvvm.Tcgen05CpShape.SHAPE_32x128b,
|
||||
}
|
||||
MCAST_MAP = {
|
||||
"none": nvvm.Tcgen05CpMulticast.NONE,
|
||||
"warpx2::02_13": nvvm.Tcgen05CpMulticast.WARPX2_02_13,
|
||||
"warpx2::01_23": nvvm.Tcgen05CpMulticast.WARPX2_01_23,
|
||||
"warpx4": nvvm.Tcgen05CpMulticast.WARPX4,
|
||||
}
|
||||
with cute.arch.elect_one():
|
||||
nvvm.tcgen05_cp(
|
||||
SHAPE_MAP[shape],
|
||||
_make_tmem_llvm_ptr(tmem, loc=loc, ip=ip),
|
||||
Uint64(sdesc).ir_value(loc=loc, ip=ip),
|
||||
group=NVVM_CTA_GROUP_MAP[cta_group],
|
||||
multicast=MCAST_MAP[mcast],
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def commit(mbar, cta_mask=None, cta_group: int = 1, *, loc=None, ip=None):
|
||||
mbar_llvm = mbar.to_llvm_ptr(loc=loc, ip=ip)
|
||||
|
||||
Reference in New Issue
Block a user