[Bugfix] Fix mHC block-M prenorm GEMM cross-row reduction carry-over (#49429)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -856,21 +856,23 @@ def hc_prenorm_gemm_block_m_tilelang(
|
||||
T.sync_threads()
|
||||
|
||||
if warp_id == 0:
|
||||
reduced_acc = T.alloc_local((block_m,), T.float32)
|
||||
reduced_sqr = T.alloc_local((block_m,), T.float32)
|
||||
T.clear(reduced_acc)
|
||||
T.clear(reduced_sqr)
|
||||
for i_m in T.unroll(block_m):
|
||||
token_idx = i_mt * block_m + i_m
|
||||
if token_idx < num_tokens:
|
||||
if lane < tile_n:
|
||||
reduced_acc = T.alloc_var(T.float32, init=0.0)
|
||||
for i_w in T.unroll(num_warps):
|
||||
reduced_acc += warp_acc[i_w, i_m, lane]
|
||||
reduced_acc[i_m] += warp_acc[i_w, i_m, lane]
|
||||
out_idx = i_t * tile_n + lane
|
||||
if out_idx < n_out:
|
||||
out[0, token_idx, out_idx] = reduced_acc
|
||||
out[0, token_idx, out_idx] = reduced_acc[i_m]
|
||||
if lane == 0 and i_t == 0:
|
||||
reduced_sqr = T.alloc_var(T.float32, init=0.0)
|
||||
for i_w in T.unroll(num_warps):
|
||||
reduced_sqr += warp_sqr[i_w, i_m]
|
||||
sqrsum[0, token_idx] = reduced_sqr
|
||||
reduced_sqr[i_m] += warp_sqr[i_w, i_m]
|
||||
sqrsum[0, token_idx] = reduced_sqr[i_m]
|
||||
|
||||
if ENABLE_PDL:
|
||||
T.pdl_trigger()
|
||||
|
||||
Reference in New Issue
Block a user