Fix write-after-write hazard in xpu KV RoPE triton kernel (#46685)
Signed-off-by: Wenbin Chen <wenbin.chen@intel.com> Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Misha Goin
parent
698e79f3bd
commit
6463d04601
@@ -88,10 +88,13 @@ def _xpu_qnorm_rope_kernel(
|
||||
kv_base = kv_ptr + token_idx * HEAD_DIM
|
||||
kv_out_base = kv_out_ptr + token_idx * HEAD_DIM
|
||||
|
||||
# Copy full KV unchanged first
|
||||
# Copy only the NoPE portion; the RoPE region is written exclusively
|
||||
# by the rotated stores below. Masking avoids a write-after-write
|
||||
# hazard on the overlapping RoPE addresses (matches the Q branch).
|
||||
offs = tl.arange(0, HEAD_DIM)
|
||||
kv_full = tl.load(kv_base + offs)
|
||||
tl.store(kv_out_base + offs, kv_full)
|
||||
nope_mask = offs < NOPE_DIM
|
||||
tl.store(kv_out_base + offs, kv_full, mask=nope_mask)
|
||||
|
||||
# GPT-J interleaved RoPE on the last ROPE_DIM dimensions
|
||||
even_offs = NOPE_DIM + rope_pair_idx * 2
|
||||
|
||||
Reference in New Issue
Block a user