[Bugfix] Fix turboquant FP8 cast failure for BF16 models on Ampere GPUs (#39988)
Signed-off-by: Xu Zhou <xuzhou9417@163.com> Co-authored-by: Xu Zhou <xuzhou9417@163.com> Co-authored-by: Hoseung Kim <ghyutjik123@gmail.com>
This commit is contained in:
co-authored by
Xu Zhou
Hoseung Kim
parent
e257faf87d
commit
b12cca6a23
@@ -188,7 +188,7 @@ def _tq_fused_store_fp8(
|
||||
# ── FP8 KEY: cast to FP8 in-kernel and store ─────────────────
|
||||
d_offs = tl.arange(0, BLOCK_D)
|
||||
d_mask = d_offs < D
|
||||
k_vals = tl.load(Key_ptr + base + d_offs, mask=d_mask, other=0.0)
|
||||
k_vals = tl.load(Key_ptr + base + d_offs, mask=d_mask, other=0.0).to(tl.float32)
|
||||
k_fp8 = k_vals.to(tl.float8e4b15) if FP8_E4B15 else k_vals.to(tl.float8e4nv)
|
||||
k_bytes = k_fp8.to(tl.uint8, bitcast=True)
|
||||
tl.store(KV_cache_ptr + slot_base + d_offs, k_bytes, mask=d_mask)
|
||||
|
||||
Reference in New Issue
Block a user