[Bugfix] Fix SM100 fp8_ds_mla cache scales (#49435)
Signed-off-by: Ariel Bereslavsky <arielb@getimpala.ai> Signed-off-by: Ariel Bereslavsky <ariel@getimpala.ai> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Ariel Bereslavsky <arielb@getimpala.ai> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Misha Goin <mgoin64@gmail.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Ariel Bereslavsky
Codex
Misha Goin
Yongye Zhu
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
674b6d95d6
commit
a7fda4c88b
@@ -315,7 +315,7 @@ def _fused_norm_rope_kernel(
|
||||
if MLA_CACHE_DS_MLA:
|
||||
# fp8_ds_mla layout (DeepSeek-V3.2, KV_DIM == 512): per-128-element
|
||||
# tile of the NoPE is dynamically quantized to fp8 with its own
|
||||
# float32 scale; the RoPE tail is stored unquantized in bf16.
|
||||
# power-of-two scale stored as float32; the RoPE tail is bf16.
|
||||
# bytes [0, KV_DIM) : KV_DIM fp8 NoPE values
|
||||
# bytes [KV_DIM, KV_DIM + 16) : MLA_NUM_TILES float32 scales
|
||||
# bytes [KV_DIM + 16, ...) : 2 * KPE_HALF_ROT_DIM bf16 RoPE
|
||||
@@ -327,9 +327,8 @@ def _fused_norm_rope_kernel(
|
||||
)
|
||||
kv_2d = tl.reshape(kv_c, (MLA_NUM_TILES, MLA_TILE_DIM))
|
||||
tile_amax = tl.max(tl.abs(kv_2d), axis=1, keep_dims=True)
|
||||
# scale = amax / 448 (fp8 e4m3 max), matching the reference
|
||||
# concat_and_cache_ds_mla kernel; floored to FLT_MIN.
|
||||
tile_scale = tl.maximum(tile_amax * (1.0 / 448.0), 1.1754944e-38)
|
||||
tile_scale = tl.maximum(tile_amax * (1.0 / 448.0), 1e-4)
|
||||
tile_scale = tl.math.exp2(tl.math.ceil(tl.math.log2(tile_scale)))
|
||||
kv_c_fp8 = tl.reshape((kv_2d / tile_scale).to(tl.float8e4nv), (KV_DIM,))
|
||||
tl.store(mla_cache_ptr + byte_base + kv_block, kv_c_fp8)
|
||||
tile_off = tl.arange(0, MLA_NUM_TILES)
|
||||
|
||||
@@ -76,9 +76,9 @@ def fused_mla_key_concat_ds_mla_insert(
|
||||
"""Concat full K (bf16) and insert the latent in the fp8_ds_mla layout.
|
||||
|
||||
The cache uses DeepSeek's 656-byte block-scaled layout (NoPE in 4 tiles of
|
||||
128 with per-tile dynamic fp8 scales, RoPE as bf16) -- self-scaling, so no
|
||||
scale argument. Returns the bf16 full key; optionally rotates ``q`` and
|
||||
writes ``kv_cache`` in place.
|
||||
128 with per-tile power-of-two scales stored as float32, RoPE as bf16) --
|
||||
self-scaling, so no scale argument. Returns the bf16 full key; optionally
|
||||
rotates ``q`` and writes ``kv_cache`` in place.
|
||||
"""
|
||||
k_pe = k_pe.reshape(k_pe.shape[0], -1)
|
||||
tp, num_heads, qk_nope_head_dim = k_nope.shape
|
||||
|
||||
Reference in New Issue
Block a user