[Bugfix] Fix SM100 fp8_ds_mla cache scales (#49435)

Signed-off-by: Ariel Bereslavsky <arielb@getimpala.ai>
Signed-off-by: Ariel Bereslavsky <ariel@getimpala.ai>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Ariel Bereslavsky <arielb@getimpala.ai>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Misha Goin <mgoin64@gmail.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Ariel Bereslavsky
2026-09-19 17:15:07 -07:00
committed by GitHub
co-authored by Ariel Bereslavsky Codex Misha Goin Yongye Zhu mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 674b6d95d6
commit a7fda4c88b
8 changed files with 76 additions and 38 deletions
+5 -5
View File
@@ -960,14 +960,14 @@ def test_concat_and_cache_ds_mla(
tile_end = (tile_idx + 1) * 128
tile_data[:] = kv_c_data[tile_start:tile_end]
# tile_scale = tile_data.amax().to(torch.float32) / 448.
# NOTE: Using torch's amax() gives different results,
# so this must be manually computed.
# Using torch's amax() gives different results, so this must be
# manually computed.
tile_data_float = tile_data.to(torch.float32)
manual_max = abs(tile_data_float[0])
for j in range(1, 128):
manual_max = max(manual_max, abs(tile_data_float[j]))
tile_scale = manual_max / 448.0
raw_scale = torch.clamp(manual_max / 448.0, min=1e-4)
tile_scale = torch.exp2(torch.ceil(torch.log2(raw_scale)))
ref_cache_32bit[kv_lora_rank // 4 + tile_idx] = tile_scale
@@ -1008,7 +1008,7 @@ def test_concat_and_cache_ds_mla(
ref_rope = ref_cache_slice.view(dtype)[kv_lora_rank // 2 + 8 :]
torch.testing.assert_close(kv_nope, ref_nope, atol=0.001, rtol=0.1)
torch.testing.assert_close(kv_scales, ref_scales, atol=0.001, rtol=0.1)
torch.testing.assert_close(kv_scales, ref_scales, atol=0, rtol=0)
torch.testing.assert_close(kv_rope, ref_rope, atol=0.001, rtol=0.1)