[Bugfix] Fix SM100 fp8_ds_mla cache scales (#49435)
Signed-off-by: Ariel Bereslavsky <arielb@getimpala.ai> Signed-off-by: Ariel Bereslavsky <ariel@getimpala.ai> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Ariel Bereslavsky <arielb@getimpala.ai> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Misha Goin <mgoin64@gmail.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Ariel Bereslavsky
Codex
Misha Goin
Yongye Zhu
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
674b6d95d6
commit
a7fda4c88b
@@ -960,14 +960,14 @@ def test_concat_and_cache_ds_mla(
|
||||
tile_end = (tile_idx + 1) * 128
|
||||
tile_data[:] = kv_c_data[tile_start:tile_end]
|
||||
|
||||
# tile_scale = tile_data.amax().to(torch.float32) / 448.
|
||||
# NOTE: Using torch's amax() gives different results,
|
||||
# so this must be manually computed.
|
||||
# Using torch's amax() gives different results, so this must be
|
||||
# manually computed.
|
||||
tile_data_float = tile_data.to(torch.float32)
|
||||
manual_max = abs(tile_data_float[0])
|
||||
for j in range(1, 128):
|
||||
manual_max = max(manual_max, abs(tile_data_float[j]))
|
||||
tile_scale = manual_max / 448.0
|
||||
raw_scale = torch.clamp(manual_max / 448.0, min=1e-4)
|
||||
tile_scale = torch.exp2(torch.ceil(torch.log2(raw_scale)))
|
||||
|
||||
ref_cache_32bit[kv_lora_rank // 4 + tile_idx] = tile_scale
|
||||
|
||||
@@ -1008,7 +1008,7 @@ def test_concat_and_cache_ds_mla(
|
||||
ref_rope = ref_cache_slice.view(dtype)[kv_lora_rank // 2 + 8 :]
|
||||
|
||||
torch.testing.assert_close(kv_nope, ref_nope, atol=0.001, rtol=0.1)
|
||||
torch.testing.assert_close(kv_scales, ref_scales, atol=0.001, rtol=0.1)
|
||||
torch.testing.assert_close(kv_scales, ref_scales, atol=0, rtol=0)
|
||||
torch.testing.assert_close(kv_rope, ref_rope, atol=0.001, rtol=0.1)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user