[ROCm][DSv4] Fix sparse-indexer logits collapse on gfx950/gfx942 (#50455)

Signed-off-by: Frida Andersson <fanderss@amd.com>
Signed-off-by: frida-andersson <fanderss@amd.com>
This commit is contained in:
frida-andersson
2026-09-18 16:57:44 +00:00
committed by GitHub
parent 8a5cf54387
commit 71fc70d3ae
2 changed files with 373 additions and 5 deletions
@@ -1430,3 +1430,189 @@ def test_get_cached_wo_a_bf16_fp8_blockscale_caches() -> None:
# Second call returns the same cached object.
assert _get_cached_wo_a_bf16(wo_a, n_local_groups, o_lora_rank, hidden_dim) is out
@requires_split_decode_arch
@torch.inference_mode()
def test_fp8_paged_mqa_logits_triton_matches_torch_ref() -> None:
"""Block-flat Triton decode logits vs the eager torch reference.
Compare only valid key positions: the kernel skips the -inf pad that
the torch ref writes past seq_len.
"""
from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod
from vllm.v1.worker.workspace import init_workspace_manager
device = torch.device("cuda")
init_workspace_manager(device)
torch.manual_seed(0)
fp8_dtype = current_platform.fp8_dtype()
batch_size = 2
next_n = 1
num_heads = 4
head_size = 128
block_size = 64
max_model_len = 256
seq_lens = [180, 64]
num_pages = [(seq_len + block_size - 1) // block_size for seq_len in seq_lens]
num_blocks = sum(num_pages)
q = torch.randn(
batch_size, next_n, num_heads, head_size, device=device, dtype=torch.bfloat16
).to(fp8_dtype)
weights = torch.rand(
batch_size * next_n, num_heads, device=device, dtype=torch.float32
)
context_lens = torch.tensor(seq_lens, device=device, dtype=torch.int32)
block_tables = torch.full(
(batch_size, max(num_pages)), 0, device=device, dtype=torch.int32
)
page = 0
for i, n_pages in enumerate(num_pages):
block_tables[i, :n_pages] = torch.arange(
page, page + n_pages, device=device, dtype=torch.int32
)
page += n_pages
kv_cache = torch.empty(
num_blocks, block_size, 1, head_size + 4, device=device, dtype=torch.uint8
)
values = torch.randn(
num_blocks, block_size, head_size, device=device, dtype=torch.bfloat16
).to(fp8_dtype)
scales = torch.rand(
num_blocks, block_size, device=device, dtype=torch.float32
).clamp(min=1e-3)
kv_flat = kv_cache.view(num_blocks, -1)
scale_off = block_size * head_size
kv_flat[:, :scale_off] = values.reshape(num_blocks, -1).view(torch.uint8)
kv_flat[:, scale_off:] = scales.contiguous().view(torch.uint8).view(num_blocks, -1)
got = mod.rocm_fp8_paged_mqa_logits_triton(
q, kv_cache, weights, context_lens, block_tables, max_model_len
)
ref = mod.fp8_paged_mqa_logits_torch(
q, kv_cache, weights, context_lens, block_tables, max_model_len
)
for i, seq_len in enumerate(seq_lens):
torch.testing.assert_close(
got[i, :seq_len], ref[i, :seq_len], atol=2e-3, rtol=2e-3
)
def test_indexer_k_is_c4a_block_flat_gate() -> None:
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_indexer_k_is_c4a_block_flat,
)
assert not _indexer_k_is_c4a_block_flat(1)
assert not _indexer_k_is_c4a_block_flat(2)
assert _indexer_k_is_c4a_block_flat(4)
assert not _indexer_k_is_c4a_block_flat(128)
@requires_split_decode_arch
@torch.inference_mode()
def test_paged_mqa_logits_gate_v41_shuffle_vs_c4a_flat() -> None:
"""Ratio 1/2 stay on AITER; ratio 4 takes Triton."""
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
fp8_paged_mqa_logits_torch,
indexer_k_quant_and_cache_triton,
rocm_fp8_paged_mqa_logits,
rocm_fp8_paged_mqa_logits_triton,
)
from vllm.v1.worker.workspace import init_workspace_manager
device = torch.device("cuda")
init_workspace_manager(device)
torch.manual_seed(1)
fp8_dtype = current_platform.fp8_dtype()
batch_size = 1
next_n = 1
num_heads = 32
head_size = 128
block_size = 64
seq_len = 180
max_model_len = 256
num_pages = (seq_len + block_size - 1) // block_size
num_tokens = num_pages * block_size
q = torch.randn(
batch_size, next_n, num_heads, head_size, device=device, dtype=torch.bfloat16
).to(fp8_dtype)
weights = torch.rand(
batch_size * next_n, num_heads, device=device, dtype=torch.float32
)
context_lens = torch.tensor([seq_len], device=device, dtype=torch.int32)
block_tables = torch.arange(num_pages, device=device, dtype=torch.int32).view(1, -1)
dummy_sched = torch.zeros(8, 2, device=device, dtype=torch.int32)
k_bf16 = torch.randn(num_tokens, head_size, device=device, dtype=torch.bfloat16)
slot = torch.arange(num_tokens, device=device, dtype=torch.int32)
cache_shuffle = torch.zeros(
num_pages, block_size, head_size + 4, device=device, dtype=torch.uint8
)
indexer_k_quant_and_cache_triton(k_bf16, cache_shuffle, slot, 128, "ue8m0")
cache_shuffle = cache_shuffle.unsqueeze(2)
aiter = rocm_fp8_paged_mqa_logits(
q,
cache_shuffle,
weights,
context_lens,
block_tables,
dummy_sched,
max_model_len,
compress_ratio=1,
)
for ratio in (1, 2):
got = rocm_fp8_paged_mqa_logits(
q,
cache_shuffle,
weights,
context_lens,
block_tables,
dummy_sched,
max_model_len,
compress_ratio=ratio,
)
torch.testing.assert_close(got[:, :seq_len], aiter[:, :seq_len], atol=0, rtol=0)
cache_flat = torch.empty(
num_pages, block_size, 1, head_size + 4, device=device, dtype=torch.uint8
)
values = torch.randn(
num_pages, block_size, head_size, device=device, dtype=torch.bfloat16
).to(fp8_dtype)
scales = torch.rand(
num_pages, block_size, device=device, dtype=torch.float32
).clamp(min=1e-3)
kv_flat = cache_flat.view(num_pages, -1)
scale_off = block_size * head_size
kv_flat[:, :scale_off] = values.reshape(num_pages, -1).view(torch.uint8)
kv_flat[:, scale_off:] = scales.contiguous().view(torch.uint8).view(num_pages, -1)
gated_c4a = rocm_fp8_paged_mqa_logits(
q,
cache_flat,
weights,
context_lens,
block_tables,
dummy_sched,
max_model_len,
compress_ratio=4,
)
triton = rocm_fp8_paged_mqa_logits_triton(
q, cache_flat, weights, context_lens, block_tables, max_model_len
)
ref = fp8_paged_mqa_logits_torch(
q, cache_flat, weights, context_lens, block_tables, max_model_len
)
torch.testing.assert_close(
gated_c4a[:, :seq_len], triton[:, :seq_len], atol=0, rtol=0
)
torch.testing.assert_close(
gated_c4a[:, :seq_len], ref[:, :seq_len], atol=2e-3, rtol=2e-3
)