[ROCm][DSv4.1] Enable DSpark adaptive verification (#58151)

Signed-off-by: Tuukka Sarvi <tuukka.sarvi@amd.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
Tuukka Sarvi
2026-10-15 12:05:43 +08:00
committed by GitHub
co-authored by Kevin H. Luu
parent f46021a7fa
commit 10d2c6083d
5 changed files with 360 additions and 26 deletions
@@ -1983,6 +1983,272 @@ def test_dsv4_adaptive_mla_swa_metadata_graph_replay(monkeypatch) -> None:
torch.testing.assert_close(out, expected_reallocated, atol=2e-2, rtol=2e-2)
@requires_gfx950
@torch.inference_mode()
def test_dsv41_adaptive_mla_swa_metadata_graph_replay(monkeypatch) -> None:
"""Replay V4.1 ROCm decode after device-only query reallocation."""
from tests.v1.attention.utils import create_vllm_config
from vllm.models.deepseek_v41.amd.rocm import (
DeepseekV4ROCMAiterSparseSWAMetadataBuilder,
DeepseekV41ROCMAiterMLASparseMetadataBuilder,
compute_global_topk_ragged_indices_and_indptr,
)
from vllm.v1.attention.backend import AttentionCGSupport, CommonAttentionMetadata
from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod
from vllm.v1.kv_cache_interface import MLAAttentionSpec, SlidingWindowMLASpec
device = torch.device("cuda")
num_reqs = 3
upper_query_len = 6
graph_tokens = num_reqs * upper_query_len
block_size = 128
compress_ratio = 2
compressed_block_size = block_size // compress_ratio
vllm_config = create_vllm_config(
model_name="facebook/opt-125m",
max_model_len=1024,
block_size=block_size,
max_num_seqs=num_reqs,
max_num_batched_tokens=graph_tokens,
hf_config_override={
"compress_ratios": [0, 1, 2],
"index_topk": 8,
"sliding_window": 32,
},
)
vllm_config.speculative_config = SimpleNamespace(
num_speculative_tokens=upper_query_len - 1,
parallel_drafting=False,
enable_adaptive_verification=True,
use_dspark=lambda: True,
)
mla_spec = MLAAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=HEAD_DIM,
dtype=torch.bfloat16,
tokens_per_state=compress_ratio,
cache_dtype_str="fp8_ds_mla",
model_version="deepseek_v4",
)
swa_spec = SlidingWindowMLASpec(
block_size=block_size,
num_kv_heads=1,
head_size=HEAD_DIM,
dtype=torch.uint8,
sliding_window=32,
cache_dtype_str="fp8_ds_mla",
model_version="deepseek_v4",
)
mla_builder = DeepseekV41ROCMAiterMLASparseMetadataBuilder(
mla_spec, ["c2a"], vllm_config, device
)
swa_builder = DeepseekV4ROCMAiterSparseSWAMetadataBuilder(
swa_spec, ["c2a"], vllm_config, device
)
assert (
mla_builder.get_cudagraph_support(vllm_config, mla_spec)
== AttentionCGSupport.ALWAYS
)
assert (
swa_builder.get_cudagraph_support(vllm_config, swa_spec)
== AttentionCGSupport.ALWAYS
)
seq_lens = torch.tensor([100, 110, 120], dtype=torch.int32, device=device)
seq_lens_cpu = seq_lens.cpu()
query_start_loc_cpu = torch.arange(
0, graph_tokens + 1, upper_query_len, dtype=torch.int32
)
block_table = torch.arange(num_reqs, dtype=torch.int32, device=device).view(
num_reqs, 1
)
def build_metadata(query_lens: list[int]):
query_lens_tensor = torch.tensor(query_lens, dtype=torch.int32, device=device)
query_start_loc = torch.cat(
[
torch.zeros(1, dtype=torch.int32, device=device),
query_lens_tensor.cumsum(0),
]
)
active_tokens = sum(query_lens)
slot_mapping = torch.full((graph_tokens,), -1, dtype=torch.int64, device=device)
slot_mapping[:active_tokens] = torch.arange(
active_tokens, dtype=torch.int64, device=device
)
common = CommonAttentionMetadata(
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc_cpu,
seq_lens=seq_lens,
seq_lens_cpu_upper_bound=seq_lens_cpu,
num_reqs=num_reqs,
num_actual_tokens=graph_tokens,
max_query_len=upper_query_len,
max_seq_len=int(seq_lens_cpu.max()),
block_table_tensor=block_table,
slot_mapping=slot_mapping,
causal=True,
)
return (
mla_builder.build_for_cudagraph_capture(common),
swa_builder.build_for_cudagraph_capture(common),
)
mla_metadata, swa_metadata = build_metadata([6, 6, 6])
metadata_ptrs = (
mla_metadata.req_id_per_token.data_ptr(),
mla_metadata.slot_mapping.data_ptr(),
swa_metadata.token_to_req_indices.data_ptr(),
swa_metadata.decode_swa_ragged_indices.data_ptr(),
swa_metadata.decode_swa_ragged_indptr.data_ptr(),
)
topk_indices = (
torch.arange(8, dtype=torch.int32, device=device)
.expand(graph_tokens, -1)
.contiguous()
)
torch.manual_seed(23)
num_heads = 16
q = (
torch.randn(
graph_tokens,
num_heads,
HEAD_DIM,
dtype=torch.bfloat16,
device=device,
)
* 0.125
)
swa_cache = _pack_fp8_ds_mla_cache(
torch.randn(
num_reqs * block_size,
HEAD_DIM,
dtype=torch.bfloat16,
device=device,
)
* 0.125,
block_size,
use_fnuz=False,
)
compressed_cache = _pack_fp8_ds_mla_cache(
torch.randn(
num_reqs * compressed_block_size,
HEAD_DIM,
dtype=torch.bfloat16,
device=device,
)
* 0.125,
compressed_block_size,
use_fnuz=False,
)
attn_sink = torch.linspace(-0.1, 0.1, num_heads, dtype=torch.float32, device=device)
out = torch.empty_like(q)
monkeypatch.setattr(mod, "_decode_gfx950_num_splits", lambda *args: 1)
def build_topk_ragged(mla_md, swa_md):
return compute_global_topk_ragged_indices_and_indptr(
topk_indices,
swa_md.token_to_req_indices,
mla_md.block_table,
compressed_block_size,
swa_md.is_valid_token,
)
def run_decode(mla_md, swa_md) -> None:
topk_ragged_indices, topk_ragged_indptr, topk_lens = build_topk_ragged(
mla_md, swa_md
)
mod.rocm_sparse_attn_decode(
q=q,
kv_cache=compressed_cache,
swa_k_cache=swa_cache,
swa_only=False,
topk_indices=None,
topk_lens=topk_lens,
swa_indices=swa_md.decode_swa_indices,
swa_lens=swa_md.decode_swa_lens,
swa_ragged_indices=swa_md.decode_swa_ragged_indices,
swa_ragged_indptr=swa_md.decode_swa_ragged_indptr,
topk_ragged_indices=topk_ragged_indices,
topk_ragged_indptr=topk_ragged_indptr,
attn_sink=attn_sink,
scale=HEAD_DIM**-0.5,
head_dim=HEAD_DIM,
nope_head_dim=NOPE_HEAD_DIM,
rope_head_dim=ROPE_HEAD_DIM,
output=out,
extra_cache_nan_free=True,
)
def reference(mla_md, swa_md) -> torch.Tensor:
topk_ragged_indices, topk_ragged_indptr, _ = build_topk_ragged(mla_md, swa_md)
return _ref_sparse_decode_ragged(
q=q,
main_cache=swa_cache,
main_rows=_rows_from_ragged(
swa_md.decode_swa_ragged_indices,
swa_md.decode_swa_ragged_indptr,
),
scale=HEAD_DIM**-0.5,
attn_sink=attn_sink,
block_size=block_size,
extra_cache=compressed_cache,
extra_rows=_rows_from_ragged(
topk_ragged_indices,
topk_ragged_indptr,
),
extra_block_size=compressed_block_size,
)
run_decode(mla_metadata, swa_metadata)
torch.accelerator.synchronize()
expected_full = reference(mla_metadata, swa_metadata)
torch.testing.assert_close(out, expected_full, atol=2e-2, rtol=2e-2)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
run_decode(mla_metadata, swa_metadata)
torch.accelerator.synchronize()
captured_full = out.clone()
reallocated_mla, reallocated_swa = build_metadata([2, 6, 4])
assert metadata_ptrs == (
reallocated_mla.req_id_per_token.data_ptr(),
reallocated_mla.slot_mapping.data_ptr(),
reallocated_swa.token_to_req_indices.data_ptr(),
reallocated_swa.decode_swa_ragged_indices.data_ptr(),
reallocated_swa.decode_swa_ragged_indptr.data_ptr(),
)
expected_owners = torch.tensor(
[0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2],
dtype=torch.int32,
device=device,
)
torch.testing.assert_close(reallocated_mla.req_id_per_token[:12], expected_owners)
torch.testing.assert_close(
reallocated_swa.token_to_req_indices[:12], expected_owners
)
assert reallocated_swa.is_valid_token[:12].all()
assert not reallocated_swa.is_valid_token[12:].any()
torch.testing.assert_close(
reallocated_swa.decode_swa_lens[12:],
torch.zeros(6, dtype=torch.int32, device=device),
)
expected_reallocated = reference(reallocated_mla, reallocated_swa)
run_decode(reallocated_mla, reallocated_swa)
torch.accelerator.synchronize()
torch.testing.assert_close(out, expected_reallocated, atol=2e-2, rtol=2e-2)
assert not torch.equal(captured_full, expected_reallocated)
graph.replay()
torch.accelerator.synchronize()
torch.testing.assert_close(out, expected_reallocated, atol=2e-2, rtol=2e-2)
# ---------------------------------------------------------------------------
# o-projection: fused inverse-RoPE + cached bf16 wo_a (rocm_inv_rope_einsum)
# ---------------------------------------------------------------------------