[ROCm][DSv4.1] Enable DSpark adaptive verification (#58151)
Signed-off-by: Tuukka Sarvi <tuukka.sarvi@amd.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Kevin H. Luu
parent
f46021a7fa
commit
10d2c6083d
@@ -1983,6 +1983,272 @@ def test_dsv4_adaptive_mla_swa_metadata_graph_replay(monkeypatch) -> None:
|
||||
torch.testing.assert_close(out, expected_reallocated, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@requires_gfx950
|
||||
@torch.inference_mode()
|
||||
def test_dsv41_adaptive_mla_swa_metadata_graph_replay(monkeypatch) -> None:
|
||||
"""Replay V4.1 ROCm decode after device-only query reallocation."""
|
||||
from tests.v1.attention.utils import create_vllm_config
|
||||
from vllm.models.deepseek_v41.amd.rocm import (
|
||||
DeepseekV4ROCMAiterSparseSWAMetadataBuilder,
|
||||
DeepseekV41ROCMAiterMLASparseMetadataBuilder,
|
||||
compute_global_topk_ragged_indices_and_indptr,
|
||||
)
|
||||
from vllm.v1.attention.backend import AttentionCGSupport, CommonAttentionMetadata
|
||||
from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod
|
||||
from vllm.v1.kv_cache_interface import MLAAttentionSpec, SlidingWindowMLASpec
|
||||
|
||||
device = torch.device("cuda")
|
||||
num_reqs = 3
|
||||
upper_query_len = 6
|
||||
graph_tokens = num_reqs * upper_query_len
|
||||
block_size = 128
|
||||
compress_ratio = 2
|
||||
compressed_block_size = block_size // compress_ratio
|
||||
|
||||
vllm_config = create_vllm_config(
|
||||
model_name="facebook/opt-125m",
|
||||
max_model_len=1024,
|
||||
block_size=block_size,
|
||||
max_num_seqs=num_reqs,
|
||||
max_num_batched_tokens=graph_tokens,
|
||||
hf_config_override={
|
||||
"compress_ratios": [0, 1, 2],
|
||||
"index_topk": 8,
|
||||
"sliding_window": 32,
|
||||
},
|
||||
)
|
||||
vllm_config.speculative_config = SimpleNamespace(
|
||||
num_speculative_tokens=upper_query_len - 1,
|
||||
parallel_drafting=False,
|
||||
enable_adaptive_verification=True,
|
||||
use_dspark=lambda: True,
|
||||
)
|
||||
mla_spec = MLAAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=HEAD_DIM,
|
||||
dtype=torch.bfloat16,
|
||||
tokens_per_state=compress_ratio,
|
||||
cache_dtype_str="fp8_ds_mla",
|
||||
model_version="deepseek_v4",
|
||||
)
|
||||
swa_spec = SlidingWindowMLASpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=HEAD_DIM,
|
||||
dtype=torch.uint8,
|
||||
sliding_window=32,
|
||||
cache_dtype_str="fp8_ds_mla",
|
||||
model_version="deepseek_v4",
|
||||
)
|
||||
mla_builder = DeepseekV41ROCMAiterMLASparseMetadataBuilder(
|
||||
mla_spec, ["c2a"], vllm_config, device
|
||||
)
|
||||
swa_builder = DeepseekV4ROCMAiterSparseSWAMetadataBuilder(
|
||||
swa_spec, ["c2a"], vllm_config, device
|
||||
)
|
||||
assert (
|
||||
mla_builder.get_cudagraph_support(vllm_config, mla_spec)
|
||||
== AttentionCGSupport.ALWAYS
|
||||
)
|
||||
assert (
|
||||
swa_builder.get_cudagraph_support(vllm_config, swa_spec)
|
||||
== AttentionCGSupport.ALWAYS
|
||||
)
|
||||
|
||||
seq_lens = torch.tensor([100, 110, 120], dtype=torch.int32, device=device)
|
||||
seq_lens_cpu = seq_lens.cpu()
|
||||
query_start_loc_cpu = torch.arange(
|
||||
0, graph_tokens + 1, upper_query_len, dtype=torch.int32
|
||||
)
|
||||
block_table = torch.arange(num_reqs, dtype=torch.int32, device=device).view(
|
||||
num_reqs, 1
|
||||
)
|
||||
|
||||
def build_metadata(query_lens: list[int]):
|
||||
query_lens_tensor = torch.tensor(query_lens, dtype=torch.int32, device=device)
|
||||
query_start_loc = torch.cat(
|
||||
[
|
||||
torch.zeros(1, dtype=torch.int32, device=device),
|
||||
query_lens_tensor.cumsum(0),
|
||||
]
|
||||
)
|
||||
active_tokens = sum(query_lens)
|
||||
slot_mapping = torch.full((graph_tokens,), -1, dtype=torch.int64, device=device)
|
||||
slot_mapping[:active_tokens] = torch.arange(
|
||||
active_tokens, dtype=torch.int64, device=device
|
||||
)
|
||||
common = CommonAttentionMetadata(
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc_cpu,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu,
|
||||
num_reqs=num_reqs,
|
||||
num_actual_tokens=graph_tokens,
|
||||
max_query_len=upper_query_len,
|
||||
max_seq_len=int(seq_lens_cpu.max()),
|
||||
block_table_tensor=block_table,
|
||||
slot_mapping=slot_mapping,
|
||||
causal=True,
|
||||
)
|
||||
return (
|
||||
mla_builder.build_for_cudagraph_capture(common),
|
||||
swa_builder.build_for_cudagraph_capture(common),
|
||||
)
|
||||
|
||||
mla_metadata, swa_metadata = build_metadata([6, 6, 6])
|
||||
metadata_ptrs = (
|
||||
mla_metadata.req_id_per_token.data_ptr(),
|
||||
mla_metadata.slot_mapping.data_ptr(),
|
||||
swa_metadata.token_to_req_indices.data_ptr(),
|
||||
swa_metadata.decode_swa_ragged_indices.data_ptr(),
|
||||
swa_metadata.decode_swa_ragged_indptr.data_ptr(),
|
||||
)
|
||||
|
||||
topk_indices = (
|
||||
torch.arange(8, dtype=torch.int32, device=device)
|
||||
.expand(graph_tokens, -1)
|
||||
.contiguous()
|
||||
)
|
||||
torch.manual_seed(23)
|
||||
num_heads = 16
|
||||
q = (
|
||||
torch.randn(
|
||||
graph_tokens,
|
||||
num_heads,
|
||||
HEAD_DIM,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
* 0.125
|
||||
)
|
||||
swa_cache = _pack_fp8_ds_mla_cache(
|
||||
torch.randn(
|
||||
num_reqs * block_size,
|
||||
HEAD_DIM,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
* 0.125,
|
||||
block_size,
|
||||
use_fnuz=False,
|
||||
)
|
||||
compressed_cache = _pack_fp8_ds_mla_cache(
|
||||
torch.randn(
|
||||
num_reqs * compressed_block_size,
|
||||
HEAD_DIM,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
* 0.125,
|
||||
compressed_block_size,
|
||||
use_fnuz=False,
|
||||
)
|
||||
attn_sink = torch.linspace(-0.1, 0.1, num_heads, dtype=torch.float32, device=device)
|
||||
out = torch.empty_like(q)
|
||||
monkeypatch.setattr(mod, "_decode_gfx950_num_splits", lambda *args: 1)
|
||||
|
||||
def build_topk_ragged(mla_md, swa_md):
|
||||
return compute_global_topk_ragged_indices_and_indptr(
|
||||
topk_indices,
|
||||
swa_md.token_to_req_indices,
|
||||
mla_md.block_table,
|
||||
compressed_block_size,
|
||||
swa_md.is_valid_token,
|
||||
)
|
||||
|
||||
def run_decode(mla_md, swa_md) -> None:
|
||||
topk_ragged_indices, topk_ragged_indptr, topk_lens = build_topk_ragged(
|
||||
mla_md, swa_md
|
||||
)
|
||||
mod.rocm_sparse_attn_decode(
|
||||
q=q,
|
||||
kv_cache=compressed_cache,
|
||||
swa_k_cache=swa_cache,
|
||||
swa_only=False,
|
||||
topk_indices=None,
|
||||
topk_lens=topk_lens,
|
||||
swa_indices=swa_md.decode_swa_indices,
|
||||
swa_lens=swa_md.decode_swa_lens,
|
||||
swa_ragged_indices=swa_md.decode_swa_ragged_indices,
|
||||
swa_ragged_indptr=swa_md.decode_swa_ragged_indptr,
|
||||
topk_ragged_indices=topk_ragged_indices,
|
||||
topk_ragged_indptr=topk_ragged_indptr,
|
||||
attn_sink=attn_sink,
|
||||
scale=HEAD_DIM**-0.5,
|
||||
head_dim=HEAD_DIM,
|
||||
nope_head_dim=NOPE_HEAD_DIM,
|
||||
rope_head_dim=ROPE_HEAD_DIM,
|
||||
output=out,
|
||||
extra_cache_nan_free=True,
|
||||
)
|
||||
|
||||
def reference(mla_md, swa_md) -> torch.Tensor:
|
||||
topk_ragged_indices, topk_ragged_indptr, _ = build_topk_ragged(mla_md, swa_md)
|
||||
return _ref_sparse_decode_ragged(
|
||||
q=q,
|
||||
main_cache=swa_cache,
|
||||
main_rows=_rows_from_ragged(
|
||||
swa_md.decode_swa_ragged_indices,
|
||||
swa_md.decode_swa_ragged_indptr,
|
||||
),
|
||||
scale=HEAD_DIM**-0.5,
|
||||
attn_sink=attn_sink,
|
||||
block_size=block_size,
|
||||
extra_cache=compressed_cache,
|
||||
extra_rows=_rows_from_ragged(
|
||||
topk_ragged_indices,
|
||||
topk_ragged_indptr,
|
||||
),
|
||||
extra_block_size=compressed_block_size,
|
||||
)
|
||||
|
||||
run_decode(mla_metadata, swa_metadata)
|
||||
torch.accelerator.synchronize()
|
||||
expected_full = reference(mla_metadata, swa_metadata)
|
||||
torch.testing.assert_close(out, expected_full, atol=2e-2, rtol=2e-2)
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
run_decode(mla_metadata, swa_metadata)
|
||||
torch.accelerator.synchronize()
|
||||
captured_full = out.clone()
|
||||
|
||||
reallocated_mla, reallocated_swa = build_metadata([2, 6, 4])
|
||||
assert metadata_ptrs == (
|
||||
reallocated_mla.req_id_per_token.data_ptr(),
|
||||
reallocated_mla.slot_mapping.data_ptr(),
|
||||
reallocated_swa.token_to_req_indices.data_ptr(),
|
||||
reallocated_swa.decode_swa_ragged_indices.data_ptr(),
|
||||
reallocated_swa.decode_swa_ragged_indptr.data_ptr(),
|
||||
)
|
||||
expected_owners = torch.tensor(
|
||||
[0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
torch.testing.assert_close(reallocated_mla.req_id_per_token[:12], expected_owners)
|
||||
torch.testing.assert_close(
|
||||
reallocated_swa.token_to_req_indices[:12], expected_owners
|
||||
)
|
||||
assert reallocated_swa.is_valid_token[:12].all()
|
||||
assert not reallocated_swa.is_valid_token[12:].any()
|
||||
torch.testing.assert_close(
|
||||
reallocated_swa.decode_swa_lens[12:],
|
||||
torch.zeros(6, dtype=torch.int32, device=device),
|
||||
)
|
||||
|
||||
expected_reallocated = reference(reallocated_mla, reallocated_swa)
|
||||
run_decode(reallocated_mla, reallocated_swa)
|
||||
torch.accelerator.synchronize()
|
||||
torch.testing.assert_close(out, expected_reallocated, atol=2e-2, rtol=2e-2)
|
||||
assert not torch.equal(captured_full, expected_reallocated)
|
||||
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
torch.testing.assert_close(out, expected_reallocated, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# o-projection: fused inverse-RoPE + cached bf16 wo_a (rocm_inv_rope_einsum)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user