[ROCm][DSv4] Enable DSpark adaptive verification (#52362)
Signed-off-by: Tuukka Sarvi <tuukka.sarvi@amd.com> Signed-off-by: larryli2-amd <larryli2@amd.com> Co-authored-by: larryli2-amd <larryli2@amd.com>
This commit is contained in:
co-authored by
larryli2-amd
parent
21aa17c128
commit
e34685dfc0
@@ -188,6 +188,7 @@ def _ref_sparse_decode_ragged(
|
||||
extra_rows: list[list[int]] | None = None,
|
||||
main_use_fnuz: bool = False,
|
||||
extra_use_fnuz: bool = False,
|
||||
extra_block_size: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
q_f32 = q.float()
|
||||
out = torch.empty_like(q_f32)
|
||||
@@ -209,7 +210,10 @@ def _ref_sparse_decode_ragged(
|
||||
)
|
||||
row_kv.append(
|
||||
_read_fp8_ds_mla_cache_rows(
|
||||
extra_cache, extra_slots, block_size, extra_use_fnuz
|
||||
extra_cache,
|
||||
extra_slots,
|
||||
extra_block_size or block_size,
|
||||
extra_use_fnuz,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -244,6 +248,12 @@ def _ragged_from_rows(
|
||||
)
|
||||
|
||||
|
||||
def _rows_from_ragged(indices: torch.Tensor, indptr: torch.Tensor) -> list[list[int]]:
|
||||
ends = indptr.cpu().tolist()
|
||||
values = indices[: ends[-1]].cpu().tolist()
|
||||
return [values[start:end] for start, end in zip(ends, ends[1:])]
|
||||
|
||||
|
||||
def _launch_sparse_decode_reduce(
|
||||
part_m: torch.Tensor,
|
||||
part_l: torch.Tensor,
|
||||
@@ -1192,6 +1202,254 @@ def test_sparse_attn_decode_gfx950_graph_replay(monkeypatch) -> None:
|
||||
torch.testing.assert_close(out, expected_long, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@requires_gfx950
|
||||
@torch.inference_mode()
|
||||
def test_dsv4_adaptive_mla_swa_metadata_graph_replay(monkeypatch) -> None:
|
||||
"""Replay the production ROCm decode path after device-only reallocation."""
|
||||
from tests.v1.attention.utils import create_vllm_config
|
||||
from vllm.models.deepseek_v4.amd.rocm import (
|
||||
DeepseekV4ROCMAiterMLASparseMetadataBuilder,
|
||||
DeepseekV4ROCMAiterSparseSWAMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.backend import AttentionCGSupport, CommonAttentionMetadata
|
||||
from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod
|
||||
from vllm.v1.kv_cache_interface import MLAAttentionSpec, SlidingWindowMLASpec
|
||||
|
||||
device = torch.device("cuda")
|
||||
num_reqs = 3
|
||||
upper_query_len = 8
|
||||
graph_tokens = num_reqs * upper_query_len
|
||||
block_size = 256
|
||||
compressed_block_size = block_size // 128
|
||||
window_size = 32
|
||||
|
||||
vllm_config = create_vllm_config(
|
||||
model_name="facebook/opt-125m",
|
||||
max_model_len=1024,
|
||||
block_size=block_size,
|
||||
max_num_seqs=num_reqs,
|
||||
max_num_batched_tokens=graph_tokens,
|
||||
hf_config_override={
|
||||
"compress_ratios": [128],
|
||||
"index_topk": 2048,
|
||||
"sliding_window": window_size,
|
||||
},
|
||||
)
|
||||
vllm_config.speculative_config = SimpleNamespace(
|
||||
num_speculative_tokens=upper_query_len - 1,
|
||||
parallel_drafting=False,
|
||||
enable_adaptive_verification=True,
|
||||
use_dspark=lambda: True,
|
||||
)
|
||||
mla_spec = MLAAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=HEAD_DIM,
|
||||
dtype=torch.bfloat16,
|
||||
tokens_per_state=128,
|
||||
cache_dtype_str="fp8_ds_mla",
|
||||
model_version="deepseek_v4",
|
||||
)
|
||||
swa_spec = SlidingWindowMLASpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=HEAD_DIM,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=window_size,
|
||||
cache_dtype_str="fp8_ds_mla",
|
||||
model_version="deepseek_v4",
|
||||
)
|
||||
mla_builder = DeepseekV4ROCMAiterMLASparseMetadataBuilder(
|
||||
mla_spec, ["c128a"], vllm_config, device
|
||||
)
|
||||
swa_builder = DeepseekV4ROCMAiterSparseSWAMetadataBuilder(
|
||||
swa_spec, ["c128a"], vllm_config, device
|
||||
)
|
||||
assert (
|
||||
mla_builder.get_cudagraph_support(vllm_config, mla_spec)
|
||||
== AttentionCGSupport.ALWAYS
|
||||
)
|
||||
assert (
|
||||
swa_builder.get_cudagraph_support(vllm_config, swa_spec)
|
||||
== AttentionCGSupport.ALWAYS
|
||||
)
|
||||
|
||||
seq_lens = torch.tensor([520, 528, 536], dtype=torch.int32, device=device)
|
||||
seq_lens_cpu = seq_lens.cpu()
|
||||
cpu_query_start_loc = torch.arange(
|
||||
0,
|
||||
graph_tokens + 1,
|
||||
upper_query_len,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
max_blocks = (int(seq_lens_cpu.max()) + block_size - 1) // block_size
|
||||
block_table = torch.arange(
|
||||
num_reqs * max_blocks, dtype=torch.int32, device=device
|
||||
).view(num_reqs, max_blocks)
|
||||
|
||||
def build_metadata(query_lens: list[int]):
|
||||
query_lens_tensor = torch.tensor(query_lens, dtype=torch.int32, device=device)
|
||||
query_start_loc = torch.cat(
|
||||
[
|
||||
torch.zeros(1, dtype=torch.int32, device=device),
|
||||
query_lens_tensor.cumsum(0),
|
||||
]
|
||||
)
|
||||
active_tokens = sum(query_lens)
|
||||
positions = torch.zeros(graph_tokens, dtype=torch.int64, device=device)
|
||||
position_rows = [
|
||||
torch.arange(
|
||||
seq_len - query_len,
|
||||
seq_len,
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
)
|
||||
for seq_len, query_len in zip(seq_lens_cpu.tolist(), query_lens)
|
||||
]
|
||||
positions[:active_tokens] = torch.cat(position_rows)
|
||||
slot_mapping = torch.full((graph_tokens,), -1, dtype=torch.int64, device=device)
|
||||
slot_mapping[:active_tokens] = torch.arange(
|
||||
active_tokens, dtype=torch.int64, device=device
|
||||
)
|
||||
common = CommonAttentionMetadata(
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=cpu_query_start_loc,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu,
|
||||
num_reqs=num_reqs,
|
||||
num_actual_tokens=graph_tokens,
|
||||
max_query_len=upper_query_len,
|
||||
max_seq_len=int(seq_lens_cpu.max()),
|
||||
block_table_tensor=block_table,
|
||||
slot_mapping=slot_mapping,
|
||||
positions=positions,
|
||||
causal=True,
|
||||
)
|
||||
return (
|
||||
mla_builder.build_for_cudagraph_capture(common),
|
||||
swa_builder.build_for_cudagraph_capture(common),
|
||||
)
|
||||
|
||||
mla_metadata, swa_metadata = build_metadata([8, 8, 8])
|
||||
assert mla_metadata.for_cudagraph_capture
|
||||
metadata_ptrs = (
|
||||
mla_metadata.c128a_decode_topk_ragged_indices.data_ptr(),
|
||||
mla_metadata.c128a_decode_topk_ragged_indptr.data_ptr(),
|
||||
swa_metadata.decode_swa_ragged_indices.data_ptr(),
|
||||
swa_metadata.decode_swa_ragged_indptr.data_ptr(),
|
||||
)
|
||||
|
||||
torch.manual_seed(19)
|
||||
num_heads = 16
|
||||
q = (
|
||||
torch.randn(
|
||||
graph_tokens,
|
||||
num_heads,
|
||||
HEAD_DIM,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
* 0.125
|
||||
)
|
||||
num_cache_blocks = num_reqs * max_blocks
|
||||
swa_cache = _pack_fp8_ds_mla_cache(
|
||||
torch.randn(
|
||||
num_cache_blocks * block_size,
|
||||
HEAD_DIM,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
* 0.125,
|
||||
block_size,
|
||||
use_fnuz=False,
|
||||
)
|
||||
compressed_cache = _pack_fp8_ds_mla_cache(
|
||||
torch.randn(
|
||||
num_cache_blocks * compressed_block_size,
|
||||
HEAD_DIM,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
* 0.125,
|
||||
compressed_block_size,
|
||||
use_fnuz=False,
|
||||
)
|
||||
attn_sink = torch.linspace(-0.1, 0.1, num_heads, dtype=torch.float32, device=device)
|
||||
out = torch.empty_like(q)
|
||||
monkeypatch.setattr(mod, "_decode_gfx950_num_splits", lambda *args: 1)
|
||||
|
||||
def run_decode(mla_md, swa_md) -> None:
|
||||
mod.rocm_sparse_attn_decode(
|
||||
q=q,
|
||||
kv_cache=compressed_cache,
|
||||
swa_k_cache=swa_cache,
|
||||
swa_only=False,
|
||||
topk_indices=mla_md.c128a_global_decode_topk_indices,
|
||||
topk_lens=mla_md.c128a_decode_topk_lens,
|
||||
swa_indices=swa_md.decode_swa_indices,
|
||||
swa_lens=swa_md.decode_swa_lens,
|
||||
swa_ragged_indices=swa_md.decode_swa_ragged_indices,
|
||||
swa_ragged_indptr=swa_md.decode_swa_ragged_indptr,
|
||||
topk_ragged_indices=mla_md.c128a_decode_topk_ragged_indices,
|
||||
topk_ragged_indptr=mla_md.c128a_decode_topk_ragged_indptr,
|
||||
attn_sink=attn_sink,
|
||||
scale=HEAD_DIM**-0.5,
|
||||
head_dim=HEAD_DIM,
|
||||
nope_head_dim=NOPE_HEAD_DIM,
|
||||
rope_head_dim=ROPE_HEAD_DIM,
|
||||
output=out,
|
||||
extra_cache_nan_free=True,
|
||||
adaptive_splits=True,
|
||||
)
|
||||
|
||||
def reference(mla_md, swa_md) -> torch.Tensor:
|
||||
return _ref_sparse_decode_ragged(
|
||||
q=q,
|
||||
main_cache=swa_cache,
|
||||
main_rows=_rows_from_ragged(
|
||||
swa_md.decode_swa_ragged_indices,
|
||||
swa_md.decode_swa_ragged_indptr,
|
||||
),
|
||||
scale=HEAD_DIM**-0.5,
|
||||
attn_sink=attn_sink,
|
||||
block_size=block_size,
|
||||
extra_cache=compressed_cache,
|
||||
extra_rows=_rows_from_ragged(
|
||||
mla_md.c128a_decode_topk_ragged_indices,
|
||||
mla_md.c128a_decode_topk_ragged_indptr,
|
||||
),
|
||||
extra_block_size=compressed_block_size,
|
||||
)
|
||||
|
||||
run_decode(mla_metadata, swa_metadata)
|
||||
torch.accelerator.synchronize()
|
||||
expected_full = reference(mla_metadata, swa_metadata)
|
||||
torch.testing.assert_close(out, expected_full, atol=2e-2, rtol=2e-2)
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
run_decode(mla_metadata, swa_metadata)
|
||||
torch.accelerator.synchronize()
|
||||
captured_full = out.clone()
|
||||
|
||||
reallocated_mla, reallocated_swa = build_metadata([3, 8, 5])
|
||||
assert metadata_ptrs == (
|
||||
reallocated_mla.c128a_decode_topk_ragged_indices.data_ptr(),
|
||||
reallocated_mla.c128a_decode_topk_ragged_indptr.data_ptr(),
|
||||
reallocated_swa.decode_swa_ragged_indices.data_ptr(),
|
||||
reallocated_swa.decode_swa_ragged_indptr.data_ptr(),
|
||||
)
|
||||
expected_reallocated = reference(reallocated_mla, reallocated_swa)
|
||||
run_decode(reallocated_mla, reallocated_swa)
|
||||
torch.accelerator.synchronize()
|
||||
torch.testing.assert_close(out, expected_reallocated, atol=2e-2, rtol=2e-2)
|
||||
assert not torch.equal(captured_full, expected_reallocated)
|
||||
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
torch.testing.assert_close(out, expected_reallocated, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# o-projection: fused inverse-RoPE + cached bf16 wo_a (rocm_inv_rope_einsum)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user