[Spec Decode] Skip metadata build for dflash/dspark when all model backends support it (#60692)
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai> Co-authored-by: Rishi Puri <20074092+puririshi98@users.noreply.github.com>
This commit is contained in:
co-authored by
Rishi Puri
parent
6a70913590
commit
36f3972812
@@ -828,3 +828,57 @@ def test_v41_flashinfer_mixed_sparse_indices_respect_replay_start():
|
||||
# Decode row is copied through; prefill rows follow the bounded window.
|
||||
assert bounded[0].cpu().tolist() == plain[0].cpu().tolist()
|
||||
assert bounded[1:].cpu().tolist() == rows[1:]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_dspark_noncausal_draft_decode_update_matches_build():
|
||||
"""A DSpark draft graph refreshes its non-causal SWA metadata in place
|
||||
instead of rebuilding it, so the refresh must match a fresh build."""
|
||||
from vllm.v1.attention.backends.mla.compressor_utils import (
|
||||
get_dspark_swa_index_width,
|
||||
)
|
||||
|
||||
num_spec = 4
|
||||
query_lens = [num_spec] * 3
|
||||
|
||||
def make_dspark_builder():
|
||||
builder = make_builder(vision=False)
|
||||
builder.is_dspark = True
|
||||
builder.noncausal_index_width = get_dspark_swa_index_width(WINDOW, num_spec)
|
||||
builder.decode_threshold = num_spec
|
||||
return builder
|
||||
|
||||
def common_metadata(seq_lens):
|
||||
query_start_loc, seq_lens_t, _, slot_mapping, block_table = make_batch(
|
||||
seq_lens, query_lens, torch.device("cuda")
|
||||
)
|
||||
return CommonAttentionMetadata(
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc.cpu(),
|
||||
seq_lens=seq_lens_t,
|
||||
seq_lens_cpu_upper_bound=seq_lens_t.cpu(),
|
||||
num_reqs=len(seq_lens),
|
||||
num_actual_tokens=int(query_start_loc[-1]),
|
||||
max_query_len=num_spec,
|
||||
max_seq_len=max(seq_lens),
|
||||
block_table_tensor=block_table,
|
||||
slot_mapping=slot_mapping,
|
||||
causal=False,
|
||||
)
|
||||
|
||||
captured_seq_lens = [6, 40, 130]
|
||||
replay_seq_lens = [130, 9, 90]
|
||||
builder = make_dspark_builder()
|
||||
captured = common_metadata(captured_seq_lens)
|
||||
md = builder.build(0, captured)
|
||||
replay = common_metadata(replay_seq_lens)
|
||||
expected = make_dspark_builder().build(0, replay)
|
||||
|
||||
# A replay sees new inputs through the same device buffers.
|
||||
captured.seq_lens.copy_(replay.seq_lens)
|
||||
captured.block_table_tensor.copy_(replay.block_table_tensor)
|
||||
builder.update_draft_decode_metadata(md)
|
||||
|
||||
assert md.decode_swa_width == builder.noncausal_index_width
|
||||
torch.testing.assert_close(md.decode_swa_lens, expected.decode_swa_lens)
|
||||
torch.testing.assert_close(md.decode_swa_indices, expected.decode_swa_indices)
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.triton_attn import TritonAttentionMetadataBuilder
|
||||
from vllm.v1.attention.backends.triton_attn_diffkv import (
|
||||
TritonAttentionDiffKVMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec
|
||||
|
||||
|
||||
# 64 segments only while num_seqs * num_kv_heads * 16 < 188 SMs on SM12.0 and
|
||||
# the 64-segment scratch holds every query token (e.g. multi-query verify).
|
||||
@pytest.mark.parametrize(
|
||||
"builder_cls",
|
||||
[TritonAttentionMetadataBuilder, TritonAttentionDiffKVMetadataBuilder],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"capability,num_kv_heads,num_seqs,query_len,segments",
|
||||
[
|
||||
((12, 0), 1, 11, 1, 64),
|
||||
((12, 0), 1, 12, 1, 16),
|
||||
((12, 0), 8, 1, 1, 64),
|
||||
((12, 0), 8, 2, 1, 16),
|
||||
((9, 0), 1, 1, 1, 16),
|
||||
((12, 0), 1, 2, 4, 64),
|
||||
((12, 0), 1, 10, 4, 16),
|
||||
],
|
||||
)
|
||||
def test_split_k_segments_follow_sm_occupancy(
|
||||
monkeypatch, builder_cls, capability, num_kv_heads, num_seqs, query_len, segments
|
||||
):
|
||||
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
current_platform, "is_device_capability", lambda cap: cap == capability
|
||||
)
|
||||
monkeypatch.setattr(current_platform, "num_compute_units", lambda: 188)
|
||||
config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
get_num_attention_heads=lambda _: 8,
|
||||
get_num_kv_heads=lambda _: num_kv_heads,
|
||||
get_head_size=lambda: 128,
|
||||
rswa_window=None,
|
||||
),
|
||||
parallel_config=None,
|
||||
scheduler_config=SimpleNamespace(max_num_seqs=16),
|
||||
speculative_config=None,
|
||||
compilation_config=SimpleNamespace(
|
||||
cudagraph_mode=CUDAGraphMode.NONE, static_forward_context={}
|
||||
),
|
||||
)
|
||||
spec = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=num_kv_heads, head_size=128, dtype=torch.bfloat16
|
||||
)
|
||||
builder = builder_cls(spec, ["layer.0"], config, "cpu")
|
||||
batch = BatchSpec(seq_lens=[128] * num_seqs, query_lens=[query_len] * num_seqs)
|
||||
metadata = builder.build(0, create_common_attn_metadata(batch, 16, "cpu"))
|
||||
assert metadata.num_par_softmax_segments == segments
|
||||
assert metadata.softmax_segm_output.shape[2] == segments
|
||||
assert metadata.softmax_segm_max.shape[0] >= num_seqs * query_len
|
||||
# DiffKV re-allocates the output; it must stay sized like the base buffers.
|
||||
assert builder.softmax_segm_output.shape[0] == builder.softmax_segm_max.shape[0]
|
||||
assert (
|
||||
metadata.softmax_segm_output.data_ptr()
|
||||
== builder.softmax_segm_output.data_ptr()
|
||||
)
|
||||
|
||||
|
||||
def _draft_builder(builder_cls, rswa_window: int | None):
|
||||
config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
get_num_attention_heads=lambda _: 8,
|
||||
get_num_kv_heads=lambda _: 1,
|
||||
get_head_size=lambda: 128,
|
||||
rswa_window=rswa_window,
|
||||
),
|
||||
parallel_config=None,
|
||||
scheduler_config=SimpleNamespace(max_num_seqs=16),
|
||||
speculative_config=None,
|
||||
compilation_config=SimpleNamespace(
|
||||
cudagraph_mode=CUDAGraphMode.NONE, static_forward_context={}
|
||||
),
|
||||
)
|
||||
spec = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=128, dtype=torch.bfloat16
|
||||
)
|
||||
return builder_cls(spec, ["layer.0"], config, "cpu")
|
||||
|
||||
|
||||
def _draft_decode_batch(seq_lens: list[int]):
|
||||
batch = BatchSpec(seq_lens=seq_lens, query_lens=[1] * len(seq_lens))
|
||||
return create_common_attn_metadata(batch, 16, "cpu")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"builder_cls",
|
||||
[TritonAttentionMetadataBuilder, TritonAttentionDiffKVMetadataBuilder],
|
||||
)
|
||||
@pytest.mark.parametrize("rswa_window", [None, 64])
|
||||
def test_draft_decode_update_flag_matches_build_writes(builder_cls, rswa_window):
|
||||
"""The update is a no-op, so the builder may claim support only when
|
||||
build() writes no builder-owned tensor that a captured graph reads."""
|
||||
builder = _draft_builder(builder_cls, rswa_window)
|
||||
common_metadata = _draft_decode_batch([5, 9])
|
||||
common_metadata.rswa_prefix_lens = torch.tensor([3, 7], dtype=torch.int32)
|
||||
|
||||
def owned_bytes():
|
||||
# Bytes, not values: uninitialized scratch may hold NaNs.
|
||||
return {
|
||||
name: value.view(torch.uint8).clone()
|
||||
for name, value in vars(builder).items()
|
||||
if isinstance(value, torch.Tensor)
|
||||
}
|
||||
|
||||
before = owned_bytes()
|
||||
builder.build(0, common_metadata)
|
||||
after = owned_bytes()
|
||||
|
||||
written = [name for name in before if not torch.equal(before[name], after[name])]
|
||||
assert builder.supports_draft_decode_metadata_update is (not written)
|
||||
assert builder.supports_draft_decode_metadata_update is (rswa_window is None)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"builder_cls",
|
||||
[TritonAttentionMetadataBuilder, TritonAttentionDiffKVMetadataBuilder],
|
||||
)
|
||||
def test_draft_decode_update_matches_fresh_build(builder_cls):
|
||||
"""A replayed graph reads metadata built at capture; after an in-place
|
||||
update it must match a fresh build of the replayed batch."""
|
||||
torch.manual_seed(0)
|
||||
builder = _draft_builder(builder_cls, rswa_window=None)
|
||||
captured = _draft_decode_batch([100, 120])
|
||||
metadata = builder.build(0, captured)
|
||||
|
||||
replayed = _draft_decode_batch([128, 30])
|
||||
# The replayed batch arrives through the same persistent input buffers.
|
||||
captured.seq_lens.copy_(replayed.seq_lens)
|
||||
captured.block_table_tensor.copy_(replayed.block_table_tensor)
|
||||
captured.slot_mapping.copy_(replayed.slot_mapping)
|
||||
builder.update_draft_decode_metadata(metadata)
|
||||
|
||||
expected = _draft_builder(builder_cls, rswa_window=None).build(0, replayed)
|
||||
assert torch.equal(metadata.seq_lens, expected.seq_lens)
|
||||
assert torch.equal(metadata.block_table, expected.block_table)
|
||||
assert torch.equal(metadata.slot_mapping, expected.slot_mapping)
|
||||
assert torch.equal(metadata.query_start_loc, expected.query_start_loc)
|
||||
@@ -1,73 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.triton_attn import TritonAttentionMetadataBuilder
|
||||
from vllm.v1.attention.backends.triton_attn_diffkv import (
|
||||
TritonAttentionDiffKVMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec
|
||||
|
||||
|
||||
# 64 segments only while num_seqs * num_kv_heads * 16 < 188 SMs on SM12.0 and
|
||||
# the 64-segment scratch holds every query token (e.g. multi-query verify).
|
||||
@pytest.mark.parametrize(
|
||||
"builder_cls",
|
||||
[TritonAttentionMetadataBuilder, TritonAttentionDiffKVMetadataBuilder],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"capability,num_kv_heads,num_seqs,query_len,segments",
|
||||
[
|
||||
((12, 0), 1, 11, 1, 64),
|
||||
((12, 0), 1, 12, 1, 16),
|
||||
((12, 0), 8, 1, 1, 64),
|
||||
((12, 0), 8, 2, 1, 16),
|
||||
((9, 0), 1, 1, 1, 16),
|
||||
((12, 0), 1, 2, 4, 64),
|
||||
((12, 0), 1, 10, 4, 16),
|
||||
],
|
||||
)
|
||||
def test_split_k_segments_follow_sm_occupancy(
|
||||
monkeypatch, builder_cls, capability, num_kv_heads, num_seqs, query_len, segments
|
||||
):
|
||||
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
current_platform, "is_device_capability", lambda cap: cap == capability
|
||||
)
|
||||
monkeypatch.setattr(current_platform, "num_compute_units", lambda: 188)
|
||||
config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
get_num_attention_heads=lambda _: 8,
|
||||
get_num_kv_heads=lambda _: num_kv_heads,
|
||||
get_head_size=lambda: 128,
|
||||
rswa_window=None,
|
||||
),
|
||||
parallel_config=None,
|
||||
scheduler_config=SimpleNamespace(max_num_seqs=16),
|
||||
speculative_config=None,
|
||||
compilation_config=SimpleNamespace(
|
||||
cudagraph_mode=CUDAGraphMode.NONE, static_forward_context={}
|
||||
),
|
||||
)
|
||||
spec = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=num_kv_heads, head_size=128, dtype=torch.bfloat16
|
||||
)
|
||||
builder = builder_cls(spec, ["layer.0"], config, "cpu")
|
||||
batch = BatchSpec(seq_lens=[128] * num_seqs, query_lens=[query_len] * num_seqs)
|
||||
metadata = builder.build(0, create_common_attn_metadata(batch, 16, "cpu"))
|
||||
assert metadata.num_par_softmax_segments == segments
|
||||
assert metadata.softmax_segm_output.shape[2] == segments
|
||||
assert metadata.softmax_segm_max.shape[0] >= num_seqs * query_len
|
||||
# DiffKV re-allocates the output; it must stay sized like the base buffers.
|
||||
assert builder.softmax_segm_output.shape[0] == builder.softmax_segm_max.shape[0]
|
||||
assert (
|
||||
metadata.softmax_segm_output.data_ptr()
|
||||
== builder.softmax_segm_output.data_ptr()
|
||||
)
|
||||
@@ -609,3 +609,25 @@ def test_update_draft_decode_metadata_skips_without_scheduler_metadata(monkeypat
|
||||
|
||||
assert not called
|
||||
assert metadata.scheduler_metadata is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("num_speculative_steps", "expected"), [(1, False), (3, True)])
|
||||
def test_decode_metadata_update_needs_draft_decode_steps(
|
||||
num_speculative_steps, expected
|
||||
):
|
||||
"""One speculative step runs no draft decodes, so the in-place metadata
|
||||
update (and its fallback log) does not apply even on a capable backend."""
|
||||
speculator = object.__new__(_TestSpeculator)
|
||||
speculator.num_speculative_steps = num_speculative_steps
|
||||
speculator.attn_groups = [
|
||||
[
|
||||
SimpleNamespace(
|
||||
supports_draft_decode_metadata_update=True,
|
||||
backend=SimpleNamespace(get_name=lambda: "CAPABLE"),
|
||||
)
|
||||
]
|
||||
]
|
||||
|
||||
speculator._configure_decode_metadata_update()
|
||||
|
||||
assert speculator.supports_decode_metadata_update is expected
|
||||
|
||||
Reference in New Issue
Block a user