[Spec Decode] Skip metadata build for dflash/dspark when all model backends support it (#60692)
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai> Co-authored-by: Rishi Puri <20074092+puririshi98@users.noreply.github.com>
This commit is contained in:
co-authored by
Rishi Puri
parent
6a70913590
commit
36f3972812
@@ -828,3 +828,57 @@ def test_v41_flashinfer_mixed_sparse_indices_respect_replay_start():
|
||||
# Decode row is copied through; prefill rows follow the bounded window.
|
||||
assert bounded[0].cpu().tolist() == plain[0].cpu().tolist()
|
||||
assert bounded[1:].cpu().tolist() == rows[1:]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_dspark_noncausal_draft_decode_update_matches_build():
|
||||
"""A DSpark draft graph refreshes its non-causal SWA metadata in place
|
||||
instead of rebuilding it, so the refresh must match a fresh build."""
|
||||
from vllm.v1.attention.backends.mla.compressor_utils import (
|
||||
get_dspark_swa_index_width,
|
||||
)
|
||||
|
||||
num_spec = 4
|
||||
query_lens = [num_spec] * 3
|
||||
|
||||
def make_dspark_builder():
|
||||
builder = make_builder(vision=False)
|
||||
builder.is_dspark = True
|
||||
builder.noncausal_index_width = get_dspark_swa_index_width(WINDOW, num_spec)
|
||||
builder.decode_threshold = num_spec
|
||||
return builder
|
||||
|
||||
def common_metadata(seq_lens):
|
||||
query_start_loc, seq_lens_t, _, slot_mapping, block_table = make_batch(
|
||||
seq_lens, query_lens, torch.device("cuda")
|
||||
)
|
||||
return CommonAttentionMetadata(
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc.cpu(),
|
||||
seq_lens=seq_lens_t,
|
||||
seq_lens_cpu_upper_bound=seq_lens_t.cpu(),
|
||||
num_reqs=len(seq_lens),
|
||||
num_actual_tokens=int(query_start_loc[-1]),
|
||||
max_query_len=num_spec,
|
||||
max_seq_len=max(seq_lens),
|
||||
block_table_tensor=block_table,
|
||||
slot_mapping=slot_mapping,
|
||||
causal=False,
|
||||
)
|
||||
|
||||
captured_seq_lens = [6, 40, 130]
|
||||
replay_seq_lens = [130, 9, 90]
|
||||
builder = make_dspark_builder()
|
||||
captured = common_metadata(captured_seq_lens)
|
||||
md = builder.build(0, captured)
|
||||
replay = common_metadata(replay_seq_lens)
|
||||
expected = make_dspark_builder().build(0, replay)
|
||||
|
||||
# A replay sees new inputs through the same device buffers.
|
||||
captured.seq_lens.copy_(replay.seq_lens)
|
||||
captured.block_table_tensor.copy_(replay.block_table_tensor)
|
||||
builder.update_draft_decode_metadata(md)
|
||||
|
||||
assert md.decode_swa_width == builder.noncausal_index_width
|
||||
torch.testing.assert_close(md.decode_swa_lens, expected.decode_swa_lens)
|
||||
torch.testing.assert_close(md.decode_swa_indices, expected.decode_swa_indices)
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.triton_attn import TritonAttentionMetadataBuilder
|
||||
from vllm.v1.attention.backends.triton_attn_diffkv import (
|
||||
TritonAttentionDiffKVMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec
|
||||
|
||||
|
||||
# 64 segments only while num_seqs * num_kv_heads * 16 < 188 SMs on SM12.0 and
|
||||
# the 64-segment scratch holds every query token (e.g. multi-query verify).
|
||||
@pytest.mark.parametrize(
|
||||
"builder_cls",
|
||||
[TritonAttentionMetadataBuilder, TritonAttentionDiffKVMetadataBuilder],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"capability,num_kv_heads,num_seqs,query_len,segments",
|
||||
[
|
||||
((12, 0), 1, 11, 1, 64),
|
||||
((12, 0), 1, 12, 1, 16),
|
||||
((12, 0), 8, 1, 1, 64),
|
||||
((12, 0), 8, 2, 1, 16),
|
||||
((9, 0), 1, 1, 1, 16),
|
||||
((12, 0), 1, 2, 4, 64),
|
||||
((12, 0), 1, 10, 4, 16),
|
||||
],
|
||||
)
|
||||
def test_split_k_segments_follow_sm_occupancy(
|
||||
monkeypatch, builder_cls, capability, num_kv_heads, num_seqs, query_len, segments
|
||||
):
|
||||
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
current_platform, "is_device_capability", lambda cap: cap == capability
|
||||
)
|
||||
monkeypatch.setattr(current_platform, "num_compute_units", lambda: 188)
|
||||
config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
get_num_attention_heads=lambda _: 8,
|
||||
get_num_kv_heads=lambda _: num_kv_heads,
|
||||
get_head_size=lambda: 128,
|
||||
rswa_window=None,
|
||||
),
|
||||
parallel_config=None,
|
||||
scheduler_config=SimpleNamespace(max_num_seqs=16),
|
||||
speculative_config=None,
|
||||
compilation_config=SimpleNamespace(
|
||||
cudagraph_mode=CUDAGraphMode.NONE, static_forward_context={}
|
||||
),
|
||||
)
|
||||
spec = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=num_kv_heads, head_size=128, dtype=torch.bfloat16
|
||||
)
|
||||
builder = builder_cls(spec, ["layer.0"], config, "cpu")
|
||||
batch = BatchSpec(seq_lens=[128] * num_seqs, query_lens=[query_len] * num_seqs)
|
||||
metadata = builder.build(0, create_common_attn_metadata(batch, 16, "cpu"))
|
||||
assert metadata.num_par_softmax_segments == segments
|
||||
assert metadata.softmax_segm_output.shape[2] == segments
|
||||
assert metadata.softmax_segm_max.shape[0] >= num_seqs * query_len
|
||||
# DiffKV re-allocates the output; it must stay sized like the base buffers.
|
||||
assert builder.softmax_segm_output.shape[0] == builder.softmax_segm_max.shape[0]
|
||||
assert (
|
||||
metadata.softmax_segm_output.data_ptr()
|
||||
== builder.softmax_segm_output.data_ptr()
|
||||
)
|
||||
|
||||
|
||||
def _draft_builder(builder_cls, rswa_window: int | None):
|
||||
config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
get_num_attention_heads=lambda _: 8,
|
||||
get_num_kv_heads=lambda _: 1,
|
||||
get_head_size=lambda: 128,
|
||||
rswa_window=rswa_window,
|
||||
),
|
||||
parallel_config=None,
|
||||
scheduler_config=SimpleNamespace(max_num_seqs=16),
|
||||
speculative_config=None,
|
||||
compilation_config=SimpleNamespace(
|
||||
cudagraph_mode=CUDAGraphMode.NONE, static_forward_context={}
|
||||
),
|
||||
)
|
||||
spec = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=128, dtype=torch.bfloat16
|
||||
)
|
||||
return builder_cls(spec, ["layer.0"], config, "cpu")
|
||||
|
||||
|
||||
def _draft_decode_batch(seq_lens: list[int]):
|
||||
batch = BatchSpec(seq_lens=seq_lens, query_lens=[1] * len(seq_lens))
|
||||
return create_common_attn_metadata(batch, 16, "cpu")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"builder_cls",
|
||||
[TritonAttentionMetadataBuilder, TritonAttentionDiffKVMetadataBuilder],
|
||||
)
|
||||
@pytest.mark.parametrize("rswa_window", [None, 64])
|
||||
def test_draft_decode_update_flag_matches_build_writes(builder_cls, rswa_window):
|
||||
"""The update is a no-op, so the builder may claim support only when
|
||||
build() writes no builder-owned tensor that a captured graph reads."""
|
||||
builder = _draft_builder(builder_cls, rswa_window)
|
||||
common_metadata = _draft_decode_batch([5, 9])
|
||||
common_metadata.rswa_prefix_lens = torch.tensor([3, 7], dtype=torch.int32)
|
||||
|
||||
def owned_bytes():
|
||||
# Bytes, not values: uninitialized scratch may hold NaNs.
|
||||
return {
|
||||
name: value.view(torch.uint8).clone()
|
||||
for name, value in vars(builder).items()
|
||||
if isinstance(value, torch.Tensor)
|
||||
}
|
||||
|
||||
before = owned_bytes()
|
||||
builder.build(0, common_metadata)
|
||||
after = owned_bytes()
|
||||
|
||||
written = [name for name in before if not torch.equal(before[name], after[name])]
|
||||
assert builder.supports_draft_decode_metadata_update is (not written)
|
||||
assert builder.supports_draft_decode_metadata_update is (rswa_window is None)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"builder_cls",
|
||||
[TritonAttentionMetadataBuilder, TritonAttentionDiffKVMetadataBuilder],
|
||||
)
|
||||
def test_draft_decode_update_matches_fresh_build(builder_cls):
|
||||
"""A replayed graph reads metadata built at capture; after an in-place
|
||||
update it must match a fresh build of the replayed batch."""
|
||||
torch.manual_seed(0)
|
||||
builder = _draft_builder(builder_cls, rswa_window=None)
|
||||
captured = _draft_decode_batch([100, 120])
|
||||
metadata = builder.build(0, captured)
|
||||
|
||||
replayed = _draft_decode_batch([128, 30])
|
||||
# The replayed batch arrives through the same persistent input buffers.
|
||||
captured.seq_lens.copy_(replayed.seq_lens)
|
||||
captured.block_table_tensor.copy_(replayed.block_table_tensor)
|
||||
captured.slot_mapping.copy_(replayed.slot_mapping)
|
||||
builder.update_draft_decode_metadata(metadata)
|
||||
|
||||
expected = _draft_builder(builder_cls, rswa_window=None).build(0, replayed)
|
||||
assert torch.equal(metadata.seq_lens, expected.seq_lens)
|
||||
assert torch.equal(metadata.block_table, expected.block_table)
|
||||
assert torch.equal(metadata.slot_mapping, expected.slot_mapping)
|
||||
assert torch.equal(metadata.query_start_loc, expected.query_start_loc)
|
||||
@@ -1,73 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.triton_attn import TritonAttentionMetadataBuilder
|
||||
from vllm.v1.attention.backends.triton_attn_diffkv import (
|
||||
TritonAttentionDiffKVMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec
|
||||
|
||||
|
||||
# 64 segments only while num_seqs * num_kv_heads * 16 < 188 SMs on SM12.0 and
|
||||
# the 64-segment scratch holds every query token (e.g. multi-query verify).
|
||||
@pytest.mark.parametrize(
|
||||
"builder_cls",
|
||||
[TritonAttentionMetadataBuilder, TritonAttentionDiffKVMetadataBuilder],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"capability,num_kv_heads,num_seqs,query_len,segments",
|
||||
[
|
||||
((12, 0), 1, 11, 1, 64),
|
||||
((12, 0), 1, 12, 1, 16),
|
||||
((12, 0), 8, 1, 1, 64),
|
||||
((12, 0), 8, 2, 1, 16),
|
||||
((9, 0), 1, 1, 1, 16),
|
||||
((12, 0), 1, 2, 4, 64),
|
||||
((12, 0), 1, 10, 4, 16),
|
||||
],
|
||||
)
|
||||
def test_split_k_segments_follow_sm_occupancy(
|
||||
monkeypatch, builder_cls, capability, num_kv_heads, num_seqs, query_len, segments
|
||||
):
|
||||
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
current_platform, "is_device_capability", lambda cap: cap == capability
|
||||
)
|
||||
monkeypatch.setattr(current_platform, "num_compute_units", lambda: 188)
|
||||
config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
get_num_attention_heads=lambda _: 8,
|
||||
get_num_kv_heads=lambda _: num_kv_heads,
|
||||
get_head_size=lambda: 128,
|
||||
rswa_window=None,
|
||||
),
|
||||
parallel_config=None,
|
||||
scheduler_config=SimpleNamespace(max_num_seqs=16),
|
||||
speculative_config=None,
|
||||
compilation_config=SimpleNamespace(
|
||||
cudagraph_mode=CUDAGraphMode.NONE, static_forward_context={}
|
||||
),
|
||||
)
|
||||
spec = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=num_kv_heads, head_size=128, dtype=torch.bfloat16
|
||||
)
|
||||
builder = builder_cls(spec, ["layer.0"], config, "cpu")
|
||||
batch = BatchSpec(seq_lens=[128] * num_seqs, query_lens=[query_len] * num_seqs)
|
||||
metadata = builder.build(0, create_common_attn_metadata(batch, 16, "cpu"))
|
||||
assert metadata.num_par_softmax_segments == segments
|
||||
assert metadata.softmax_segm_output.shape[2] == segments
|
||||
assert metadata.softmax_segm_max.shape[0] >= num_seqs * query_len
|
||||
# DiffKV re-allocates the output; it must stay sized like the base buffers.
|
||||
assert builder.softmax_segm_output.shape[0] == builder.softmax_segm_max.shape[0]
|
||||
assert (
|
||||
metadata.softmax_segm_output.data_ptr()
|
||||
== builder.softmax_segm_output.data_ptr()
|
||||
)
|
||||
@@ -609,3 +609,25 @@ def test_update_draft_decode_metadata_skips_without_scheduler_metadata(monkeypat
|
||||
|
||||
assert not called
|
||||
assert metadata.scheduler_metadata is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("num_speculative_steps", "expected"), [(1, False), (3, True)])
|
||||
def test_decode_metadata_update_needs_draft_decode_steps(
|
||||
num_speculative_steps, expected
|
||||
):
|
||||
"""One speculative step runs no draft decodes, so the in-place metadata
|
||||
update (and its fallback log) does not apply even on a capable backend."""
|
||||
speculator = object.__new__(_TestSpeculator)
|
||||
speculator.num_speculative_steps = num_speculative_steps
|
||||
speculator.attn_groups = [
|
||||
[
|
||||
SimpleNamespace(
|
||||
supports_draft_decode_metadata_update=True,
|
||||
backend=SimpleNamespace(get_name=lambda: "CAPABLE"),
|
||||
)
|
||||
]
|
||||
]
|
||||
|
||||
speculator._configure_decode_metadata_update()
|
||||
|
||||
assert speculator.supports_decode_metadata_update is expected
|
||||
|
||||
@@ -206,24 +206,36 @@ class DeepseekSparseSWAFlashInferMetadataBuilder(DeepseekV41SparseSWAMetadataBui
|
||||
metadata = super().build(
|
||||
common_prefix_len, common_attn_metadata, fast_build, replay_start
|
||||
)
|
||||
num_tokens = metadata.num_decode_tokens
|
||||
if not common_attn_metadata.causal and num_tokens > 0:
|
||||
assert metadata.decode_swa_lens is not None
|
||||
assert metadata.seq_lens is not None
|
||||
assert metadata.token_to_req_indices is not None
|
||||
topk_lens = self._decode_topk_lens[:num_tokens]
|
||||
seq_lens = self._decode_seq_lens[:num_tokens]
|
||||
torch.clamp(metadata.decode_swa_lens, min=self.window_size, out=topk_lens)
|
||||
torch.index_select(
|
||||
metadata.seq_lens,
|
||||
0,
|
||||
metadata.token_to_req_indices[:num_tokens],
|
||||
out=seq_lens,
|
||||
)
|
||||
metadata.flashinfer_decode_topk_lens = topk_lens
|
||||
metadata.flashinfer_decode_seq_lens = seq_lens
|
||||
if not common_attn_metadata.causal and metadata.num_decode_tokens > 0:
|
||||
self._fill_noncausal_decode_lens(metadata)
|
||||
return metadata
|
||||
|
||||
def update_draft_decode_metadata(
|
||||
self, metadata: "DeepseekSparseSWAMetadata"
|
||||
) -> None:
|
||||
super().update_draft_decode_metadata(metadata)
|
||||
if metadata.flashinfer_decode_topk_lens is not None:
|
||||
self._fill_noncausal_decode_lens(metadata)
|
||||
|
||||
def _fill_noncausal_decode_lens(
|
||||
self, metadata: "DeepseekSparseSWAMetadata"
|
||||
) -> None:
|
||||
num_tokens = metadata.num_decode_tokens
|
||||
assert metadata.decode_swa_lens is not None
|
||||
assert metadata.seq_lens is not None
|
||||
assert metadata.token_to_req_indices is not None
|
||||
topk_lens = self._decode_topk_lens[:num_tokens]
|
||||
seq_lens = self._decode_seq_lens[:num_tokens]
|
||||
torch.clamp(metadata.decode_swa_lens, min=self.window_size, out=topk_lens)
|
||||
torch.index_select(
|
||||
metadata.seq_lens,
|
||||
0,
|
||||
metadata.token_to_req_indices[:num_tokens],
|
||||
out=seq_lens,
|
||||
)
|
||||
metadata.flashinfer_decode_topk_lens = topk_lens
|
||||
metadata.flashinfer_decode_seq_lens = seq_lens
|
||||
|
||||
|
||||
class DeepseekSparseSWAFlashInferBackend(DeepseekSparseSWABackend):
|
||||
@staticmethod
|
||||
|
||||
@@ -852,23 +852,44 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
|
||||
num_tokens,
|
||||
)
|
||||
|
||||
_COMPUTE_SWA_INDICES_AND_LENS_KERNEL(
|
||||
metadata.decode_swa_indices,
|
||||
metadata.decode_swa_lens,
|
||||
metadata.decode_swa_indices.shape[-1],
|
||||
metadata.decode_swa_indices.shape[-1],
|
||||
metadata.decode_swa_lens, # unused (HAS_IMAGE=False)
|
||||
metadata.decode_swa_lens, # unused (HAS_IMAGE=False)
|
||||
metadata.query_start_loc,
|
||||
metadata.seq_lens,
|
||||
metadata.token_to_req_indices,
|
||||
metadata.is_valid_token,
|
||||
metadata.block_table,
|
||||
self.block_size,
|
||||
metadata.replay_start,
|
||||
num_tokens=metadata.num_decode_tokens,
|
||||
token_offset=0,
|
||||
)
|
||||
# build() points non-causal (DSpark draft) metadata at its own buffer.
|
||||
noncausal = self.decode_swa_indices_noncausal
|
||||
if (
|
||||
noncausal is not None
|
||||
and metadata.decode_swa_indices.data_ptr() == noncausal.data_ptr()
|
||||
):
|
||||
_COMPUTE_DSPARK_NONCAUSAL_SWA_INDICES_KERNEL(
|
||||
metadata.decode_swa_indices,
|
||||
metadata.decode_swa_lens,
|
||||
self.window_size,
|
||||
self.noncausal_index_width,
|
||||
metadata.query_start_loc,
|
||||
metadata.seq_lens,
|
||||
metadata.token_to_req_indices,
|
||||
metadata.is_valid_token,
|
||||
metadata.block_table,
|
||||
self.block_size,
|
||||
num_tokens=num_tokens,
|
||||
token_offset=0,
|
||||
)
|
||||
else:
|
||||
_COMPUTE_SWA_INDICES_AND_LENS_KERNEL(
|
||||
metadata.decode_swa_indices,
|
||||
metadata.decode_swa_lens,
|
||||
metadata.decode_swa_indices.shape[-1],
|
||||
metadata.decode_swa_indices.shape[-1],
|
||||
metadata.decode_swa_lens, # unused (HAS_IMAGE=False)
|
||||
metadata.decode_swa_lens, # unused (HAS_IMAGE=False)
|
||||
metadata.query_start_loc,
|
||||
metadata.seq_lens,
|
||||
metadata.token_to_req_indices,
|
||||
metadata.is_valid_token,
|
||||
metadata.block_table,
|
||||
self.block_size,
|
||||
metadata.replay_start,
|
||||
num_tokens=metadata.num_decode_tokens,
|
||||
token_offset=0,
|
||||
)
|
||||
tile_sched = self.build_tile_scheduler(metadata.num_decode_tokens)
|
||||
metadata.tile_sched_swaonly = tile_sched[_LAYER_TYPE_SWAONLY]
|
||||
metadata.tile_sched_c4a = tile_sched[_LAYER_TYPE_C4A]
|
||||
|
||||
@@ -193,6 +193,9 @@ class TritonAttentionMetadataBuilder(AttentionMetadataBuilder[TritonAttentionMet
|
||||
device=device,
|
||||
)
|
||||
self.rswa_window = model_config.rswa_window
|
||||
# R-SWA restages prefix lengths in build(), which the no-op
|
||||
# update_draft_decode_metadata() does not refresh.
|
||||
self.supports_draft_decode_metadata_update = self.rswa_window is None
|
||||
self.persistent_rswa_prefix_lens: torch.Tensor | None = None
|
||||
if self.rswa_window is not None:
|
||||
self.persistent_rswa_prefix_lens = torch.empty(
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable, Mapping
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
@@ -84,11 +85,11 @@ class DFlashCudaGraphManager(CudaGraphManager):
|
||||
kv_cache_config: KVCacheConfig,
|
||||
max_model_len: int,
|
||||
causal: bool | Mapping[int, bool],
|
||||
precompute_context_kv: Callable[[int], None],
|
||||
prepare_forward: Callable[[int, dict[str, Any] | None], None],
|
||||
progress_bar_desc: str = "Capturing CUDA graphs",
|
||||
) -> None:
|
||||
"""``precompute_context_kv(num_reqs)`` is captured ahead of the query
|
||||
forward."""
|
||||
"""``prepare_forward(num_reqs, attn_metadata)`` is captured ahead of the
|
||||
query forward."""
|
||||
|
||||
def create_forward_fn(
|
||||
desc: BatchExecutionDescriptor,
|
||||
@@ -115,7 +116,7 @@ class DFlashCudaGraphManager(CudaGraphManager):
|
||||
attn_metadata, slot_mappings = attn_state
|
||||
|
||||
def forward(cg_mode: CUDAGraphMode) -> None:
|
||||
precompute_context_kv(num_reqs)
|
||||
prepare_forward(num_reqs, attn_metadata)
|
||||
forward_fn(
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
|
||||
@@ -166,12 +166,18 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
self.kv_cache_config,
|
||||
self.max_model_len,
|
||||
causal=self._group_causal,
|
||||
precompute_context_kv=lambda num_reqs: self._precompute_context_kv(
|
||||
0, self._num_graph_context_tokens(num_reqs)
|
||||
),
|
||||
prepare_forward=self._prepare_graph_forward,
|
||||
progress_bar_desc=f"Capturing {self._speculator_name.lower()} CUDA graphs",
|
||||
)
|
||||
|
||||
def _prepare_graph_forward(
|
||||
self, num_reqs: int, attn_metadata: dict[str, Any] | None
|
||||
) -> None:
|
||||
self._precompute_context_kv(0, self._num_graph_context_tokens(num_reqs))
|
||||
# Recorded in the FULL graph so replays skip the eager metadata build.
|
||||
if attn_metadata is not None and self.supports_decode_metadata_update:
|
||||
self._update_draft_decode_metadata(attn_metadata, num_reqs)
|
||||
|
||||
def load_draft_model(
|
||||
self,
|
||||
target_model: nn.Module,
|
||||
@@ -486,29 +492,32 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
else:
|
||||
self._precompute_context_kv(0, num_target_tokens, dummy_run)
|
||||
|
||||
# Rebuild the draft attention metadata even when replaying the FULL
|
||||
# graph so that any attention metadata builder state is updated.
|
||||
draft_attn_metadata = self._build_uniform_attn_metadata(
|
||||
num_reqs=num_reqs,
|
||||
batch_desc=batch_desc,
|
||||
num_query_per_req=self.num_query_per_req,
|
||||
seq_lens_cpu_upper_bound=input_batch.seq_lens_cpu_upper_bound,
|
||||
step=self.num_query_per_req,
|
||||
causal=self._group_causal,
|
||||
)
|
||||
draft_slot_mappings_by_layer = build_slot_mappings_by_layer(
|
||||
self.block_tables.slot_mappings[:, :num_tokens_padded],
|
||||
self.kv_cache_config,
|
||||
)
|
||||
is_full_cudagraph = batch_desc.cg_mode == CUDAGraphMode.FULL
|
||||
# During FULL cudagraph with a model that supports decode metadata updates,
|
||||
# the metadata update is captured in the graph, and we can skip the full
|
||||
# build below.
|
||||
if not (is_full_cudagraph and self.supports_decode_metadata_update):
|
||||
draft_attn_metadata = self._build_uniform_attn_metadata(
|
||||
num_reqs=num_reqs,
|
||||
batch_desc=batch_desc,
|
||||
num_query_per_req=self.num_query_per_req,
|
||||
seq_lens_cpu_upper_bound=input_batch.seq_lens_cpu_upper_bound,
|
||||
step=self.num_query_per_req,
|
||||
causal=self._group_causal,
|
||||
)
|
||||
|
||||
# DFlash processes all speculative tokens in one forward pass,
|
||||
# so the real token count is num_query_tokens.
|
||||
self._prepare_eplb_forward(num_query_tokens)
|
||||
|
||||
if batch_desc.cg_mode == CUDAGraphMode.FULL:
|
||||
if is_full_cudagraph:
|
||||
assert self.query_cudagraph_manager is not None
|
||||
self.query_cudagraph_manager.run_fullgraph(batch_desc)
|
||||
else:
|
||||
draft_slot_mappings_by_layer = build_slot_mappings_by_layer(
|
||||
self.block_tables.slot_mappings[:, :num_tokens_padded],
|
||||
self.kv_cache_config,
|
||||
)
|
||||
self._generate_draft(
|
||||
num_reqs,
|
||||
num_tokens_padded,
|
||||
|
||||
@@ -207,6 +207,9 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
|
||||
self.supports_mm_inputs = False
|
||||
self.pcp_manager: PCPManager | None = None
|
||||
# Whether every draft attention builder can refresh its decode metadata
|
||||
# in place, so the refresh can be recorded in the draft CUDA graph.
|
||||
self.supports_decode_metadata_update = False
|
||||
|
||||
@abstractmethod
|
||||
def load_draft_model(
|
||||
@@ -299,6 +302,24 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
# builders and buffers.
|
||||
self.target_input_buffers = target_input_buffers
|
||||
self.target_attn_groups = target_attn_groups
|
||||
self._configure_decode_metadata_update()
|
||||
|
||||
def _configure_decode_metadata_update(self) -> None:
|
||||
unsupported_backends = sorted(
|
||||
{
|
||||
attn_group.backend.get_name()
|
||||
for attn_groups in self.attn_groups
|
||||
for attn_group in attn_groups
|
||||
if not attn_group.supports_draft_decode_metadata_update
|
||||
}
|
||||
)
|
||||
self.supports_decode_metadata_update = not unsupported_backends
|
||||
if unsupported_backends:
|
||||
logger.info_once(
|
||||
"Attention backend(s) %s cannot update draft decode metadata in "
|
||||
"place; falling back to rebuilding attention metadata eagerly.",
|
||||
", ".join(unsupported_backends),
|
||||
)
|
||||
|
||||
def _build_attn_metadata(
|
||||
self,
|
||||
|
||||
@@ -10,18 +10,15 @@ from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.forward_context import BatchDescriptor, set_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
|
||||
from vllm.v1.worker.gpu.dp_utils import DPSyncState, dispatch_cg_and_sync_dp
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
|
||||
from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.cudagraph_utils import (
|
||||
SpeculatorCudaGraphManager,
|
||||
)
|
||||
from vllm.v1.worker.utils import AttentionGroup, get_uniform_decode_token_count
|
||||
from vllm.v1.worker.utils import get_uniform_decode_token_count
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -51,7 +48,6 @@ class TargetDependentARSpeculator(DraftModelSpeculator):
|
||||
|
||||
self.prefill_cudagraph_manager: SpeculatorCudaGraphManager | None = None
|
||||
self.decode_cudagraph_manager: SpeculatorCudaGraphManager | None = None
|
||||
self.use_fused_multi_step_decode = False
|
||||
|
||||
def load_model(self, target_model: nn.Module) -> None:
|
||||
super().load_model(target_model)
|
||||
@@ -83,44 +79,12 @@ class TargetDependentARSpeculator(DraftModelSpeculator):
|
||||
"""
|
||||
return True
|
||||
|
||||
def set_attn(
|
||||
self,
|
||||
model_state: ModelState,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
block_tables: BlockTables,
|
||||
target_input_buffers: InputBuffers,
|
||||
target_attn_groups: list[list[AttentionGroup]],
|
||||
) -> None:
|
||||
super().set_attn(
|
||||
model_state,
|
||||
kv_cache_config,
|
||||
block_tables,
|
||||
target_input_buffers,
|
||||
target_attn_groups,
|
||||
)
|
||||
self._configure_fused_multi_step_decode()
|
||||
|
||||
def _configure_fused_multi_step_decode(self) -> None:
|
||||
def _configure_decode_metadata_update(self) -> None:
|
||||
# With one speculative step there are no draft decode steps to update.
|
||||
if self.num_speculative_steps == 1:
|
||||
self.use_fused_multi_step_decode = False
|
||||
self.supports_decode_metadata_update = False
|
||||
return
|
||||
|
||||
unsupported_backends = sorted(
|
||||
{
|
||||
attn_group.backend.get_name()
|
||||
for attn_groups in self.attn_groups
|
||||
for attn_group in attn_groups
|
||||
if not attn_group.supports_draft_decode_metadata_update
|
||||
}
|
||||
)
|
||||
self.use_fused_multi_step_decode = not unsupported_backends
|
||||
if unsupported_backends:
|
||||
logger.info_once(
|
||||
"Fused multi-step draft decode is not supported by attention "
|
||||
"backend(s) %s; falling back to rebuilding attention metadata "
|
||||
"between draft steps.",
|
||||
", ".join(unsupported_backends),
|
||||
)
|
||||
super()._configure_decode_metadata_update()
|
||||
|
||||
def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None:
|
||||
# Initialize cudagraph manager for draft prefill (draft position 0).
|
||||
@@ -181,7 +145,7 @@ class TargetDependentARSpeculator(DraftModelSpeculator):
|
||||
assert self.decode_cudagraph_manager is not None
|
||||
decode_fn = (
|
||||
self._generate_fused_drafts
|
||||
if self.use_fused_multi_step_decode
|
||||
if self.supports_decode_metadata_update
|
||||
else self._generate_draft
|
||||
)
|
||||
self.decode_cudagraph_manager.capture(
|
||||
@@ -192,7 +156,7 @@ class TargetDependentARSpeculator(DraftModelSpeculator):
|
||||
self.attn_groups,
|
||||
self.kv_cache_config,
|
||||
progress_bar_desc="Capturing decode CUDA graphs",
|
||||
specialize_spec_tokens=self.use_fused_multi_step_decode
|
||||
specialize_spec_tokens=self.supports_decode_metadata_update
|
||||
and self.speculative_config.uses_dynamic_speculative_decoding(),
|
||||
)
|
||||
self.on_multi_step_decode_end(self.max_num_reqs)
|
||||
@@ -384,7 +348,7 @@ class TargetDependentARSpeculator(DraftModelSpeculator):
|
||||
# Generate the remaining num_speculative_steps - 1 draft tokens.
|
||||
decode_fn = (
|
||||
self._fused_multi_step_decode
|
||||
if self.use_fused_multi_step_decode
|
||||
if self.supports_decode_metadata_update
|
||||
else self._multi_step_decode
|
||||
)
|
||||
decode_fn(
|
||||
|
||||
Reference in New Issue
Block a user