[MRV2][Spec] Fuse AR speculator multi-step decodes back into one CUDA graph (#46849)
Signed-off-by: Yizhou Liu <liu_yizhou@outlook.com>
This commit is contained in:
@@ -3,6 +3,7 @@
|
||||
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -16,6 +17,9 @@ from vllm.model_executor.models.mistral_eagle import EagleMistralForCausalLM
|
||||
from vllm.model_executor.models.mistral_large_3_eagle import (
|
||||
EagleMistralLarge3ForCausalLM,
|
||||
)
|
||||
from vllm.v1.attention.backends import flash_attn as flash_attn_module
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionMetadata
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
|
||||
from vllm.v1.worker.gpu.spec_decode import speculator as base_spec_module
|
||||
from vllm.v1.worker.gpu.spec_decode.autoregressive import speculator as spec_module
|
||||
from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
|
||||
@@ -278,3 +282,165 @@ def test_run_model_reuses_tensor_return_for_mtp(monkeypatch):
|
||||
|
||||
assert actual_logits_hidden is hidden
|
||||
assert actual_feedback_hidden is hidden
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"method_name",
|
||||
"cg_mode",
|
||||
"expected_eager_calls",
|
||||
"expected_graph_replays",
|
||||
),
|
||||
[
|
||||
("_multi_step_decode", CUDAGraphMode.NONE, 3, 0),
|
||||
("_multi_step_decode", CUDAGraphMode.FULL, 0, 3),
|
||||
("_fused_multi_step_decode", CUDAGraphMode.NONE, 3, 0),
|
||||
("_fused_multi_step_decode", CUDAGraphMode.FULL, 0, 1),
|
||||
],
|
||||
)
|
||||
def test_multi_step_decode_replays_captured_graph_as_expected(
|
||||
method_name,
|
||||
cg_mode,
|
||||
expected_eager_calls,
|
||||
expected_graph_replays,
|
||||
):
|
||||
speculator = object.__new__(_TestSpeculator)
|
||||
speculator.num_speculative_steps = 4
|
||||
speculator.current_draft_step = torch.tensor(0)
|
||||
speculator.input_buffers = SimpleNamespace(
|
||||
positions=torch.arange(2),
|
||||
query_start_loc=torch.arange(3),
|
||||
)
|
||||
speculator.idx_mapping = torch.arange(2)
|
||||
generate_draft = Mock()
|
||||
speculator._generate_draft = generate_draft
|
||||
run_fullgraph = Mock()
|
||||
speculator.decode_cudagraph_manager = SimpleNamespace(run_fullgraph=run_fullgraph)
|
||||
batch_desc = BatchExecutionDescriptor(
|
||||
cg_mode=cg_mode,
|
||||
num_tokens=2,
|
||||
num_reqs=2,
|
||||
)
|
||||
|
||||
getattr(speculator, method_name)(
|
||||
num_reqs=2,
|
||||
skip_attn=True,
|
||||
batch_desc=batch_desc,
|
||||
seq_lens_cpu_upper_bound=None,
|
||||
num_tokens_across_dp=None,
|
||||
)
|
||||
|
||||
assert generate_draft.call_count == expected_eager_calls
|
||||
assert run_fullgraph.call_count == expected_graph_replays
|
||||
|
||||
|
||||
def test_update_draft_decode_metadata_updates_fa3_scheduler_metadata(
|
||||
monkeypatch,
|
||||
):
|
||||
builder = object.__new__(flash_attn_module.FlashAttentionMetadataBuilder)
|
||||
builder.aot_schedule = True
|
||||
builder.use_full_cuda_graph = True
|
||||
builder.scheduler_metadata = torch.zeros(8, dtype=torch.int32)
|
||||
builder.cache_config = SimpleNamespace(cache_dtype="bfloat16")
|
||||
builder.kv_cache_dtype = torch.bfloat16
|
||||
builder.num_heads_q = 2
|
||||
builder.num_heads_kv = 1
|
||||
builder.headdim = 128
|
||||
builder.block_size = 16
|
||||
builder.dcp_world_size = 1
|
||||
builder.dcp_rank = 0
|
||||
builder.cp_kv_cache_interleave_size = 1
|
||||
builder.aot_sliding_window = None
|
||||
|
||||
expected = torch.tensor([7, 8, 9], dtype=torch.int32)
|
||||
|
||||
def fake_get_scheduler_metadata(**kwargs):
|
||||
return expected
|
||||
|
||||
monkeypatch.setattr(builder, "_get_scheduler_metadata", fake_get_scheduler_metadata)
|
||||
|
||||
metadata = FlashAttentionMetadata(
|
||||
num_actual_tokens=3,
|
||||
max_query_len=2,
|
||||
query_start_loc=torch.tensor([0, 1, 3], dtype=torch.int32),
|
||||
max_seq_len=8,
|
||||
seq_lens=torch.tensor([5, 6], dtype=torch.int32),
|
||||
block_table=torch.zeros((2, 1), dtype=torch.int32),
|
||||
slot_mapping=torch.zeros(3, dtype=torch.int32),
|
||||
use_cascade=False,
|
||||
common_prefix_len=0,
|
||||
cu_prefix_query_lens=None,
|
||||
prefix_kv_lens=None,
|
||||
suffix_kv_lens=None,
|
||||
max_dcp_context_kv_len=None,
|
||||
dcp_context_kv_lens=None,
|
||||
num_decode_reqs=2,
|
||||
num_prefill_reqs=0,
|
||||
num_decode_tokens=3,
|
||||
num_prefill_tokens=0,
|
||||
scheduler_metadata=torch.tensor([-1, -1, -1], dtype=torch.int32),
|
||||
prefix_scheduler_metadata=None,
|
||||
max_num_splits=4,
|
||||
causal=True,
|
||||
sliding_window=None,
|
||||
mm_prefix_query_range_tensor=None,
|
||||
rswa_prefix_lens=None,
|
||||
rswa_window=None,
|
||||
rswa_window_tensor=None,
|
||||
)
|
||||
|
||||
builder.update_draft_decode_metadata(metadata)
|
||||
|
||||
assert torch.equal(metadata.scheduler_metadata, expected)
|
||||
assert torch.equal(builder.scheduler_metadata[:3], expected)
|
||||
|
||||
|
||||
def test_update_draft_decode_metadata_skips_without_scheduler_metadata(monkeypatch):
|
||||
builder = object.__new__(flash_attn_module.FlashAttentionMetadataBuilder)
|
||||
builder.aot_schedule = True
|
||||
builder.use_full_cuda_graph = True
|
||||
builder.scheduler_metadata = torch.zeros(4, dtype=torch.int32)
|
||||
|
||||
called = False
|
||||
|
||||
def fake_get_scheduler_metadata(**kwargs):
|
||||
nonlocal called
|
||||
called = True
|
||||
return torch.tensor([1], dtype=torch.int32)
|
||||
|
||||
monkeypatch.setattr(builder, "_get_scheduler_metadata", fake_get_scheduler_metadata)
|
||||
|
||||
metadata = FlashAttentionMetadata(
|
||||
num_actual_tokens=1,
|
||||
max_query_len=1,
|
||||
query_start_loc=torch.tensor([0, 1], dtype=torch.int32),
|
||||
max_seq_len=1,
|
||||
seq_lens=torch.tensor([1], dtype=torch.int32),
|
||||
block_table=torch.zeros((1, 1), dtype=torch.int32),
|
||||
slot_mapping=torch.zeros(1, dtype=torch.int32),
|
||||
use_cascade=False,
|
||||
common_prefix_len=0,
|
||||
cu_prefix_query_lens=None,
|
||||
prefix_kv_lens=None,
|
||||
suffix_kv_lens=None,
|
||||
max_dcp_context_kv_len=None,
|
||||
dcp_context_kv_lens=None,
|
||||
num_decode_reqs=1,
|
||||
num_prefill_reqs=0,
|
||||
num_decode_tokens=1,
|
||||
num_prefill_tokens=0,
|
||||
scheduler_metadata=None,
|
||||
prefix_scheduler_metadata=None,
|
||||
max_num_splits=1,
|
||||
causal=True,
|
||||
sliding_window=None,
|
||||
mm_prefix_query_range_tensor=None,
|
||||
rswa_prefix_lens=None,
|
||||
rswa_window=None,
|
||||
rswa_window_tensor=None,
|
||||
)
|
||||
|
||||
builder.update_draft_decode_metadata(metadata)
|
||||
|
||||
assert not called
|
||||
assert metadata.scheduler_metadata is None
|
||||
|
||||
Reference in New Issue
Block a user