[Bugfix][MRV2][Spec Decode] Honor dynamic K in autoregressive speculators, 1.2~1.3x kernel perf improvement (#57053)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -20,8 +20,12 @@ from vllm.distributed.device_communicators import pynccl_allocator
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.worker.gpu import cudagraph_utils as gpu_cudagraph_utils
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
|
||||
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
|
||||
from vllm.v1.worker.gpu.input_batch import InputBuffers
|
||||
from vllm.v1.worker.gpu.pcp_manager import PCPManager
|
||||
from vllm.v1.worker.gpu.spec_decode.autoregressive import (
|
||||
cudagraph_utils as spec_cudagraph_utils,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import (
|
||||
SpeculatorCudaGraphManager,
|
||||
)
|
||||
@@ -226,8 +230,10 @@ def test_speculator_capture_preserves_decode_query_bounds(
|
||||
)
|
||||
manager = SpeculatorCudaGraphManager.__new__(SpeculatorCudaGraphManager)
|
||||
manager.max_num_reqs = num_reqs
|
||||
manager.decode_query_len = uniform_token_count or max_query_len or num_tokens
|
||||
manager.vllm_config = SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(data_parallel_size=1)
|
||||
parallel_config=SimpleNamespace(data_parallel_size=1),
|
||||
num_speculative_tokens=3,
|
||||
)
|
||||
buffers = InputBuffers(num_reqs, num_tokens, torch.device("cpu"))
|
||||
block_tables = MagicMock()
|
||||
@@ -488,6 +494,78 @@ def test_dynamic_spec_decode_shared_token_count_stays_reachable(monkeypatch):
|
||||
), desc
|
||||
|
||||
|
||||
@pytest.mark.parametrize("specialize", [False, True])
|
||||
def test_speculator_replays_graph_for_runtime_width(monkeypatch, specialize):
|
||||
"""Equal batch shapes must replay the graph captured for the selected K."""
|
||||
monkeypatch.setattr(
|
||||
gpu_cudagraph_utils,
|
||||
"get_pp_group",
|
||||
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
gpu_cudagraph_utils.current_platform, "get_global_graph_pool", lambda: object()
|
||||
)
|
||||
monkeypatch.setattr(gpu_cudagraph_utils, "get_offloader", MagicMock)
|
||||
monkeypatch.setattr(
|
||||
spec_cudagraph_utils, "prepare_inputs_to_capture", lambda *a, **kw: ({}, {})
|
||||
)
|
||||
forward = MagicMock()
|
||||
replayed_widths: list[int] = []
|
||||
|
||||
def capture(manager, create_forward_fn, progress_bar_desc):
|
||||
for desc in manager._capture_descs[CUDAGraphMode.FULL]:
|
||||
create_forward_fn(desc, warmup=False)(CUDAGraphMode.NONE)
|
||||
k = forward.call_args.kwargs["num_speculative_steps"]
|
||||
graph = MagicMock()
|
||||
graph.replay.side_effect = lambda k=k: replayed_widths.append(k)
|
||||
manager.graphs[desc] = graph
|
||||
manager._graphs_captured = True
|
||||
|
||||
monkeypatch.setattr(gpu_cudagraph_utils.CudaGraphManager, "capture", capture)
|
||||
manager = spec_cudagraph_utils.SpeculatorCudaGraphManager(
|
||||
_create_decode_vllm_config(
|
||||
capture_sizes=[4],
|
||||
num_speculative_tokens=4,
|
||||
dynamic_spec_schedule=[(1, 2, 4), (3, 4, 2), (5, 6, 1), (7, 8, 0)],
|
||||
),
|
||||
device=torch.device("cpu"),
|
||||
cudagraph_mode=CUDAGraphMode.FULL_DECODE_ONLY,
|
||||
decode_query_len=1,
|
||||
)
|
||||
manager.capture(
|
||||
forward,
|
||||
MagicMock(),
|
||||
MagicMock(),
|
||||
MagicMock(),
|
||||
[],
|
||||
MagicMock(),
|
||||
specialize_spec_tokens=specialize,
|
||||
)
|
||||
captured_widths = [4, 2] if specialize else [4]
|
||||
assert [c.kwargs["num_speculative_steps"] for c in forward.call_args_list] == (
|
||||
captured_widths
|
||||
)
|
||||
|
||||
for k in (2, 4, 2, 3):
|
||||
desc, _ = dispatch_cg_and_sync_dp(
|
||||
manager,
|
||||
num_reqs=3,
|
||||
num_tokens=3,
|
||||
uniform_token_count=1,
|
||||
dp_size=1,
|
||||
dp_rank=0,
|
||||
)
|
||||
desc = manager.specialize_spec_tokens(desc, k)
|
||||
captured_k = k if specialize else 4
|
||||
if captured_k in captured_widths:
|
||||
assert desc.cg_mode == CUDAGraphMode.FULL
|
||||
assert desc.num_tokens == 4
|
||||
manager.run_fullgraph(desc)
|
||||
assert replayed_widths[-1] == captured_k
|
||||
else:
|
||||
assert desc.cg_mode == CUDAGraphMode.NONE
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"decode_query_len,capture_sizes",
|
||||
[(1, [1, 2, 4, 8]), (2, [2, 4, 8, 16]), (8, [8, 16, 32, 64])],
|
||||
|
||||
@@ -374,19 +374,22 @@ def test_run_model_reuses_tensor_return_for_mtp(monkeypatch):
|
||||
(
|
||||
"method_name",
|
||||
"cg_mode",
|
||||
"num_speculative_steps",
|
||||
"expected_eager_calls",
|
||||
"expected_graph_replays",
|
||||
),
|
||||
[
|
||||
("_multi_step_decode", CUDAGraphMode.NONE, 3, 0),
|
||||
("_multi_step_decode", CUDAGraphMode.FULL, 0, 3),
|
||||
("_fused_multi_step_decode", CUDAGraphMode.NONE, 3, 0),
|
||||
("_fused_multi_step_decode", CUDAGraphMode.FULL, 0, 1),
|
||||
("_multi_step_decode", CUDAGraphMode.NONE, 4, 3, 0),
|
||||
("_multi_step_decode", CUDAGraphMode.FULL, 4, 0, 3),
|
||||
("_fused_multi_step_decode", CUDAGraphMode.NONE, 4, 3, 0),
|
||||
("_fused_multi_step_decode", CUDAGraphMode.FULL, 4, 0, 1),
|
||||
("_multi_step_decode", CUDAGraphMode.FULL, 3, 0, 2),
|
||||
],
|
||||
)
|
||||
def test_multi_step_decode_replays_captured_graph_as_expected(
|
||||
method_name,
|
||||
cg_mode,
|
||||
num_speculative_steps,
|
||||
expected_eager_calls,
|
||||
expected_graph_replays,
|
||||
):
|
||||
@@ -416,6 +419,7 @@ def test_multi_step_decode_replays_captured_graph_as_expected(
|
||||
batch_desc=batch_desc,
|
||||
seq_lens_cpu_upper_bound=None,
|
||||
num_tokens_across_dp=None,
|
||||
num_speculative_steps=num_speculative_steps,
|
||||
)
|
||||
|
||||
assert generate_draft.call_count == expected_eager_calls
|
||||
|
||||
@@ -46,6 +46,7 @@ def test_non_last_pp_rank_uses_global_batch_for_sample_feedback():
|
||||
finished_req_ids=set(),
|
||||
ec_connector_output=None,
|
||||
cudagraph_stats=None,
|
||||
num_spec_tokens_to_schedule=0,
|
||||
)
|
||||
|
||||
runner.sample_tokens(None)
|
||||
|
||||
@@ -190,6 +190,7 @@ def test_v2_sample_tokens_runs_eplb_on_non_last_pp_rank(monkeypatch):
|
||||
ec_connector_output=None,
|
||||
routed_experts=None,
|
||||
cudagraph_stats=None,
|
||||
num_spec_tokens_to_schedule=0,
|
||||
)
|
||||
runner.req_states = SimpleNamespace()
|
||||
|
||||
|
||||
@@ -116,9 +116,14 @@ class _StepRecorder:
|
||||
def __init__(self) -> None:
|
||||
# (blocks held per group, num_computed_tokens, num_scheduled_tokens)
|
||||
self.steps: list[tuple[list[int], int, int]] = []
|
||||
self.num_spec_tokens_to_schedule: list[int] = []
|
||||
self._held: dict[str, list[int]] = {}
|
||||
|
||||
def execute_model(self, scheduler_output) -> None:
|
||||
if scheduler_output.total_num_scheduled_tokens:
|
||||
self.num_spec_tokens_to_schedule.append(
|
||||
scheduler_output.num_spec_tokens_to_schedule
|
||||
)
|
||||
for new_req in scheduler_output.scheduled_new_reqs:
|
||||
self._held[new_req.req_id] = [len(ids) for ids in new_req.block_ids]
|
||||
self._record(new_req.req_id, new_req.num_computed_tokens, scheduler_output)
|
||||
@@ -161,7 +166,7 @@ def _assert_covers_lookahead(
|
||||
|
||||
# 0 covers eagle / MTP / draft models, 1 covers DFlash's extra in-fill query.
|
||||
@pytest.mark.parametrize("extra_lookahead", [0, 1])
|
||||
@pytest.mark.parametrize("num_spec_steps", [2, 3, 5, 7])
|
||||
@pytest.mark.parametrize("num_spec_steps", [1, 2, 3, 5, 7])
|
||||
def test_warmup_kernels_reserves_lookahead_blocks(num_spec_steps, extra_lookahead):
|
||||
num_lookahead_tokens = num_spec_steps + extra_lookahead
|
||||
recorder = _StepRecorder()
|
||||
@@ -173,6 +178,7 @@ def test_warmup_kernels_reserves_lookahead_blocks(num_spec_steps, extra_lookahea
|
||||
)
|
||||
|
||||
_assert_covers_lookahead(recorder.steps, num_lookahead_tokens)
|
||||
assert set(recorder.num_spec_tokens_to_schedule) == {num_spec_steps}
|
||||
|
||||
|
||||
def test_mixed_warmup_reserves_lookahead_blocks():
|
||||
@@ -187,6 +193,7 @@ def test_mixed_warmup_reserves_lookahead_blocks():
|
||||
)
|
||||
|
||||
_assert_covers_lookahead(recorder.steps, num_lookahead_tokens)
|
||||
assert set(recorder.num_spec_tokens_to_schedule) == {NUM_SPEC_STEPS}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mamba_cache_mode", ["none", "align"])
|
||||
|
||||
Reference in New Issue
Block a user