[Bugfix][MRV2][Spec Decode] Honor dynamic K in autoregressive speculators, 1.2~1.3x kernel perf improvement (#57053)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-10-05 14:54:11 -04:00
committed by GitHub
parent c32495d58f
commit 74c5cbcd7b
15 changed files with 190 additions and 18 deletions
+79 -1
View File
@@ -20,8 +20,12 @@ from vllm.distributed.device_communicators import pynccl_allocator
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.worker.gpu import cudagraph_utils as gpu_cudagraph_utils
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
from vllm.v1.worker.gpu.input_batch import InputBuffers
from vllm.v1.worker.gpu.pcp_manager import PCPManager
from vllm.v1.worker.gpu.spec_decode.autoregressive import (
cudagraph_utils as spec_cudagraph_utils,
)
from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import (
SpeculatorCudaGraphManager,
)
@@ -226,8 +230,10 @@ def test_speculator_capture_preserves_decode_query_bounds(
)
manager = SpeculatorCudaGraphManager.__new__(SpeculatorCudaGraphManager)
manager.max_num_reqs = num_reqs
manager.decode_query_len = uniform_token_count or max_query_len or num_tokens
manager.vllm_config = SimpleNamespace(
parallel_config=SimpleNamespace(data_parallel_size=1)
parallel_config=SimpleNamespace(data_parallel_size=1),
num_speculative_tokens=3,
)
buffers = InputBuffers(num_reqs, num_tokens, torch.device("cpu"))
block_tables = MagicMock()
@@ -488,6 +494,78 @@ def test_dynamic_spec_decode_shared_token_count_stays_reachable(monkeypatch):
), desc
@pytest.mark.parametrize("specialize", [False, True])
def test_speculator_replays_graph_for_runtime_width(monkeypatch, specialize):
"""Equal batch shapes must replay the graph captured for the selected K."""
monkeypatch.setattr(
gpu_cudagraph_utils,
"get_pp_group",
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
)
monkeypatch.setattr(
gpu_cudagraph_utils.current_platform, "get_global_graph_pool", lambda: object()
)
monkeypatch.setattr(gpu_cudagraph_utils, "get_offloader", MagicMock)
monkeypatch.setattr(
spec_cudagraph_utils, "prepare_inputs_to_capture", lambda *a, **kw: ({}, {})
)
forward = MagicMock()
replayed_widths: list[int] = []
def capture(manager, create_forward_fn, progress_bar_desc):
for desc in manager._capture_descs[CUDAGraphMode.FULL]:
create_forward_fn(desc, warmup=False)(CUDAGraphMode.NONE)
k = forward.call_args.kwargs["num_speculative_steps"]
graph = MagicMock()
graph.replay.side_effect = lambda k=k: replayed_widths.append(k)
manager.graphs[desc] = graph
manager._graphs_captured = True
monkeypatch.setattr(gpu_cudagraph_utils.CudaGraphManager, "capture", capture)
manager = spec_cudagraph_utils.SpeculatorCudaGraphManager(
_create_decode_vllm_config(
capture_sizes=[4],
num_speculative_tokens=4,
dynamic_spec_schedule=[(1, 2, 4), (3, 4, 2), (5, 6, 1), (7, 8, 0)],
),
device=torch.device("cpu"),
cudagraph_mode=CUDAGraphMode.FULL_DECODE_ONLY,
decode_query_len=1,
)
manager.capture(
forward,
MagicMock(),
MagicMock(),
MagicMock(),
[],
MagicMock(),
specialize_spec_tokens=specialize,
)
captured_widths = [4, 2] if specialize else [4]
assert [c.kwargs["num_speculative_steps"] for c in forward.call_args_list] == (
captured_widths
)
for k in (2, 4, 2, 3):
desc, _ = dispatch_cg_and_sync_dp(
manager,
num_reqs=3,
num_tokens=3,
uniform_token_count=1,
dp_size=1,
dp_rank=0,
)
desc = manager.specialize_spec_tokens(desc, k)
captured_k = k if specialize else 4
if captured_k in captured_widths:
assert desc.cg_mode == CUDAGraphMode.FULL
assert desc.num_tokens == 4
manager.run_fullgraph(desc)
assert replayed_widths[-1] == captured_k
else:
assert desc.cg_mode == CUDAGraphMode.NONE
@pytest.mark.parametrize(
"decode_query_len,capture_sizes",
[(1, [1, 2, 4, 8]), (2, [2, 4, 8, 16]), (8, [8, 16, 32, 64])],