[Bugfix][MRV2][Spec Decode] Honor dynamic K in autoregressive speculators, 1.2~1.3x kernel perf improvement (#57053)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-10-05 14:54:11 -04:00
committed by GitHub
parent c32495d58f
commit 74c5cbcd7b
15 changed files with 190 additions and 18 deletions
@@ -374,19 +374,22 @@ def test_run_model_reuses_tensor_return_for_mtp(monkeypatch):
(
"method_name",
"cg_mode",
"num_speculative_steps",
"expected_eager_calls",
"expected_graph_replays",
),
[
("_multi_step_decode", CUDAGraphMode.NONE, 3, 0),
("_multi_step_decode", CUDAGraphMode.FULL, 0, 3),
("_fused_multi_step_decode", CUDAGraphMode.NONE, 3, 0),
("_fused_multi_step_decode", CUDAGraphMode.FULL, 0, 1),
("_multi_step_decode", CUDAGraphMode.NONE, 4, 3, 0),
("_multi_step_decode", CUDAGraphMode.FULL, 4, 0, 3),
("_fused_multi_step_decode", CUDAGraphMode.NONE, 4, 3, 0),
("_fused_multi_step_decode", CUDAGraphMode.FULL, 4, 0, 1),
("_multi_step_decode", CUDAGraphMode.FULL, 3, 0, 2),
],
)
def test_multi_step_decode_replays_captured_graph_as_expected(
method_name,
cg_mode,
num_speculative_steps,
expected_eager_calls,
expected_graph_replays,
):
@@ -416,6 +419,7 @@ def test_multi_step_decode_replays_captured_graph_as_expected(
batch_desc=batch_desc,
seq_lens_cpu_upper_bound=None,
num_tokens_across_dp=None,
num_speculative_steps=num_speculative_steps,
)
assert generate_draft.call_count == expected_eager_calls
@@ -46,6 +46,7 @@ def test_non_last_pp_rank_uses_global_batch_for_sample_feedback():
finished_req_ids=set(),
ec_connector_output=None,
cudagraph_stats=None,
num_spec_tokens_to_schedule=0,
)
runner.sample_tokens(None)
@@ -190,6 +190,7 @@ def test_v2_sample_tokens_runs_eplb_on_non_last_pp_rank(monkeypatch):
ec_connector_output=None,
routed_experts=None,
cudagraph_stats=None,
num_spec_tokens_to_schedule=0,
)
runner.req_states = SimpleNamespace()
+8 -1
View File
@@ -116,9 +116,14 @@ class _StepRecorder:
def __init__(self) -> None:
# (blocks held per group, num_computed_tokens, num_scheduled_tokens)
self.steps: list[tuple[list[int], int, int]] = []
self.num_spec_tokens_to_schedule: list[int] = []
self._held: dict[str, list[int]] = {}
def execute_model(self, scheduler_output) -> None:
if scheduler_output.total_num_scheduled_tokens:
self.num_spec_tokens_to_schedule.append(
scheduler_output.num_spec_tokens_to_schedule
)
for new_req in scheduler_output.scheduled_new_reqs:
self._held[new_req.req_id] = [len(ids) for ids in new_req.block_ids]
self._record(new_req.req_id, new_req.num_computed_tokens, scheduler_output)
@@ -161,7 +166,7 @@ def _assert_covers_lookahead(
# 0 covers eagle / MTP / draft models, 1 covers DFlash's extra in-fill query.
@pytest.mark.parametrize("extra_lookahead", [0, 1])
@pytest.mark.parametrize("num_spec_steps", [2, 3, 5, 7])
@pytest.mark.parametrize("num_spec_steps", [1, 2, 3, 5, 7])
def test_warmup_kernels_reserves_lookahead_blocks(num_spec_steps, extra_lookahead):
num_lookahead_tokens = num_spec_steps + extra_lookahead
recorder = _StepRecorder()
@@ -173,6 +178,7 @@ def test_warmup_kernels_reserves_lookahead_blocks(num_spec_steps, extra_lookahea
)
_assert_covers_lookahead(recorder.steps, num_lookahead_tokens)
assert set(recorder.num_spec_tokens_to_schedule) == {num_spec_steps}
def test_mixed_warmup_reserves_lookahead_blocks():
@@ -187,6 +193,7 @@ def test_mixed_warmup_reserves_lookahead_blocks():
)
_assert_covers_lookahead(recorder.steps, num_lookahead_tokens)
assert set(recorder.num_spec_tokens_to_schedule) == {NUM_SPEC_STEPS}
@pytest.mark.parametrize("mamba_cache_mode", ["none", "align"])