[Bugfix][MRV2][Spec Decode] Honor dynamic K in autoregressive speculators, 1.2~1.3x kernel perf improvement (#57053)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -116,9 +116,14 @@ class _StepRecorder:
|
||||
def __init__(self) -> None:
|
||||
# (blocks held per group, num_computed_tokens, num_scheduled_tokens)
|
||||
self.steps: list[tuple[list[int], int, int]] = []
|
||||
self.num_spec_tokens_to_schedule: list[int] = []
|
||||
self._held: dict[str, list[int]] = {}
|
||||
|
||||
def execute_model(self, scheduler_output) -> None:
|
||||
if scheduler_output.total_num_scheduled_tokens:
|
||||
self.num_spec_tokens_to_schedule.append(
|
||||
scheduler_output.num_spec_tokens_to_schedule
|
||||
)
|
||||
for new_req in scheduler_output.scheduled_new_reqs:
|
||||
self._held[new_req.req_id] = [len(ids) for ids in new_req.block_ids]
|
||||
self._record(new_req.req_id, new_req.num_computed_tokens, scheduler_output)
|
||||
@@ -161,7 +166,7 @@ def _assert_covers_lookahead(
|
||||
|
||||
# 0 covers eagle / MTP / draft models, 1 covers DFlash's extra in-fill query.
|
||||
@pytest.mark.parametrize("extra_lookahead", [0, 1])
|
||||
@pytest.mark.parametrize("num_spec_steps", [2, 3, 5, 7])
|
||||
@pytest.mark.parametrize("num_spec_steps", [1, 2, 3, 5, 7])
|
||||
def test_warmup_kernels_reserves_lookahead_blocks(num_spec_steps, extra_lookahead):
|
||||
num_lookahead_tokens = num_spec_steps + extra_lookahead
|
||||
recorder = _StepRecorder()
|
||||
@@ -173,6 +178,7 @@ def test_warmup_kernels_reserves_lookahead_blocks(num_spec_steps, extra_lookahea
|
||||
)
|
||||
|
||||
_assert_covers_lookahead(recorder.steps, num_lookahead_tokens)
|
||||
assert set(recorder.num_spec_tokens_to_schedule) == {num_spec_steps}
|
||||
|
||||
|
||||
def test_mixed_warmup_reserves_lookahead_blocks():
|
||||
@@ -187,6 +193,7 @@ def test_mixed_warmup_reserves_lookahead_blocks():
|
||||
)
|
||||
|
||||
_assert_covers_lookahead(recorder.steps, num_lookahead_tokens)
|
||||
assert set(recorder.num_spec_tokens_to_schedule) == {NUM_SPEC_STEPS}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mamba_cache_mode", ["none", "align"])
|
||||
|
||||
Reference in New Issue
Block a user