[Model Runner V2][Spec Decode] Support spec decode with draft model (#43091)

Signed-off-by: Icey <1790571317@qq.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Xiaoshuang Wang
2026-09-28 19:38:05 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 2f4a975901
commit ef42093a36
6 changed files with 552 additions and 4 deletions
+12 -1
View File
@@ -799,7 +799,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
"skip_attn must only be True for initial memory profiling."
)
# Create a dummy scheduler output.
# Create a dummy scheduler output. Plain draft-model speculation adds
# one correction slot per request during prefill. The scheduler
# accounts for these slots, while dummy runs bypass the scheduler.
# Adjust the dummy token count only when the expanded draft batch
# would exceed the token budget.
num_reqs = min(num_tokens, self.max_num_reqs)
if uniform_decode:
# HACK(lucas): for now since the worker is shared between MRV1 and MRV2,
@@ -813,8 +817,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
num_reqs = min(
num_reqs, self.max_num_tokens // self.speculator.num_query_per_req
)
num_tokens = min(
num_tokens,
self.max_num_tokens
- num_reqs * self.speculator.num_extra_query_per_req,
)
# Distribute the remainder evenly so no dummy request exceeds
# ceil(num_tokens / num_reqs) <= max_model_len tokens.
num_tokens_per_request = [
num_tokens // num_reqs + (i >= num_reqs - num_tokens % num_reqs)
for i in range(num_reqs)