[Model Runner V2][Spec Decode] Support spec decode with draft model (#43091)
Signed-off-by: Icey <1790571317@qq.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
2f4a975901
commit
ef42093a36
@@ -799,7 +799,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
"skip_attn must only be True for initial memory profiling."
|
||||
)
|
||||
|
||||
# Create a dummy scheduler output.
|
||||
# Create a dummy scheduler output. Plain draft-model speculation adds
|
||||
# one correction slot per request during prefill. The scheduler
|
||||
# accounts for these slots, while dummy runs bypass the scheduler.
|
||||
# Adjust the dummy token count only when the expanded draft batch
|
||||
# would exceed the token budget.
|
||||
num_reqs = min(num_tokens, self.max_num_reqs)
|
||||
if uniform_decode:
|
||||
# HACK(lucas): for now since the worker is shared between MRV1 and MRV2,
|
||||
@@ -813,8 +817,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
num_reqs = min(
|
||||
num_reqs, self.max_num_tokens // self.speculator.num_query_per_req
|
||||
)
|
||||
num_tokens = min(
|
||||
num_tokens,
|
||||
self.max_num_tokens
|
||||
- num_reqs * self.speculator.num_extra_query_per_req,
|
||||
)
|
||||
|
||||
# Distribute the remainder evenly so no dummy request exceeds
|
||||
# ceil(num_tokens / num_reqs) <= max_model_len tokens.
|
||||
|
||||
num_tokens_per_request = [
|
||||
num_tokens // num_reqs + (i >= num_reqs - num_tokens % num_reqs)
|
||||
for i in range(num_reqs)
|
||||
|
||||
Reference in New Issue
Block a user