[V1][Spec Decode] Handle draft tokens beyond max_model_len (#16087)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
This commit is contained in:
Woosuk Kwon
2025-04-21 12:38:50 -07:00
committed by GitHub
parent 299ebb62b2
commit 3a0fba5cf4
7 changed files with 137 additions and 15 deletions
+7 -1
View File
@@ -1271,7 +1271,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
draft_token_ids.append([])
continue
# Skip requests that require top-p, top-k, etc.
# Skip requests that require sampling parameters that are not
# supported with speculative decoding.
req_id = self.input_batch.req_ids[i]
if not is_spec_decode_supported(req_id, self.input_batch):
draft_token_ids.append([])
@@ -1280,6 +1281,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# Add sampled_token_ids to token_ids_cpu.
start_idx = self.input_batch.num_tokens_no_spec[i]
end_idx = start_idx + num_sampled_ids
if end_idx >= self.max_model_len:
# Skip requests that have already reached the max model length.
draft_token_ids.append([])
continue
self.input_batch.token_ids_cpu[i, start_idx:end_idx] = sampled_ids
drafter_output = self.drafter.propose(
self.input_batch.token_ids_cpu[i, :end_idx])