[Core] Enable decode of context length equal to max model length (#26168)

Signed-off-by: Yannick Schnider <yannick.schnider1@ibm.com>
This commit is contained in:
Yannick Schnider
2025-10-04 09:59:26 +00:00
committed by GitHub
parent d0df145c2a
commit f05fea1f5e
4 changed files with 32 additions and 15 deletions
+1 -1
View File
@@ -224,7 +224,7 @@ class Scheduler(SchedulerInterface):
# This is necessary when using spec decoding.
num_new_tokens = min(
num_new_tokens,
self.max_model_len - 1 - request.num_computed_tokens)
self.max_model_len - request.num_computed_tokens)
# Schedule encoder inputs.
encoder_inputs_to_schedule = None
+1 -1
View File
@@ -43,7 +43,7 @@ def remove_all(lst: list, items_to_remove: set) -> list:
def check_stop(request: Request,
max_model_len: int,
pooler_output: Optional[torch.Tensor] = None) -> bool:
if (request.num_tokens >= max_model_len
if (request.num_tokens > max_model_len
or request.num_output_tokens >= request.max_tokens):
request.status = RequestStatus.FINISHED_LENGTH_CAPPED
return True