[Bugfix][Scheduler] Refresh max tokens for streaming continuations (#57676)
Signed-off-by: 0xsensei <prblmslvr.aditya@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
@@ -1648,6 +1648,7 @@ class Scheduler(SchedulerInterface):
|
||||
session.num_prompt_tokens = len(session.prompt_token_ids)
|
||||
session.arrival_time = update.arrival_time
|
||||
session.sampling_params = update.sampling_params
|
||||
session.max_tokens = update.max_tokens
|
||||
if session.status == RequestStatus.WAITING_FOR_STREAMING_REQ:
|
||||
self.num_waiting_for_streaming_input -= 1
|
||||
session.status = RequestStatus.WAITING
|
||||
|
||||
Reference in New Issue
Block a user