[Core] Add max_num_queued_reqs and max_num_queued_tokens for queue size management (#49445)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
Nicolò Lucchesi
2026-08-31 14:26:49 +00:00
committed by GitHub
parent 76ff0cdff2
commit dafbef15a1
18 changed files with 644 additions and 35 deletions
+14
View File
@@ -467,6 +467,20 @@ class OutputProcessor:
def get_num_unfinished_requests(self):
return len(self.request_states)
def has_request(self, request_id: str) -> bool:
return request_id in self.request_states
def get_num_queued_tokens(self) -> int:
"""Total prompt tokens of requests currently in the prefill phase.
Uses ``prompt_len`` rather than remaining prefill work because the
scheduler's ``num_computed_tokens`` is not propagated to the API
server until prefill completes. See ``SchedulerConfig`` docs.
"""
return sum(
req.prompt_len for req in self.request_states.values() if req.is_prefilling
)
def has_unfinished_requests(self) -> bool:
return len(self.request_states) > 0