[Core] Add max_num_queued_reqs and max_num_queued_tokens for queue size management (#49445)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
@@ -467,6 +467,20 @@ class OutputProcessor:
|
||||
def get_num_unfinished_requests(self):
|
||||
return len(self.request_states)
|
||||
|
||||
def has_request(self, request_id: str) -> bool:
|
||||
return request_id in self.request_states
|
||||
|
||||
def get_num_queued_tokens(self) -> int:
|
||||
"""Total prompt tokens of requests currently in the prefill phase.
|
||||
|
||||
Uses ``prompt_len`` rather than remaining prefill work because the
|
||||
scheduler's ``num_computed_tokens`` is not propagated to the API
|
||||
server until prefill completes. See ``SchedulerConfig`` docs.
|
||||
"""
|
||||
return sum(
|
||||
req.prompt_len for req in self.request_states.values() if req.is_prefilling
|
||||
)
|
||||
|
||||
def has_unfinished_requests(self) -> bool:
|
||||
return len(self.request_states) > 0
|
||||
|
||||
|
||||
Reference in New Issue
Block a user