[V1][Metrics] Add request_success_total counter, labelled with finish reason (#12579)

Signed-off-by: Mark McLoughlin <markmc@redhat.com>
This commit is contained in:
Mark McLoughlin
2025-02-04 19:46:54 -05:00
committed by GitHub
parent 18016a5e62
commit 233df6f5c4
7 changed files with 66 additions and 27 deletions
+12 -10
View File
@@ -161,8 +161,10 @@ class OutputProcessor:
engine_core_output)
# 3) Create and handle RequestOutput objects.
if request_output := self._make_request_output(
req_state, detokenizer_output):
if detokenizer_output is not None:
request_output = self._make_request_output(
req_state, detokenizer_output)
if req_state.queue is not None:
# AsyncLLM: put into queue for handling by generate().
req_state.queue.put_nowait(request_output)
@@ -172,6 +174,8 @@ class OutputProcessor:
# Free completed requests.
if request_output.finished:
assert detokenizer_output.finish_reason is not None
self.request_states.pop(req_id)
if not engine_core_output.finished:
# If req not finished in EngineCore, but Detokenizer
@@ -180,7 +184,8 @@ class OutputProcessor:
# Track per-request stats
iteration_stats.update_from_finished_request(
request_output, req_state.stats)
detokenizer_output.finish_reason, request_output,
req_state.stats)
return OutputProcessorOutput(
request_outputs=request_outputs,
@@ -191,12 +196,8 @@ class OutputProcessor:
@staticmethod
def _make_request_output(
request_state: RequestState,
detokenizer_output: Optional[DetokenizerOutput],
) -> Optional[RequestOutput]:
if detokenizer_output is None:
return None
detokenizer_output: DetokenizerOutput,
) -> RequestOutput:
request_output = RequestOutput.new(
request_state.request_id,
request_state.prompt,
@@ -207,7 +208,8 @@ class OutputProcessor:
)
if detokenizer_output.finished:
completion_output = request_output.outputs[0]
completion_output.finish_reason = detokenizer_output.finish_reason
completion_output.finish_reason = str(
detokenizer_output.finish_reason)
completion_output.stop_reason = detokenizer_output.stop_reason
return request_output