[V1][Metrics] Add request_success_total counter, labelled with finish reason (#12579)
Signed-off-by: Mark McLoughlin <markmc@redhat.com>
This commit is contained in:
@@ -161,8 +161,10 @@ class OutputProcessor:
|
||||
engine_core_output)
|
||||
|
||||
# 3) Create and handle RequestOutput objects.
|
||||
if request_output := self._make_request_output(
|
||||
req_state, detokenizer_output):
|
||||
if detokenizer_output is not None:
|
||||
request_output = self._make_request_output(
|
||||
req_state, detokenizer_output)
|
||||
|
||||
if req_state.queue is not None:
|
||||
# AsyncLLM: put into queue for handling by generate().
|
||||
req_state.queue.put_nowait(request_output)
|
||||
@@ -172,6 +174,8 @@ class OutputProcessor:
|
||||
|
||||
# Free completed requests.
|
||||
if request_output.finished:
|
||||
assert detokenizer_output.finish_reason is not None
|
||||
|
||||
self.request_states.pop(req_id)
|
||||
if not engine_core_output.finished:
|
||||
# If req not finished in EngineCore, but Detokenizer
|
||||
@@ -180,7 +184,8 @@ class OutputProcessor:
|
||||
|
||||
# Track per-request stats
|
||||
iteration_stats.update_from_finished_request(
|
||||
request_output, req_state.stats)
|
||||
detokenizer_output.finish_reason, request_output,
|
||||
req_state.stats)
|
||||
|
||||
return OutputProcessorOutput(
|
||||
request_outputs=request_outputs,
|
||||
@@ -191,12 +196,8 @@ class OutputProcessor:
|
||||
@staticmethod
|
||||
def _make_request_output(
|
||||
request_state: RequestState,
|
||||
detokenizer_output: Optional[DetokenizerOutput],
|
||||
) -> Optional[RequestOutput]:
|
||||
|
||||
if detokenizer_output is None:
|
||||
return None
|
||||
|
||||
detokenizer_output: DetokenizerOutput,
|
||||
) -> RequestOutput:
|
||||
request_output = RequestOutput.new(
|
||||
request_state.request_id,
|
||||
request_state.prompt,
|
||||
@@ -207,7 +208,8 @@ class OutputProcessor:
|
||||
)
|
||||
if detokenizer_output.finished:
|
||||
completion_output = request_output.outputs[0]
|
||||
completion_output.finish_reason = detokenizer_output.finish_reason
|
||||
completion_output.finish_reason = str(
|
||||
detokenizer_output.finish_reason)
|
||||
completion_output.stop_reason = detokenizer_output.stop_reason
|
||||
|
||||
return request_output
|
||||
|
||||
Reference in New Issue
Block a user