[V1][Metrics] Add request_success_total counter, labelled with finish reason (#12579)

Signed-off-by: Mark McLoughlin <markmc@redhat.com>
This commit is contained in:
Mark McLoughlin
2025-02-04 19:46:54 -05:00
committed by GitHub
parent 18016a5e62
commit 233df6f5c4
7 changed files with 66 additions and 27 deletions
+19 -2
View File
@@ -15,6 +15,23 @@ if TYPE_CHECKING:
from vllm.sampling_params import SamplingParams
class RequestFinishedReason(enum.IntEnum):
"""
Reason a request finished - stop, length, or abort.
stop - a stop string was emitted
length - max_tokens was consumed, or max_model_len was reached
abort - aborted for another reason
"""
STOP = 0
LENGTH = 1
ABORT = 2
def __str__(self):
return self.name.lower()
@dataclass
class EngineCoreRequest:
@@ -45,7 +62,7 @@ class EngineCoreOutput(
request_id: str
new_token_ids: List[int]
finished: bool
finish_reason: Optional[str] = None
finish_reason: Optional[RequestFinishedReason] = None
stop_reason: Union[int, str, None] = None
@@ -56,7 +73,7 @@ class EngineCoreOutputs(
gc=False): # type: ignore[call-arg]
#NOTE(Nick): We could consider ways to make this more compact,
# e.g. columnwise layout and using an int enum for finish/stop reason
# e.g. columnwise layout
# [num_reqs]
outputs: List[EngineCoreOutput]
+5 -4
View File
@@ -8,7 +8,8 @@ from vllm.logger import init_logger
from vllm.sampling_params import RequestOutputKind
from vllm.transformers_utils.detokenizer_utils import (
AnyTokenizer, convert_prompt_ids_to_tokens, detokenize_incrementally)
from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest
from vllm.v1.engine import (EngineCoreOutput, EngineCoreRequest,
RequestFinishedReason)
logger = init_logger(__name__)
@@ -18,7 +19,7 @@ class DetokenizerOutput:
output_text: str
token_ids: List[int]
finished: bool
finish_reason: Optional[str] = None
finish_reason: Optional[RequestFinishedReason] = None
stop_reason: Union[int, str, None] = None
@@ -147,13 +148,13 @@ class IncrementalDetokenizer:
stop_str, truncate_to = stop
if truncate_to != -1:
self.output_text = self.output_text[:truncate_to]
finish_reason = "stop" # TODO: use constant
finish_reason = RequestFinishedReason.STOP
stop_reason = stop_str
# TODO: handle stop_token_ids here too?
# 3) Update the RequestOutput object with the new text.
finished = bool(finish_reason)
finished = finish_reason is not None
if self.output_kind == RequestOutputKind.FINAL_ONLY \
and not finished:
return None
+12 -10
View File
@@ -161,8 +161,10 @@ class OutputProcessor:
engine_core_output)
# 3) Create and handle RequestOutput objects.
if request_output := self._make_request_output(
req_state, detokenizer_output):
if detokenizer_output is not None:
request_output = self._make_request_output(
req_state, detokenizer_output)
if req_state.queue is not None:
# AsyncLLM: put into queue for handling by generate().
req_state.queue.put_nowait(request_output)
@@ -172,6 +174,8 @@ class OutputProcessor:
# Free completed requests.
if request_output.finished:
assert detokenizer_output.finish_reason is not None
self.request_states.pop(req_id)
if not engine_core_output.finished:
# If req not finished in EngineCore, but Detokenizer
@@ -180,7 +184,8 @@ class OutputProcessor:
# Track per-request stats
iteration_stats.update_from_finished_request(
request_output, req_state.stats)
detokenizer_output.finish_reason, request_output,
req_state.stats)
return OutputProcessorOutput(
request_outputs=request_outputs,
@@ -191,12 +196,8 @@ class OutputProcessor:
@staticmethod
def _make_request_output(
request_state: RequestState,
detokenizer_output: Optional[DetokenizerOutput],
) -> Optional[RequestOutput]:
if detokenizer_output is None:
return None
detokenizer_output: DetokenizerOutput,
) -> RequestOutput:
request_output = RequestOutput.new(
request_state.request_id,
request_state.prompt,
@@ -207,7 +208,8 @@ class OutputProcessor:
)
if detokenizer_output.finished:
completion_output = request_output.outputs[0]
completion_output.finish_reason = detokenizer_output.finish_reason
completion_output.finish_reason = str(
detokenizer_output.finish_reason)
completion_output.stop_reason = detokenizer_output.stop_reason
return request_output