[V1][Metrics] Add request_success_total counter, labelled with finish reason (#12579)
Signed-off-by: Mark McLoughlin <markmc@redhat.com>
This commit is contained in:
@@ -15,6 +15,23 @@ if TYPE_CHECKING:
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
|
||||
class RequestFinishedReason(enum.IntEnum):
|
||||
"""
|
||||
Reason a request finished - stop, length, or abort.
|
||||
|
||||
stop - a stop string was emitted
|
||||
length - max_tokens was consumed, or max_model_len was reached
|
||||
abort - aborted for another reason
|
||||
|
||||
"""
|
||||
STOP = 0
|
||||
LENGTH = 1
|
||||
ABORT = 2
|
||||
|
||||
def __str__(self):
|
||||
return self.name.lower()
|
||||
|
||||
|
||||
@dataclass
|
||||
class EngineCoreRequest:
|
||||
|
||||
@@ -45,7 +62,7 @@ class EngineCoreOutput(
|
||||
request_id: str
|
||||
new_token_ids: List[int]
|
||||
finished: bool
|
||||
finish_reason: Optional[str] = None
|
||||
finish_reason: Optional[RequestFinishedReason] = None
|
||||
stop_reason: Union[int, str, None] = None
|
||||
|
||||
|
||||
@@ -56,7 +73,7 @@ class EngineCoreOutputs(
|
||||
gc=False): # type: ignore[call-arg]
|
||||
|
||||
#NOTE(Nick): We could consider ways to make this more compact,
|
||||
# e.g. columnwise layout and using an int enum for finish/stop reason
|
||||
# e.g. columnwise layout
|
||||
|
||||
# [num_reqs]
|
||||
outputs: List[EngineCoreOutput]
|
||||
|
||||
@@ -8,7 +8,8 @@ from vllm.logger import init_logger
|
||||
from vllm.sampling_params import RequestOutputKind
|
||||
from vllm.transformers_utils.detokenizer_utils import (
|
||||
AnyTokenizer, convert_prompt_ids_to_tokens, detokenize_incrementally)
|
||||
from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest
|
||||
from vllm.v1.engine import (EngineCoreOutput, EngineCoreRequest,
|
||||
RequestFinishedReason)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -18,7 +19,7 @@ class DetokenizerOutput:
|
||||
output_text: str
|
||||
token_ids: List[int]
|
||||
finished: bool
|
||||
finish_reason: Optional[str] = None
|
||||
finish_reason: Optional[RequestFinishedReason] = None
|
||||
stop_reason: Union[int, str, None] = None
|
||||
|
||||
|
||||
@@ -147,13 +148,13 @@ class IncrementalDetokenizer:
|
||||
stop_str, truncate_to = stop
|
||||
if truncate_to != -1:
|
||||
self.output_text = self.output_text[:truncate_to]
|
||||
finish_reason = "stop" # TODO: use constant
|
||||
finish_reason = RequestFinishedReason.STOP
|
||||
stop_reason = stop_str
|
||||
|
||||
# TODO: handle stop_token_ids here too?
|
||||
|
||||
# 3) Update the RequestOutput object with the new text.
|
||||
finished = bool(finish_reason)
|
||||
finished = finish_reason is not None
|
||||
if self.output_kind == RequestOutputKind.FINAL_ONLY \
|
||||
and not finished:
|
||||
return None
|
||||
|
||||
@@ -161,8 +161,10 @@ class OutputProcessor:
|
||||
engine_core_output)
|
||||
|
||||
# 3) Create and handle RequestOutput objects.
|
||||
if request_output := self._make_request_output(
|
||||
req_state, detokenizer_output):
|
||||
if detokenizer_output is not None:
|
||||
request_output = self._make_request_output(
|
||||
req_state, detokenizer_output)
|
||||
|
||||
if req_state.queue is not None:
|
||||
# AsyncLLM: put into queue for handling by generate().
|
||||
req_state.queue.put_nowait(request_output)
|
||||
@@ -172,6 +174,8 @@ class OutputProcessor:
|
||||
|
||||
# Free completed requests.
|
||||
if request_output.finished:
|
||||
assert detokenizer_output.finish_reason is not None
|
||||
|
||||
self.request_states.pop(req_id)
|
||||
if not engine_core_output.finished:
|
||||
# If req not finished in EngineCore, but Detokenizer
|
||||
@@ -180,7 +184,8 @@ class OutputProcessor:
|
||||
|
||||
# Track per-request stats
|
||||
iteration_stats.update_from_finished_request(
|
||||
request_output, req_state.stats)
|
||||
detokenizer_output.finish_reason, request_output,
|
||||
req_state.stats)
|
||||
|
||||
return OutputProcessorOutput(
|
||||
request_outputs=request_outputs,
|
||||
@@ -191,12 +196,8 @@ class OutputProcessor:
|
||||
@staticmethod
|
||||
def _make_request_output(
|
||||
request_state: RequestState,
|
||||
detokenizer_output: Optional[DetokenizerOutput],
|
||||
) -> Optional[RequestOutput]:
|
||||
|
||||
if detokenizer_output is None:
|
||||
return None
|
||||
|
||||
detokenizer_output: DetokenizerOutput,
|
||||
) -> RequestOutput:
|
||||
request_output = RequestOutput.new(
|
||||
request_state.request_id,
|
||||
request_state.prompt,
|
||||
@@ -207,7 +208,8 @@ class OutputProcessor:
|
||||
)
|
||||
if detokenizer_output.finished:
|
||||
completion_output = request_output.outputs[0]
|
||||
completion_output.finish_reason = detokenizer_output.finish_reason
|
||||
completion_output.finish_reason = str(
|
||||
detokenizer_output.finish_reason)
|
||||
completion_output.stop_reason = detokenizer_output.stop_reason
|
||||
|
||||
return request_output
|
||||
|
||||
Reference in New Issue
Block a user