[V1][Metrics] Add per-request prompt/generation_tokens histograms (#12516)

Signed-off-by: Mark McLoughlin <markmc@redhat.com>
This commit is contained in:
Mark McLoughlin
2025-01-28 22:07:22 +00:00
committed by GitHub
parent f26d790718
commit c386c43ca3
5 changed files with 103 additions and 15 deletions
+9 -2
View File
@@ -8,7 +8,7 @@ from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest
from vllm.v1.engine.detokenizer import (DetokenizerOutput,
IncrementalDetokenizer)
from vllm.v1.metrics.stats import IterationStats
from vllm.v1.metrics.stats import IterationStats, RequestStateStats
@dataclass
@@ -37,6 +37,8 @@ class RequestState:
self.is_prefilling = True
self.queue = queue
self.stats = RequestStateStats()
@classmethod
def from_new_request(
cls,
@@ -146,7 +148,8 @@ class OutputProcessor:
# 1) Compute stats for this iteration.
iteration_stats.update_from_output(engine_core_output,
req_state.is_prefilling,
req_state.prompt_len)
req_state.prompt_len,
req_state.stats)
req_state.is_prefilling = False
# 2) Detokenize the token ids into text.
@@ -171,6 +174,10 @@ class OutputProcessor:
# detected stop string, abort needed in EngineCore.
reqs_to_abort.append(req_id)
# Track per-request stats
iteration_stats.update_from_finished_request(
request_output, req_state.stats)
return OutputProcessorOutput(
request_outputs=request_outputs,
reqs_to_abort=reqs_to_abort,