[V1][Metrics] Add per-request prompt/generation_tokens histograms (#12516)
Signed-off-by: Mark McLoughlin <markmc@redhat.com>
This commit is contained in:
@@ -8,7 +8,7 @@ from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
|
||||
from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest
|
||||
from vllm.v1.engine.detokenizer import (DetokenizerOutput,
|
||||
IncrementalDetokenizer)
|
||||
from vllm.v1.metrics.stats import IterationStats
|
||||
from vllm.v1.metrics.stats import IterationStats, RequestStateStats
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -37,6 +37,8 @@ class RequestState:
|
||||
self.is_prefilling = True
|
||||
self.queue = queue
|
||||
|
||||
self.stats = RequestStateStats()
|
||||
|
||||
@classmethod
|
||||
def from_new_request(
|
||||
cls,
|
||||
@@ -146,7 +148,8 @@ class OutputProcessor:
|
||||
# 1) Compute stats for this iteration.
|
||||
iteration_stats.update_from_output(engine_core_output,
|
||||
req_state.is_prefilling,
|
||||
req_state.prompt_len)
|
||||
req_state.prompt_len,
|
||||
req_state.stats)
|
||||
req_state.is_prefilling = False
|
||||
|
||||
# 2) Detokenize the token ids into text.
|
||||
@@ -171,6 +174,10 @@ class OutputProcessor:
|
||||
# detected stop string, abort needed in EngineCore.
|
||||
reqs_to_abort.append(req_id)
|
||||
|
||||
# Track per-request stats
|
||||
iteration_stats.update_from_finished_request(
|
||||
request_output, req_state.stats)
|
||||
|
||||
return OutputProcessorOutput(
|
||||
request_outputs=request_outputs,
|
||||
reqs_to_abort=reqs_to_abort,
|
||||
|
||||
Reference in New Issue
Block a user