[V1][Metrics] Add TTFT and TPOT histograms (#12530)

Signed-off-by: Mark McLoughlin <markmc@redhat.com>
This commit is contained in:
Mark McLoughlin
2025-01-29 04:11:16 +00:00
committed by GitHub
parent dd6a3a02cb
commit 46fb056749
4 changed files with 45 additions and 1 deletions
+3 -1
View File
@@ -27,6 +27,7 @@ class RequestState:
prompt: Optional[str],
prompt_token_ids: List[int],
detokenizer: IncrementalDetokenizer,
arrival_time: float,
queue: Optional[asyncio.Queue[RequestOutput]],
):
self.request_id = request_id
@@ -37,7 +38,7 @@ class RequestState:
self.is_prefilling = True
self.queue = queue
self.stats = RequestStateStats()
self.stats = RequestStateStats(last_token_time=arrival_time)
@classmethod
def from_new_request(
@@ -54,6 +55,7 @@ class RequestState:
tokenizer=tokenizer,
request=request,
),
arrival_time=request.arrival_time,
queue=queue,
)