[Metrics] Add request_id to FinishedRequestStats to enable correlation between metrics and requests (#39710)
Enables external `StatLogger` plugins to correlate per-request metrics with request-level context. Also, this is a pre-requisite for Prometheus exemplars in #30972. Signed-off-by: Csrayz <33659823+Csrayz@users.noreply.github.com>
This commit is contained in:
@@ -799,6 +799,7 @@ class OutputProcessor:
|
||||
assert req_state.stats is not None
|
||||
iteration_stats.update_from_finished_request(
|
||||
finish_reason=finish_reason,
|
||||
request_id=req_state.external_req_id,
|
||||
num_prompt_tokens=req_state.prompt_len,
|
||||
max_tokens_param=req_state.max_tokens_param,
|
||||
req_stats=req_state.stats,
|
||||
|
||||
@@ -225,6 +225,7 @@ class FinishedRequestStats:
|
||||
"""Stats associated with a finished request."""
|
||||
|
||||
finish_reason: "FinishReason"
|
||||
request_id: str | None = None
|
||||
e2e_latency: float = 0.0
|
||||
num_prompt_tokens: int = 0
|
||||
num_generation_tokens: int = 0
|
||||
@@ -427,6 +428,7 @@ class IterationStats:
|
||||
def update_from_finished_request(
|
||||
self,
|
||||
finish_reason: "FinishReason",
|
||||
request_id: str,
|
||||
num_prompt_tokens: int,
|
||||
max_tokens_param: int | None,
|
||||
req_stats: RequestStateStats,
|
||||
@@ -458,6 +460,7 @@ class IterationStats:
|
||||
|
||||
finished_req = FinishedRequestStats(
|
||||
finish_reason=finish_reason,
|
||||
request_id=request_id,
|
||||
e2e_latency=e2e_latency,
|
||||
num_prompt_tokens=num_prompt_tokens,
|
||||
num_generation_tokens=req_stats.num_generation_tokens,
|
||||
|
||||
Reference in New Issue
Block a user