[Metrics] Add request_id to FinishedRequestStats to enable correlation between metrics and requests (#39710)
Enables external `StatLogger` plugins to correlate per-request metrics with request-level context. Also, this is a pre-requisite for Prometheus exemplars in #30972. Signed-off-by: Csrayz <33659823+Csrayz@users.noreply.github.com>
This commit is contained in:
@@ -26,6 +26,7 @@ def test_prefill_kv_computed_with_cache():
|
||||
# Case 1: With prefix cache (1200 tokens cached)
|
||||
iteration_stats.update_from_finished_request(
|
||||
finish_reason=FinishReason.STOP,
|
||||
request_id="test-req-001",
|
||||
num_prompt_tokens=10000,
|
||||
max_tokens_param=100,
|
||||
req_stats=req_stats,
|
||||
@@ -35,6 +36,7 @@ def test_prefill_kv_computed_with_cache():
|
||||
finished_req = iteration_stats.finished_requests[0]
|
||||
assert finished_req.num_prompt_tokens == 10000
|
||||
assert finished_req.num_cached_tokens == 1200
|
||||
assert finished_req.request_id == "test-req-001"
|
||||
|
||||
# Verify calculation: prefill KV = prompt tokens - cached tokens
|
||||
prefill_kv_computed = finished_req.num_prompt_tokens - max(
|
||||
@@ -55,6 +57,7 @@ def test_prefill_kv_computed_no_cache():
|
||||
# Case 2: No prefix cache
|
||||
iteration_stats.update_from_finished_request(
|
||||
finish_reason=FinishReason.STOP,
|
||||
request_id="test-req-002",
|
||||
num_prompt_tokens=2000,
|
||||
max_tokens_param=100,
|
||||
req_stats=req_stats,
|
||||
@@ -64,6 +67,7 @@ def test_prefill_kv_computed_no_cache():
|
||||
finished_req = iteration_stats.finished_requests[0]
|
||||
assert finished_req.num_prompt_tokens == 2000
|
||||
assert finished_req.num_cached_tokens == 0
|
||||
assert finished_req.request_id == "test-req-002"
|
||||
|
||||
# Verify calculation: prefill KV = full prompt when no cache
|
||||
prefill_kv_computed = finished_req.num_prompt_tokens - max(
|
||||
@@ -84,6 +88,7 @@ def test_prefill_kv_computed_edge_cases():
|
||||
# Case 3: Negative num_cached_tokens (shouldn't happen, but handle gracefully)
|
||||
iteration_stats.update_from_finished_request(
|
||||
finish_reason=FinishReason.STOP,
|
||||
request_id="test-req-003",
|
||||
num_prompt_tokens=100,
|
||||
max_tokens_param=10,
|
||||
req_stats=req_stats,
|
||||
@@ -96,11 +101,13 @@ def test_prefill_kv_computed_edge_cases():
|
||||
finished_req.num_cached_tokens, 0
|
||||
)
|
||||
assert prefill_kv_computed == 100 # Should treat negative as 0
|
||||
assert finished_req.request_id == "test-req-003"
|
||||
|
||||
# Case 4: All tokens cached (shouldn't happen in practice)
|
||||
iteration_stats2 = IterationStats()
|
||||
iteration_stats2.update_from_finished_request(
|
||||
finish_reason=FinishReason.STOP,
|
||||
request_id="test-req-004",
|
||||
num_prompt_tokens=100,
|
||||
max_tokens_param=10,
|
||||
req_stats=req_stats,
|
||||
@@ -112,6 +119,7 @@ def test_prefill_kv_computed_edge_cases():
|
||||
finished_req2.num_cached_tokens, 0
|
||||
)
|
||||
assert prefill_kv_computed2 == 0 # All cached, nothing computed
|
||||
assert finished_req2.request_id == "test-req-004"
|
||||
|
||||
|
||||
def test_prompt_token_stats_all_computed():
|
||||
|
||||
Reference in New Issue
Block a user