[Bugfix][Benchmark] Make streaming TTFT/E2E latency accounting consistent across endpoints (#55508)

Signed-off-by: surajm20061998 <surajm20061998@gmail.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Suraj
2026-09-13 05:20:54 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 2671fedfc7
commit 93e051e290
3 changed files with 505 additions and 23 deletions
+8 -6
View File
@@ -235,8 +235,7 @@ async def async_request_openai_completions(
# First token
if not first_chunk_received:
first_chunk_received = True
ttft = time.perf_counter() - st
output.ttft = ttft
output.ttft = timestamp - st
# Decoding phase
else:
@@ -415,13 +414,14 @@ async def async_request_openai_chat_completions(
output.itl.append(timestamp - most_recent_timestamp)
generated_text += content or ""
# Only token chunks advance the request end;
# the trailing usage chunk carries no token.
most_recent_timestamp = timestamp
elif usage := data.get("usage"):
output.output_tokens = usage.get("completion_tokens")
if (pt := usage.get("prompt_tokens")) is not None:
output.prompt_len = pt
most_recent_timestamp = timestamp
output.generated_text = generated_text
if first_chunk_received:
output.success = True
@@ -540,13 +540,15 @@ async def async_request_openai_audio(
)
generated_text += content or ""
# Only token chunks advance the request
# end; the trailing usage chunk carries no
# token.
most_recent_timestamp = timestamp
elif usage := data.get("usage"):
output.output_tokens = usage.get(
"completion_tokens"
)
most_recent_timestamp = timestamp
output.generated_text = generated_text
if first_chunk_received:
output.success = True