[Bugfix][Benchmark] Make streaming TTFT/E2E latency accounting consistent across endpoints (#55508)

Signed-off-by: surajm20061998 <surajm20061998@gmail.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Suraj
2026-09-13 05:20:54 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 2671fedfc7
commit 93e051e290
3 changed files with 505 additions and 23 deletions
+33 -17
View File
@@ -105,7 +105,7 @@ async def async_request_tgi(
timestamp = time.perf_counter()
# First token
if ttft == 0.0:
ttft = time.perf_counter() - st
ttft = timestamp - st
output.ttft = ttft
# Decoding phase
@@ -330,8 +330,7 @@ async def async_request_openai_completions(
# First token
if not first_chunk_received:
first_chunk_received = True
ttft = time.perf_counter() - st
output.ttft = ttft
output.ttft = timestamp - st
# Decoding phase
else:
@@ -416,7 +415,7 @@ async def async_request_openai_chat_completions(
output.prompt_len = request_func_input.prompt_len
generated_text = ""
ttft = 0.0
first_chunk_received = False
st = time.perf_counter()
most_recent_timestamp = st
try:
@@ -443,22 +442,30 @@ async def async_request_openai_chat_completions(
if choices := data.get("choices"):
content = choices[0]["delta"].get("content")
# First token
if ttft == 0.0:
ttft = timestamp - st
output.ttft = ttft
if not first_chunk_received:
first_chunk_received = True
output.ttft = timestamp - st
# Decoding phase
else:
output.itl.append(timestamp - most_recent_timestamp)
generated_text += content or ""
# Only token chunks advance the request end;
# the trailing usage chunk carries no token.
most_recent_timestamp = timestamp
elif usage := data.get("usage"):
output.output_tokens = usage.get("completion_tokens")
most_recent_timestamp = timestamp
output.generated_text = generated_text
output.success = True
if first_chunk_received:
output.success = True
else:
output.success = False
output.error = (
"Never received a valid chunk to calculate TTFT."
"This response will be marked as failed!"
)
output.latency = most_recent_timestamp - st
else:
output.error = response.reason or ""
@@ -530,7 +537,7 @@ async def async_request_openai_audio(
output.prompt_len = request_func_input.prompt_len
generated_text = ""
ttft = 0.0
first_chunk_received = False
st = time.perf_counter()
most_recent_timestamp = st
try:
@@ -551,9 +558,9 @@ async def async_request_openai_audio(
if choices := data.get("choices"):
content = choices[0]["delta"].get("content")
# First token
if ttft == 0.0:
ttft = timestamp - st
output.ttft = ttft
if not first_chunk_received:
first_chunk_received = True
output.ttft = timestamp - st
# Decoding phase
else:
@@ -562,15 +569,24 @@ async def async_request_openai_audio(
)
generated_text += content or ""
# Only token chunks advance the request
# end; the trailing usage chunk carries no
# token.
most_recent_timestamp = timestamp
elif usage := data.get("usage"):
output.output_tokens = usage.get(
"completion_tokens"
)
most_recent_timestamp = timestamp
output.generated_text = generated_text
output.success = True
if first_chunk_received:
output.success = True
else:
output.success = False
output.error = (
"Never received a valid chunk to calculate TTFT."
"This response will be marked as failed!"
)
output.latency = most_recent_timestamp - st
else:
output.error = response.reason or ""