[Bugfix] Account for client queue time in serve benchmarks (#54136)

Signed-off-by: maithilijoshi20 <maithilijoshi2003@gmail.com>
Signed-off-by: maithilijoshi20 <97733343+maithilijoshi20@users.noreply.github.com>
Signed-off-by: maithilijoshi20 <maithilij2003@gmail.com>
Co-authored-by: maithilijoshi20 <maithilijoshi2003@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
maithilijoshi20
2026-09-01 03:40:51 -07:00
committed by GitHub
co-authored by maithilijoshi20 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 92ccd2c306
commit 9e905f7450
3 changed files with 64 additions and 27 deletions
@@ -99,6 +99,8 @@ class RequestFuncOutput:
prompt_len: int = 0
error: str = ""
start_time: float = 0.0
# Time spent awaiting the benchmark client concurrency semaphore.
client_queue_time: float = 0.0
input_audio_duration: float = 0.0 # in seconds
num_input_sequences: int = 1