[Bugfix] Account for client queue time in serve benchmarks (#54136)
Signed-off-by: maithilijoshi20 <maithilijoshi2003@gmail.com> Signed-off-by: maithilijoshi20 <97733343+maithilijoshi20@users.noreply.github.com> Signed-off-by: maithilijoshi20 <maithilij2003@gmail.com> Co-authored-by: maithilijoshi20 <maithilijoshi2003@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
maithilijoshi20
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
92ccd2c306
commit
9e905f7450
@@ -99,6 +99,8 @@ class RequestFuncOutput:
|
||||
prompt_len: int = 0
|
||||
error: str = ""
|
||||
start_time: float = 0.0
|
||||
# Time spent awaiting the benchmark client concurrency semaphore.
|
||||
client_queue_time: float = 0.0
|
||||
input_audio_duration: float = 0.0 # in seconds
|
||||
num_input_sequences: int = 1
|
||||
|
||||
|
||||
Reference in New Issue
Block a user