From 40828ce5fea04a66e219675f8018e60f9479646b Mon Sep 17 00:00:00 2001 From: "Wang, Yi" Date: Thu, 20 Mar 2025 11:56:16 +0800 Subject: [PATCH] =?UTF-8?q?fix=20"Total=20generated=20tokens:"=20is=200=20?= =?UTF-8?q?if=20using=20--backend=20tgi=20and=20--endpo=E2=80=A6=20(#14673?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Wang, Yi A --- benchmarks/backend_request_func.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/benchmarks/backend_request_func.py b/benchmarks/backend_request_func.py index 09c8e23ebb..0f13c79ae2 100644 --- a/benchmarks/backend_request_func.py +++ b/benchmarks/backend_request_func.py @@ -63,7 +63,7 @@ async def async_request_tgi( "temperature": 0.01, # TGI does not accept 0.0 temperature. "top_p": 0.99, # TGI does not accept 1.0 top_p. "truncate": request_func_input.prompt_len, - # TGI does not accept ignore_eos flag. + "ignore_eos_token": request_func_input.ignore_eos, } payload = { "inputs": request_func_input.prompt, @@ -71,6 +71,10 @@ async def async_request_tgi( } output = RequestFuncOutput() output.prompt_len = request_func_input.prompt_len + if request_func_input.ignore_eos: + output.output_tokens = request_func_input.output_len + else: + output.output_tokens = None ttft = 0.0 st = time.perf_counter()