iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Bugfix] Reject tokenless chat and audio streams (#54708)

Browse Source 
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
Taneem Ibrahim
2026-09-01 11:24:19 -04:00
committed by GitHub
parent 339e16cbb6
commit d9eb4e344f
1 changed files with 24 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
vllm/benchmarks/lib/endpoint_request_func.py
+24 -10
View File
@@ -376,13 +376,13 @@ async def async_request_openai_chat_completions(
output.prompt_len = request_func_input.prompt_len
generated_text = ""
ttft = 0.0
st = time.perf_counter()
output.start_time = st
most_recent_timestamp = st
try:
async with session.post(url=api_url, json=payload, headers=headers) as response:
if response.status == 200:
first_chunk_received = False
handler = StreamedResponseHandler()
async for chunk_bytes in response.content.iter_any():
chunk_bytes = chunk_bytes.strip()
@@ -406,9 +406,9 @@ async def async_request_openai_chat_completions(
if choices := data.get("choices"):
content = choices[0]["delta"].get("content")
# First token
if ttft == 0.0:
ttft = timestamp - st
output.ttft = ttft
if not first_chunk_received:
first_chunk_received = True
output.ttft = timestamp - st
# Decoding phase
else:
@@ -423,7 +423,14 @@ async def async_request_openai_chat_completions(
most_recent_timestamp = timestamp
output.generated_text = generated_text
output.success = True
if first_chunk_received:
output.success = True
else:
output.success = False
output.error = (
"Never received a valid chunk to calculate TTFT."
"This response will be marked as failed!"
)
output.latency = most_recent_timestamp - st
else:
output.error = response.reason or ""
@@ -494,7 +501,6 @@ async def async_request_openai_audio(
output.input_audio_duration = input_audio_duration
generated_text = ""
ttft = 0.0
st = time.perf_counter()
output.start_time = st
most_recent_timestamp = st
@@ -503,6 +509,7 @@ async def async_request_openai_audio(
url=api_url, data=form, headers=headers
) as response:
if response.status == 200:
first_chunk_received = False
handler = StreamedResponseHandler()
async for chunk_bytes in response.content.iter_any():
@@ -522,9 +529,9 @@ async def async_request_openai_audio(
if choices := data.get("choices"):
content = choices[0]["delta"].get("content")
# First token
if ttft == 0.0:
ttft = timestamp - st
output.ttft = ttft
if not first_chunk_received:
first_chunk_received = True
output.ttft = timestamp - st
# Decoding phase
else:
@@ -541,7 +548,14 @@ async def async_request_openai_audio(
most_recent_timestamp = timestamp
output.generated_text = generated_text
output.success = True
if first_chunk_received:
output.success = True
else:
output.success = False
output.error = (
"Never received a valid chunk to calculate TTFT."
"This response will be marked as failed!"
)
output.latency = most_recent_timestamp - st
else:
output.error = response.reason or ""