[Bugfix] Reject tokenless chat and audio streams (#54708)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
@@ -376,13 +376,13 @@ async def async_request_openai_chat_completions(
|
|||||||
output.prompt_len = request_func_input.prompt_len
|
output.prompt_len = request_func_input.prompt_len
|
||||||
|
|
||||||
generated_text = ""
|
generated_text = ""
|
||||||
ttft = 0.0
|
|
||||||
st = time.perf_counter()
|
st = time.perf_counter()
|
||||||
output.start_time = st
|
output.start_time = st
|
||||||
most_recent_timestamp = st
|
most_recent_timestamp = st
|
||||||
try:
|
try:
|
||||||
async with session.post(url=api_url, json=payload, headers=headers) as response:
|
async with session.post(url=api_url, json=payload, headers=headers) as response:
|
||||||
if response.status == 200:
|
if response.status == 200:
|
||||||
|
first_chunk_received = False
|
||||||
handler = StreamedResponseHandler()
|
handler = StreamedResponseHandler()
|
||||||
async for chunk_bytes in response.content.iter_any():
|
async for chunk_bytes in response.content.iter_any():
|
||||||
chunk_bytes = chunk_bytes.strip()
|
chunk_bytes = chunk_bytes.strip()
|
||||||
@@ -406,9 +406,9 @@ async def async_request_openai_chat_completions(
|
|||||||
if choices := data.get("choices"):
|
if choices := data.get("choices"):
|
||||||
content = choices[0]["delta"].get("content")
|
content = choices[0]["delta"].get("content")
|
||||||
# First token
|
# First token
|
||||||
if ttft == 0.0:
|
if not first_chunk_received:
|
||||||
ttft = timestamp - st
|
first_chunk_received = True
|
||||||
output.ttft = ttft
|
output.ttft = timestamp - st
|
||||||
|
|
||||||
# Decoding phase
|
# Decoding phase
|
||||||
else:
|
else:
|
||||||
@@ -423,7 +423,14 @@ async def async_request_openai_chat_completions(
|
|||||||
most_recent_timestamp = timestamp
|
most_recent_timestamp = timestamp
|
||||||
|
|
||||||
output.generated_text = generated_text
|
output.generated_text = generated_text
|
||||||
output.success = True
|
if first_chunk_received:
|
||||||
|
output.success = True
|
||||||
|
else:
|
||||||
|
output.success = False
|
||||||
|
output.error = (
|
||||||
|
"Never received a valid chunk to calculate TTFT."
|
||||||
|
"This response will be marked as failed!"
|
||||||
|
)
|
||||||
output.latency = most_recent_timestamp - st
|
output.latency = most_recent_timestamp - st
|
||||||
else:
|
else:
|
||||||
output.error = response.reason or ""
|
output.error = response.reason or ""
|
||||||
@@ -494,7 +501,6 @@ async def async_request_openai_audio(
|
|||||||
output.input_audio_duration = input_audio_duration
|
output.input_audio_duration = input_audio_duration
|
||||||
|
|
||||||
generated_text = ""
|
generated_text = ""
|
||||||
ttft = 0.0
|
|
||||||
st = time.perf_counter()
|
st = time.perf_counter()
|
||||||
output.start_time = st
|
output.start_time = st
|
||||||
most_recent_timestamp = st
|
most_recent_timestamp = st
|
||||||
@@ -503,6 +509,7 @@ async def async_request_openai_audio(
|
|||||||
url=api_url, data=form, headers=headers
|
url=api_url, data=form, headers=headers
|
||||||
) as response:
|
) as response:
|
||||||
if response.status == 200:
|
if response.status == 200:
|
||||||
|
first_chunk_received = False
|
||||||
handler = StreamedResponseHandler()
|
handler = StreamedResponseHandler()
|
||||||
|
|
||||||
async for chunk_bytes in response.content.iter_any():
|
async for chunk_bytes in response.content.iter_any():
|
||||||
@@ -522,9 +529,9 @@ async def async_request_openai_audio(
|
|||||||
if choices := data.get("choices"):
|
if choices := data.get("choices"):
|
||||||
content = choices[0]["delta"].get("content")
|
content = choices[0]["delta"].get("content")
|
||||||
# First token
|
# First token
|
||||||
if ttft == 0.0:
|
if not first_chunk_received:
|
||||||
ttft = timestamp - st
|
first_chunk_received = True
|
||||||
output.ttft = ttft
|
output.ttft = timestamp - st
|
||||||
|
|
||||||
# Decoding phase
|
# Decoding phase
|
||||||
else:
|
else:
|
||||||
@@ -541,7 +548,14 @@ async def async_request_openai_audio(
|
|||||||
most_recent_timestamp = timestamp
|
most_recent_timestamp = timestamp
|
||||||
|
|
||||||
output.generated_text = generated_text
|
output.generated_text = generated_text
|
||||||
output.success = True
|
if first_chunk_received:
|
||||||
|
output.success = True
|
||||||
|
else:
|
||||||
|
output.success = False
|
||||||
|
output.error = (
|
||||||
|
"Never received a valid chunk to calculate TTFT."
|
||||||
|
"This response will be marked as failed!"
|
||||||
|
)
|
||||||
output.latency = most_recent_timestamp - st
|
output.latency = most_recent_timestamp - st
|
||||||
else:
|
else:
|
||||||
output.error = response.reason or ""
|
output.error = response.reason or ""
|
||||||
|
|||||||
Reference in New Issue
Block a user