[Frontend][Bugfix] Abort ASR engine requests on cancellation (#41266)

Signed-off-by: abdulrahman-cohere <abdulrahman.abdulrazzag@cohere.com>
Signed-off-by: <>
Co-authored-by: Cursor Agent <cursor-agent@cursor.com>
Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
This commit is contained in:
abdulrahman-cohere
2026-05-09 23:51:11 -07:00
committed by GitHub
co-authored by Cursor Agent Nicolò Lucchesi
parent f2840120f6
commit a5d0a5afba
2 changed files with 285 additions and 55 deletions
@@ -49,6 +49,7 @@ from vllm.renderers.inputs import DictPrompt, EncoderDecoderDictPrompt
from vllm.renderers.inputs.preprocess import parse_enc_dec_prompt, parse_model_prompt
from vllm.sampling_params import BeamSearchParams, SamplingParams
from vllm.tokenizers import get_tokenizer
from vllm.utils.async_utils import merge_async_iterators
SpeechToTextResponse: TypeAlias = TranscriptionResponse | TranslationResponse
SpeechToTextResponseVerbose: TypeAlias = (
@@ -163,10 +164,17 @@ class OpenAISpeechToText(OpenAIServing):
request_id,
)
final_output: RequestOutput
async for final_output in result_generator:
if final_output.finished:
break
try:
final_output: RequestOutput
async for final_output in result_generator:
if final_output.finished:
break
except asyncio.CancelledError:
await asyncio.gather(
self.engine_client.abort(request_id),
return_exceptions=True,
)
raise
token_ids = list(final_output.outputs[0].token_ids)
lang = self.model_cls.parse_language_detection_output(
@@ -458,41 +466,55 @@ class OpenAISpeechToText(OpenAIServing):
if request.response_format == "verbose_json":
sampling_params.logprobs = 1
engine_request_ids = [
request_id if len(engine_inputs) == 1 else f"{request_id}-{idx}"
for idx in range(len(engine_inputs))
]
list_result_generator = []
for i, engine_input in enumerate(engine_inputs):
request_id_item = f"{request_id}_{i}"
self._log_inputs(
request_id_item,
engine_input,
params=sampling_params,
lora_request=lora_request,
)
trace_headers = (
None
if raw_request is None
else await self._get_trace_headers(raw_request.headers)
)
if isinstance(sampling_params, BeamSearchParams):
generator = self.beam_search(
prompt=engine_input,
params=sampling_params,
request_id=request_id_item,
lora_request=lora_request,
trace_headers=trace_headers,
)
else:
generator = self.engine_client.generate(
engine_input,
sampling_params,
try:
for request_id_item, engine_input in zip(engine_request_ids, engine_inputs):
self._log_inputs(
request_id_item,
engine_input,
params=sampling_params,
lora_request=lora_request,
trace_headers=trace_headers,
)
list_result_generator.append(generator)
trace_headers = (
None
if raw_request is None
else await self._get_trace_headers(raw_request.headers)
)
if isinstance(sampling_params, BeamSearchParams):
generator = self.beam_search(
prompt=engine_input,
params=sampling_params,
request_id=request_id_item,
lora_request=lora_request,
trace_headers=trace_headers,
)
else:
generator = self.engine_client.generate(
engine_input,
sampling_params,
request_id_item,
lora_request=lora_request,
trace_headers=trace_headers,
)
list_result_generator.append(generator)
except asyncio.CancelledError:
logger.info(
"Request %s cancelled; aborting %d transcription engine request(s).",
request_id,
len(engine_request_ids),
)
await asyncio.gather(
self.engine_client.abort(engine_request_ids),
return_exceptions=True,
)
raise
separator = asr_inter_chunk_separator(
request.language, self.model_cls.no_space_languages
@@ -508,10 +530,12 @@ class OpenAISpeechToText(OpenAIServing):
separator,
)
# Non-streaming response.
total_segments = []
text_parts = []
try:
assert list_result_generator is not None
chunk_segment_parts: list[list[SpeechToTextSegment]] = [
[] for _ in list_result_generator
]
chunk_text_parts: list[list[str]] = [[] for _ in list_result_generator]
segments_types: dict[str, type[SpeechToTextSegment]] = {
"transcribe": TranscriptionSegment,
"translate": TranslationSegment,
@@ -522,28 +546,34 @@ class OpenAISpeechToText(OpenAIServing):
assert len(list_result_generator) == 1, (
"`max_audio_clip_s` is set to None, audio cannot be chunked"
)
for idx, result_generator in enumerate(list_result_generator):
result_generator = merge_async_iterators(*list_result_generator)
async for idx, op in result_generator:
start_time = (
float(idx * chunk_size_in_s) if chunk_size_in_s is not None else 0.0
)
async for op in result_generator:
if request.response_format == "verbose_json":
assert op.outputs[0].logprobs
segments: list[SpeechToTextSegment] = (
self._get_verbose_segments(
tokens=tuple(op.outputs[0].token_ids),
segment_class=segment_class,
request=request,
start_time=start_time,
log_probs=op.outputs[0].logprobs,
)
)
if request.response_format == "verbose_json":
assert op.outputs[0].logprobs
segments: list[SpeechToTextSegment] = self._get_verbose_segments(
tokens=tuple(op.outputs[0].token_ids),
segment_class=segment_class,
request=request,
start_time=start_time,
log_probs=op.outputs[0].logprobs,
)
total_segments.extend(segments)
text_parts.extend([seg.text for seg in segments])
else:
raw_text = op.outputs[0].text
text_parts.append(self.model_cls.post_process_output(raw_text))
chunk_segment_parts[idx].extend(segments)
chunk_text_parts[idx].extend([seg.text for seg in segments])
else:
raw_text = op.outputs[0].text
chunk_text_parts[idx].append(
self.model_cls.post_process_output(raw_text)
)
total_segments = [
segment
for segment_parts in chunk_segment_parts
for segment in segment_parts
]
text_parts = [text for text_part in chunk_text_parts for text in text_part]
text = separator.join(text_parts)
if self.task_type == "transcribe":
final_response: ResponseType
@@ -583,7 +613,16 @@ class OpenAISpeechToText(OpenAIServing):
)
return final_response
except asyncio.CancelledError:
return self.create_error_response("Client disconnected")
logger.info(
"Request %s cancelled; aborting %d transcription engine request(s).",
request_id,
len(engine_request_ids),
)
await asyncio.gather(
self.engine_client.abort(engine_request_ids),
return_exceptions=True,
)
raise
async def _speech_to_text_stream_generator(
self,