[Frontend][Bugfix] Abort ASR engine requests on cancellation (#41266)

Signed-off-by: abdulrahman-cohere <abdulrahman.abdulrazzag@cohere.com>
Signed-off-by: <>
Co-authored-by: Cursor Agent <cursor-agent@cursor.com>
Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
This commit is contained in:
abdulrahman-cohere
2026-05-09 23:51:11 -07:00
committed by GitHub
co-authored by Cursor Agent Nicolò Lucchesi
parent f2840120f6
commit a5d0a5afba
2 changed files with 285 additions and 55 deletions
@@ -0,0 +1,191 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock
import pytest
from vllm.entrypoints.openai.speech_to_text.protocol import TranscriptionResponse
from vllm.entrypoints.openai.speech_to_text.speech_to_text import OpenAISpeechToText
async def _never_finishes():
await asyncio.Event().wait()
yield
async def _records_start_then_never_finishes(started_request_ids, request_id):
started_request_ids.append(request_id)
await asyncio.Event().wait()
yield
@pytest.mark.asyncio
@pytest.mark.parametrize(
("engine_inputs", "expected_request_ids"),
[
([{"prompt": "chunk"}], ["transcribe-outer-request"]),
(
[{"prompt": "chunk-0"}, {"prompt": "chunk-1"}],
["transcribe-outer-request-0", "transcribe-outer-request-1"],
),
],
)
async def test_non_streaming_cancel_aborts_engine_requests(
engine_inputs, expected_request_ids
):
engine_client = SimpleNamespace(
errored=False,
generate=Mock(side_effect=lambda *_args, **_kwargs: _never_finishes()),
abort=AsyncMock(),
is_tracing_enabled=AsyncMock(return_value=False),
)
server = OpenAISpeechToText.__new__(OpenAISpeechToText)
server.engine_client = engine_client
server.task_type = "transcribe"
server.models = SimpleNamespace(model_name=lambda: "audio")
server.model_config = SimpleNamespace(max_model_len=1024)
server.model_cls = SimpleNamespace(no_space_languages=set())
server.default_sampling_params = {}
server.asr_config = SimpleNamespace(max_audio_clip_s=30)
server._check_model = AsyncMock(return_value=None)
server._maybe_get_adapters = Mock(return_value=None)
server._preprocess_speech_to_text = AsyncMock(return_value=(engine_inputs, 40.0))
server._log_inputs = Mock()
request = SimpleNamespace(
model="audio",
response_format="json",
stream=False,
use_beam_search=False,
max_completion_tokens=None,
language="en",
prompt="",
to_sampling_params=Mock(return_value=object()),
)
raw_request = SimpleNamespace(
headers={"X-Request-Id": "outer-request"},
state=SimpleNamespace(),
)
task = asyncio.create_task(
server._create_speech_to_text(
audio_data=b"audio",
request=request,
raw_request=raw_request,
response_class=TranscriptionResponse,
stream_generator_method=Mock(),
)
)
await asyncio.sleep(0)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
generated_request_ids = [
call.args[2] for call in engine_client.generate.call_args_list
]
assert generated_request_ids == expected_request_ids
engine_client.abort.assert_awaited_once_with(expected_request_ids)
@pytest.mark.asyncio
async def test_non_streaming_cancel_advances_all_chunk_generators():
started_request_ids: list[str] = []
engine_client = SimpleNamespace(
errored=False,
generate=Mock(
side_effect=lambda *_args, **_kwargs: (
_records_start_then_never_finishes(started_request_ids, _args[2])
)
),
abort=AsyncMock(),
is_tracing_enabled=AsyncMock(return_value=False),
)
engine_inputs = [
{"prompt": "chunk-0"},
{"prompt": "chunk-1"},
{"prompt": "chunk-2"},
]
server = OpenAISpeechToText.__new__(OpenAISpeechToText)
server.engine_client = engine_client
server.task_type = "transcribe"
server.models = SimpleNamespace(model_name=lambda: "audio")
server.model_config = SimpleNamespace(max_model_len=1024)
server.model_cls = SimpleNamespace(no_space_languages=set())
server.default_sampling_params = {}
server.asr_config = SimpleNamespace(max_audio_clip_s=30)
server._check_model = AsyncMock(return_value=None)
server._maybe_get_adapters = Mock(return_value=None)
server._preprocess_speech_to_text = AsyncMock(return_value=(engine_inputs, 90.0))
server._log_inputs = Mock()
request = SimpleNamespace(
model="audio",
response_format="json",
stream=False,
use_beam_search=False,
max_completion_tokens=None,
language="en",
prompt="",
to_sampling_params=Mock(return_value=object()),
)
raw_request = SimpleNamespace(
headers={"X-Request-Id": "outer-request"},
state=SimpleNamespace(),
)
task = asyncio.create_task(
server._create_speech_to_text(
audio_data=b"audio",
request=request,
raw_request=raw_request,
response_class=TranscriptionResponse,
stream_generator_method=Mock(),
)
)
await asyncio.sleep(0.01)
expected_request_ids = [
"transcribe-outer-request-0",
"transcribe-outer-request-1",
"transcribe-outer-request-2",
]
assert set(started_request_ids) == set(expected_request_ids)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
@pytest.mark.asyncio
async def test_language_detection_cancel_aborts_engine_request():
engine_client = SimpleNamespace(
generate=Mock(return_value=_never_finishes()),
abort=AsyncMock(),
)
server = OpenAISpeechToText.__new__(OpenAISpeechToText)
server.engine_client = engine_client
server.asr_config = SimpleNamespace()
server.tokenizer = Mock()
server.model_cls = SimpleNamespace(
get_language_detection_prompt=Mock(return_value={"prompt": "detect"}),
get_language_token_ids=Mock(return_value=[1]),
parse_language_detection_output=Mock(),
)
request_id = "transcribe-outer-request-lang_detect"
task = asyncio.create_task(server._detect_language(Mock(), request_id))
await asyncio.sleep(0)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
engine_client.abort.assert_awaited_once_with(request_id)
@@ -49,6 +49,7 @@ from vllm.renderers.inputs import DictPrompt, EncoderDecoderDictPrompt
from vllm.renderers.inputs.preprocess import parse_enc_dec_prompt, parse_model_prompt
from vllm.sampling_params import BeamSearchParams, SamplingParams
from vllm.tokenizers import get_tokenizer
from vllm.utils.async_utils import merge_async_iterators
SpeechToTextResponse: TypeAlias = TranscriptionResponse | TranslationResponse
SpeechToTextResponseVerbose: TypeAlias = (
@@ -163,10 +164,17 @@ class OpenAISpeechToText(OpenAIServing):
request_id,
)
final_output: RequestOutput
async for final_output in result_generator:
if final_output.finished:
break
try:
final_output: RequestOutput
async for final_output in result_generator:
if final_output.finished:
break
except asyncio.CancelledError:
await asyncio.gather(
self.engine_client.abort(request_id),
return_exceptions=True,
)
raise
token_ids = list(final_output.outputs[0].token_ids)
lang = self.model_cls.parse_language_detection_output(
@@ -458,41 +466,55 @@ class OpenAISpeechToText(OpenAIServing):
if request.response_format == "verbose_json":
sampling_params.logprobs = 1
engine_request_ids = [
request_id if len(engine_inputs) == 1 else f"{request_id}-{idx}"
for idx in range(len(engine_inputs))
]
list_result_generator = []
for i, engine_input in enumerate(engine_inputs):
request_id_item = f"{request_id}_{i}"
self._log_inputs(
request_id_item,
engine_input,
params=sampling_params,
lora_request=lora_request,
)
trace_headers = (
None
if raw_request is None
else await self._get_trace_headers(raw_request.headers)
)
if isinstance(sampling_params, BeamSearchParams):
generator = self.beam_search(
prompt=engine_input,
params=sampling_params,
request_id=request_id_item,
lora_request=lora_request,
trace_headers=trace_headers,
)
else:
generator = self.engine_client.generate(
engine_input,
sampling_params,
try:
for request_id_item, engine_input in zip(engine_request_ids, engine_inputs):
self._log_inputs(
request_id_item,
engine_input,
params=sampling_params,
lora_request=lora_request,
trace_headers=trace_headers,
)
list_result_generator.append(generator)
trace_headers = (
None
if raw_request is None
else await self._get_trace_headers(raw_request.headers)
)
if isinstance(sampling_params, BeamSearchParams):
generator = self.beam_search(
prompt=engine_input,
params=sampling_params,
request_id=request_id_item,
lora_request=lora_request,
trace_headers=trace_headers,
)
else:
generator = self.engine_client.generate(
engine_input,
sampling_params,
request_id_item,
lora_request=lora_request,
trace_headers=trace_headers,
)
list_result_generator.append(generator)
except asyncio.CancelledError:
logger.info(
"Request %s cancelled; aborting %d transcription engine request(s).",
request_id,
len(engine_request_ids),
)
await asyncio.gather(
self.engine_client.abort(engine_request_ids),
return_exceptions=True,
)
raise
separator = asr_inter_chunk_separator(
request.language, self.model_cls.no_space_languages
@@ -508,10 +530,12 @@ class OpenAISpeechToText(OpenAIServing):
separator,
)
# Non-streaming response.
total_segments = []
text_parts = []
try:
assert list_result_generator is not None
chunk_segment_parts: list[list[SpeechToTextSegment]] = [
[] for _ in list_result_generator
]
chunk_text_parts: list[list[str]] = [[] for _ in list_result_generator]
segments_types: dict[str, type[SpeechToTextSegment]] = {
"transcribe": TranscriptionSegment,
"translate": TranslationSegment,
@@ -522,28 +546,34 @@ class OpenAISpeechToText(OpenAIServing):
assert len(list_result_generator) == 1, (
"`max_audio_clip_s` is set to None, audio cannot be chunked"
)
for idx, result_generator in enumerate(list_result_generator):
result_generator = merge_async_iterators(*list_result_generator)
async for idx, op in result_generator:
start_time = (
float(idx * chunk_size_in_s) if chunk_size_in_s is not None else 0.0
)
async for op in result_generator:
if request.response_format == "verbose_json":
assert op.outputs[0].logprobs
segments: list[SpeechToTextSegment] = (
self._get_verbose_segments(
tokens=tuple(op.outputs[0].token_ids),
segment_class=segment_class,
request=request,
start_time=start_time,
log_probs=op.outputs[0].logprobs,
)
)
if request.response_format == "verbose_json":
assert op.outputs[0].logprobs
segments: list[SpeechToTextSegment] = self._get_verbose_segments(
tokens=tuple(op.outputs[0].token_ids),
segment_class=segment_class,
request=request,
start_time=start_time,
log_probs=op.outputs[0].logprobs,
)
total_segments.extend(segments)
text_parts.extend([seg.text for seg in segments])
else:
raw_text = op.outputs[0].text
text_parts.append(self.model_cls.post_process_output(raw_text))
chunk_segment_parts[idx].extend(segments)
chunk_text_parts[idx].extend([seg.text for seg in segments])
else:
raw_text = op.outputs[0].text
chunk_text_parts[idx].append(
self.model_cls.post_process_output(raw_text)
)
total_segments = [
segment
for segment_parts in chunk_segment_parts
for segment in segment_parts
]
text_parts = [text for text_part in chunk_text_parts for text in text_part]
text = separator.join(text_parts)
if self.task_type == "transcribe":
final_response: ResponseType
@@ -583,7 +613,16 @@ class OpenAISpeechToText(OpenAIServing):
)
return final_response
except asyncio.CancelledError:
return self.create_error_response("Client disconnected")
logger.info(
"Request %s cancelled; aborting %d transcription engine request(s).",
request_id,
len(engine_request_ids),
)
await asyncio.gather(
self.engine_client.abort(engine_request_ids),
return_exceptions=True,
)
raise
async def _speech_to_text_stream_generator(
self,