[Frontend][Bugfix] Abort ASR engine requests on cancellation (#41266)
Signed-off-by: abdulrahman-cohere <abdulrahman.abdulrazzag@cohere.com> Signed-off-by: <> Co-authored-by: Cursor Agent <cursor-agent@cursor.com> Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
This commit is contained in:
co-authored by
Cursor Agent
Nicolò Lucchesi
parent
f2840120f6
commit
a5d0a5afba
@@ -0,0 +1,191 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.openai.speech_to_text.protocol import TranscriptionResponse
|
||||
from vllm.entrypoints.openai.speech_to_text.speech_to_text import OpenAISpeechToText
|
||||
|
||||
|
||||
async def _never_finishes():
|
||||
await asyncio.Event().wait()
|
||||
yield
|
||||
|
||||
|
||||
async def _records_start_then_never_finishes(started_request_ids, request_id):
|
||||
started_request_ids.append(request_id)
|
||||
await asyncio.Event().wait()
|
||||
yield
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("engine_inputs", "expected_request_ids"),
|
||||
[
|
||||
([{"prompt": "chunk"}], ["transcribe-outer-request"]),
|
||||
(
|
||||
[{"prompt": "chunk-0"}, {"prompt": "chunk-1"}],
|
||||
["transcribe-outer-request-0", "transcribe-outer-request-1"],
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_non_streaming_cancel_aborts_engine_requests(
|
||||
engine_inputs, expected_request_ids
|
||||
):
|
||||
engine_client = SimpleNamespace(
|
||||
errored=False,
|
||||
generate=Mock(side_effect=lambda *_args, **_kwargs: _never_finishes()),
|
||||
abort=AsyncMock(),
|
||||
is_tracing_enabled=AsyncMock(return_value=False),
|
||||
)
|
||||
|
||||
server = OpenAISpeechToText.__new__(OpenAISpeechToText)
|
||||
server.engine_client = engine_client
|
||||
server.task_type = "transcribe"
|
||||
server.models = SimpleNamespace(model_name=lambda: "audio")
|
||||
server.model_config = SimpleNamespace(max_model_len=1024)
|
||||
server.model_cls = SimpleNamespace(no_space_languages=set())
|
||||
server.default_sampling_params = {}
|
||||
server.asr_config = SimpleNamespace(max_audio_clip_s=30)
|
||||
server._check_model = AsyncMock(return_value=None)
|
||||
server._maybe_get_adapters = Mock(return_value=None)
|
||||
server._preprocess_speech_to_text = AsyncMock(return_value=(engine_inputs, 40.0))
|
||||
server._log_inputs = Mock()
|
||||
|
||||
request = SimpleNamespace(
|
||||
model="audio",
|
||||
response_format="json",
|
||||
stream=False,
|
||||
use_beam_search=False,
|
||||
max_completion_tokens=None,
|
||||
language="en",
|
||||
prompt="",
|
||||
to_sampling_params=Mock(return_value=object()),
|
||||
)
|
||||
raw_request = SimpleNamespace(
|
||||
headers={"X-Request-Id": "outer-request"},
|
||||
state=SimpleNamespace(),
|
||||
)
|
||||
|
||||
task = asyncio.create_task(
|
||||
server._create_speech_to_text(
|
||||
audio_data=b"audio",
|
||||
request=request,
|
||||
raw_request=raw_request,
|
||||
response_class=TranscriptionResponse,
|
||||
stream_generator_method=Mock(),
|
||||
)
|
||||
)
|
||||
await asyncio.sleep(0)
|
||||
|
||||
task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
|
||||
generated_request_ids = [
|
||||
call.args[2] for call in engine_client.generate.call_args_list
|
||||
]
|
||||
assert generated_request_ids == expected_request_ids
|
||||
engine_client.abort.assert_awaited_once_with(expected_request_ids)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_streaming_cancel_advances_all_chunk_generators():
|
||||
started_request_ids: list[str] = []
|
||||
engine_client = SimpleNamespace(
|
||||
errored=False,
|
||||
generate=Mock(
|
||||
side_effect=lambda *_args, **_kwargs: (
|
||||
_records_start_then_never_finishes(started_request_ids, _args[2])
|
||||
)
|
||||
),
|
||||
abort=AsyncMock(),
|
||||
is_tracing_enabled=AsyncMock(return_value=False),
|
||||
)
|
||||
|
||||
engine_inputs = [
|
||||
{"prompt": "chunk-0"},
|
||||
{"prompt": "chunk-1"},
|
||||
{"prompt": "chunk-2"},
|
||||
]
|
||||
server = OpenAISpeechToText.__new__(OpenAISpeechToText)
|
||||
server.engine_client = engine_client
|
||||
server.task_type = "transcribe"
|
||||
server.models = SimpleNamespace(model_name=lambda: "audio")
|
||||
server.model_config = SimpleNamespace(max_model_len=1024)
|
||||
server.model_cls = SimpleNamespace(no_space_languages=set())
|
||||
server.default_sampling_params = {}
|
||||
server.asr_config = SimpleNamespace(max_audio_clip_s=30)
|
||||
server._check_model = AsyncMock(return_value=None)
|
||||
server._maybe_get_adapters = Mock(return_value=None)
|
||||
server._preprocess_speech_to_text = AsyncMock(return_value=(engine_inputs, 90.0))
|
||||
server._log_inputs = Mock()
|
||||
|
||||
request = SimpleNamespace(
|
||||
model="audio",
|
||||
response_format="json",
|
||||
stream=False,
|
||||
use_beam_search=False,
|
||||
max_completion_tokens=None,
|
||||
language="en",
|
||||
prompt="",
|
||||
to_sampling_params=Mock(return_value=object()),
|
||||
)
|
||||
raw_request = SimpleNamespace(
|
||||
headers={"X-Request-Id": "outer-request"},
|
||||
state=SimpleNamespace(),
|
||||
)
|
||||
|
||||
task = asyncio.create_task(
|
||||
server._create_speech_to_text(
|
||||
audio_data=b"audio",
|
||||
request=request,
|
||||
raw_request=raw_request,
|
||||
response_class=TranscriptionResponse,
|
||||
stream_generator_method=Mock(),
|
||||
)
|
||||
)
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
expected_request_ids = [
|
||||
"transcribe-outer-request-0",
|
||||
"transcribe-outer-request-1",
|
||||
"transcribe-outer-request-2",
|
||||
]
|
||||
assert set(started_request_ids) == set(expected_request_ids)
|
||||
|
||||
task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_language_detection_cancel_aborts_engine_request():
|
||||
engine_client = SimpleNamespace(
|
||||
generate=Mock(return_value=_never_finishes()),
|
||||
abort=AsyncMock(),
|
||||
)
|
||||
|
||||
server = OpenAISpeechToText.__new__(OpenAISpeechToText)
|
||||
server.engine_client = engine_client
|
||||
server.asr_config = SimpleNamespace()
|
||||
server.tokenizer = Mock()
|
||||
server.model_cls = SimpleNamespace(
|
||||
get_language_detection_prompt=Mock(return_value={"prompt": "detect"}),
|
||||
get_language_token_ids=Mock(return_value=[1]),
|
||||
parse_language_detection_output=Mock(),
|
||||
)
|
||||
|
||||
request_id = "transcribe-outer-request-lang_detect"
|
||||
task = asyncio.create_task(server._detect_language(Mock(), request_id))
|
||||
await asyncio.sleep(0)
|
||||
|
||||
task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
|
||||
engine_client.abort.assert_awaited_once_with(request_id)
|
||||
@@ -49,6 +49,7 @@ from vllm.renderers.inputs import DictPrompt, EncoderDecoderDictPrompt
|
||||
from vllm.renderers.inputs.preprocess import parse_enc_dec_prompt, parse_model_prompt
|
||||
from vllm.sampling_params import BeamSearchParams, SamplingParams
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
from vllm.utils.async_utils import merge_async_iterators
|
||||
|
||||
SpeechToTextResponse: TypeAlias = TranscriptionResponse | TranslationResponse
|
||||
SpeechToTextResponseVerbose: TypeAlias = (
|
||||
@@ -163,10 +164,17 @@ class OpenAISpeechToText(OpenAIServing):
|
||||
request_id,
|
||||
)
|
||||
|
||||
final_output: RequestOutput
|
||||
async for final_output in result_generator:
|
||||
if final_output.finished:
|
||||
break
|
||||
try:
|
||||
final_output: RequestOutput
|
||||
async for final_output in result_generator:
|
||||
if final_output.finished:
|
||||
break
|
||||
except asyncio.CancelledError:
|
||||
await asyncio.gather(
|
||||
self.engine_client.abort(request_id),
|
||||
return_exceptions=True,
|
||||
)
|
||||
raise
|
||||
|
||||
token_ids = list(final_output.outputs[0].token_ids)
|
||||
lang = self.model_cls.parse_language_detection_output(
|
||||
@@ -458,41 +466,55 @@ class OpenAISpeechToText(OpenAIServing):
|
||||
if request.response_format == "verbose_json":
|
||||
sampling_params.logprobs = 1
|
||||
|
||||
engine_request_ids = [
|
||||
request_id if len(engine_inputs) == 1 else f"{request_id}-{idx}"
|
||||
for idx in range(len(engine_inputs))
|
||||
]
|
||||
list_result_generator = []
|
||||
for i, engine_input in enumerate(engine_inputs):
|
||||
request_id_item = f"{request_id}_{i}"
|
||||
|
||||
self._log_inputs(
|
||||
request_id_item,
|
||||
engine_input,
|
||||
params=sampling_params,
|
||||
lora_request=lora_request,
|
||||
)
|
||||
|
||||
trace_headers = (
|
||||
None
|
||||
if raw_request is None
|
||||
else await self._get_trace_headers(raw_request.headers)
|
||||
)
|
||||
|
||||
if isinstance(sampling_params, BeamSearchParams):
|
||||
generator = self.beam_search(
|
||||
prompt=engine_input,
|
||||
params=sampling_params,
|
||||
request_id=request_id_item,
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
)
|
||||
else:
|
||||
generator = self.engine_client.generate(
|
||||
engine_input,
|
||||
sampling_params,
|
||||
try:
|
||||
for request_id_item, engine_input in zip(engine_request_ids, engine_inputs):
|
||||
self._log_inputs(
|
||||
request_id_item,
|
||||
engine_input,
|
||||
params=sampling_params,
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
)
|
||||
|
||||
list_result_generator.append(generator)
|
||||
trace_headers = (
|
||||
None
|
||||
if raw_request is None
|
||||
else await self._get_trace_headers(raw_request.headers)
|
||||
)
|
||||
|
||||
if isinstance(sampling_params, BeamSearchParams):
|
||||
generator = self.beam_search(
|
||||
prompt=engine_input,
|
||||
params=sampling_params,
|
||||
request_id=request_id_item,
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
)
|
||||
else:
|
||||
generator = self.engine_client.generate(
|
||||
engine_input,
|
||||
sampling_params,
|
||||
request_id_item,
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
)
|
||||
|
||||
list_result_generator.append(generator)
|
||||
except asyncio.CancelledError:
|
||||
logger.info(
|
||||
"Request %s cancelled; aborting %d transcription engine request(s).",
|
||||
request_id,
|
||||
len(engine_request_ids),
|
||||
)
|
||||
await asyncio.gather(
|
||||
self.engine_client.abort(engine_request_ids),
|
||||
return_exceptions=True,
|
||||
)
|
||||
raise
|
||||
|
||||
separator = asr_inter_chunk_separator(
|
||||
request.language, self.model_cls.no_space_languages
|
||||
@@ -508,10 +530,12 @@ class OpenAISpeechToText(OpenAIServing):
|
||||
separator,
|
||||
)
|
||||
# Non-streaming response.
|
||||
total_segments = []
|
||||
text_parts = []
|
||||
try:
|
||||
assert list_result_generator is not None
|
||||
chunk_segment_parts: list[list[SpeechToTextSegment]] = [
|
||||
[] for _ in list_result_generator
|
||||
]
|
||||
chunk_text_parts: list[list[str]] = [[] for _ in list_result_generator]
|
||||
segments_types: dict[str, type[SpeechToTextSegment]] = {
|
||||
"transcribe": TranscriptionSegment,
|
||||
"translate": TranslationSegment,
|
||||
@@ -522,28 +546,34 @@ class OpenAISpeechToText(OpenAIServing):
|
||||
assert len(list_result_generator) == 1, (
|
||||
"`max_audio_clip_s` is set to None, audio cannot be chunked"
|
||||
)
|
||||
for idx, result_generator in enumerate(list_result_generator):
|
||||
result_generator = merge_async_iterators(*list_result_generator)
|
||||
async for idx, op in result_generator:
|
||||
start_time = (
|
||||
float(idx * chunk_size_in_s) if chunk_size_in_s is not None else 0.0
|
||||
)
|
||||
async for op in result_generator:
|
||||
if request.response_format == "verbose_json":
|
||||
assert op.outputs[0].logprobs
|
||||
segments: list[SpeechToTextSegment] = (
|
||||
self._get_verbose_segments(
|
||||
tokens=tuple(op.outputs[0].token_ids),
|
||||
segment_class=segment_class,
|
||||
request=request,
|
||||
start_time=start_time,
|
||||
log_probs=op.outputs[0].logprobs,
|
||||
)
|
||||
)
|
||||
if request.response_format == "verbose_json":
|
||||
assert op.outputs[0].logprobs
|
||||
segments: list[SpeechToTextSegment] = self._get_verbose_segments(
|
||||
tokens=tuple(op.outputs[0].token_ids),
|
||||
segment_class=segment_class,
|
||||
request=request,
|
||||
start_time=start_time,
|
||||
log_probs=op.outputs[0].logprobs,
|
||||
)
|
||||
|
||||
total_segments.extend(segments)
|
||||
text_parts.extend([seg.text for seg in segments])
|
||||
else:
|
||||
raw_text = op.outputs[0].text
|
||||
text_parts.append(self.model_cls.post_process_output(raw_text))
|
||||
chunk_segment_parts[idx].extend(segments)
|
||||
chunk_text_parts[idx].extend([seg.text for seg in segments])
|
||||
else:
|
||||
raw_text = op.outputs[0].text
|
||||
chunk_text_parts[idx].append(
|
||||
self.model_cls.post_process_output(raw_text)
|
||||
)
|
||||
total_segments = [
|
||||
segment
|
||||
for segment_parts in chunk_segment_parts
|
||||
for segment in segment_parts
|
||||
]
|
||||
text_parts = [text for text_part in chunk_text_parts for text in text_part]
|
||||
text = separator.join(text_parts)
|
||||
if self.task_type == "transcribe":
|
||||
final_response: ResponseType
|
||||
@@ -583,7 +613,16 @@ class OpenAISpeechToText(OpenAIServing):
|
||||
)
|
||||
return final_response
|
||||
except asyncio.CancelledError:
|
||||
return self.create_error_response("Client disconnected")
|
||||
logger.info(
|
||||
"Request %s cancelled; aborting %d transcription engine request(s).",
|
||||
request_id,
|
||||
len(engine_request_ids),
|
||||
)
|
||||
await asyncio.gather(
|
||||
self.engine_client.abort(engine_request_ids),
|
||||
return_exceptions=True,
|
||||
)
|
||||
raise
|
||||
|
||||
async def _speech_to_text_stream_generator(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user