diff --git a/tests/entrypoints/openai/speech_to_text/test_speech_to_text_cancellation.py b/tests/entrypoints/openai/speech_to_text/test_speech_to_text_cancellation.py new file mode 100644 index 0000000000..2c8c1229e8 --- /dev/null +++ b/tests/entrypoints/openai/speech_to_text/test_speech_to_text_cancellation.py @@ -0,0 +1,191 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import asyncio +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock + +import pytest + +from vllm.entrypoints.openai.speech_to_text.protocol import TranscriptionResponse +from vllm.entrypoints.openai.speech_to_text.speech_to_text import OpenAISpeechToText + + +async def _never_finishes(): + await asyncio.Event().wait() + yield + + +async def _records_start_then_never_finishes(started_request_ids, request_id): + started_request_ids.append(request_id) + await asyncio.Event().wait() + yield + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("engine_inputs", "expected_request_ids"), + [ + ([{"prompt": "chunk"}], ["transcribe-outer-request"]), + ( + [{"prompt": "chunk-0"}, {"prompt": "chunk-1"}], + ["transcribe-outer-request-0", "transcribe-outer-request-1"], + ), + ], +) +async def test_non_streaming_cancel_aborts_engine_requests( + engine_inputs, expected_request_ids +): + engine_client = SimpleNamespace( + errored=False, + generate=Mock(side_effect=lambda *_args, **_kwargs: _never_finishes()), + abort=AsyncMock(), + is_tracing_enabled=AsyncMock(return_value=False), + ) + + server = OpenAISpeechToText.__new__(OpenAISpeechToText) + server.engine_client = engine_client + server.task_type = "transcribe" + server.models = SimpleNamespace(model_name=lambda: "audio") + server.model_config = SimpleNamespace(max_model_len=1024) + server.model_cls = SimpleNamespace(no_space_languages=set()) + server.default_sampling_params = {} + server.asr_config = SimpleNamespace(max_audio_clip_s=30) + server._check_model = AsyncMock(return_value=None) + server._maybe_get_adapters = Mock(return_value=None) + server._preprocess_speech_to_text = AsyncMock(return_value=(engine_inputs, 40.0)) + server._log_inputs = Mock() + + request = SimpleNamespace( + model="audio", + response_format="json", + stream=False, + use_beam_search=False, + max_completion_tokens=None, + language="en", + prompt="", + to_sampling_params=Mock(return_value=object()), + ) + raw_request = SimpleNamespace( + headers={"X-Request-Id": "outer-request"}, + state=SimpleNamespace(), + ) + + task = asyncio.create_task( + server._create_speech_to_text( + audio_data=b"audio", + request=request, + raw_request=raw_request, + response_class=TranscriptionResponse, + stream_generator_method=Mock(), + ) + ) + await asyncio.sleep(0) + + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + generated_request_ids = [ + call.args[2] for call in engine_client.generate.call_args_list + ] + assert generated_request_ids == expected_request_ids + engine_client.abort.assert_awaited_once_with(expected_request_ids) + + +@pytest.mark.asyncio +async def test_non_streaming_cancel_advances_all_chunk_generators(): + started_request_ids: list[str] = [] + engine_client = SimpleNamespace( + errored=False, + generate=Mock( + side_effect=lambda *_args, **_kwargs: ( + _records_start_then_never_finishes(started_request_ids, _args[2]) + ) + ), + abort=AsyncMock(), + is_tracing_enabled=AsyncMock(return_value=False), + ) + + engine_inputs = [ + {"prompt": "chunk-0"}, + {"prompt": "chunk-1"}, + {"prompt": "chunk-2"}, + ] + server = OpenAISpeechToText.__new__(OpenAISpeechToText) + server.engine_client = engine_client + server.task_type = "transcribe" + server.models = SimpleNamespace(model_name=lambda: "audio") + server.model_config = SimpleNamespace(max_model_len=1024) + server.model_cls = SimpleNamespace(no_space_languages=set()) + server.default_sampling_params = {} + server.asr_config = SimpleNamespace(max_audio_clip_s=30) + server._check_model = AsyncMock(return_value=None) + server._maybe_get_adapters = Mock(return_value=None) + server._preprocess_speech_to_text = AsyncMock(return_value=(engine_inputs, 90.0)) + server._log_inputs = Mock() + + request = SimpleNamespace( + model="audio", + response_format="json", + stream=False, + use_beam_search=False, + max_completion_tokens=None, + language="en", + prompt="", + to_sampling_params=Mock(return_value=object()), + ) + raw_request = SimpleNamespace( + headers={"X-Request-Id": "outer-request"}, + state=SimpleNamespace(), + ) + + task = asyncio.create_task( + server._create_speech_to_text( + audio_data=b"audio", + request=request, + raw_request=raw_request, + response_class=TranscriptionResponse, + stream_generator_method=Mock(), + ) + ) + await asyncio.sleep(0.01) + + expected_request_ids = [ + "transcribe-outer-request-0", + "transcribe-outer-request-1", + "transcribe-outer-request-2", + ] + assert set(started_request_ids) == set(expected_request_ids) + + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + +@pytest.mark.asyncio +async def test_language_detection_cancel_aborts_engine_request(): + engine_client = SimpleNamespace( + generate=Mock(return_value=_never_finishes()), + abort=AsyncMock(), + ) + + server = OpenAISpeechToText.__new__(OpenAISpeechToText) + server.engine_client = engine_client + server.asr_config = SimpleNamespace() + server.tokenizer = Mock() + server.model_cls = SimpleNamespace( + get_language_detection_prompt=Mock(return_value={"prompt": "detect"}), + get_language_token_ids=Mock(return_value=[1]), + parse_language_detection_output=Mock(), + ) + + request_id = "transcribe-outer-request-lang_detect" + task = asyncio.create_task(server._detect_language(Mock(), request_id)) + await asyncio.sleep(0) + + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + engine_client.abort.assert_awaited_once_with(request_id) diff --git a/vllm/entrypoints/openai/speech_to_text/speech_to_text.py b/vllm/entrypoints/openai/speech_to_text/speech_to_text.py index c4c10c35f3..8c3aed4531 100644 --- a/vllm/entrypoints/openai/speech_to_text/speech_to_text.py +++ b/vllm/entrypoints/openai/speech_to_text/speech_to_text.py @@ -49,6 +49,7 @@ from vllm.renderers.inputs import DictPrompt, EncoderDecoderDictPrompt from vllm.renderers.inputs.preprocess import parse_enc_dec_prompt, parse_model_prompt from vllm.sampling_params import BeamSearchParams, SamplingParams from vllm.tokenizers import get_tokenizer +from vllm.utils.async_utils import merge_async_iterators SpeechToTextResponse: TypeAlias = TranscriptionResponse | TranslationResponse SpeechToTextResponseVerbose: TypeAlias = ( @@ -163,10 +164,17 @@ class OpenAISpeechToText(OpenAIServing): request_id, ) - final_output: RequestOutput - async for final_output in result_generator: - if final_output.finished: - break + try: + final_output: RequestOutput + async for final_output in result_generator: + if final_output.finished: + break + except asyncio.CancelledError: + await asyncio.gather( + self.engine_client.abort(request_id), + return_exceptions=True, + ) + raise token_ids = list(final_output.outputs[0].token_ids) lang = self.model_cls.parse_language_detection_output( @@ -458,41 +466,55 @@ class OpenAISpeechToText(OpenAIServing): if request.response_format == "verbose_json": sampling_params.logprobs = 1 + engine_request_ids = [ + request_id if len(engine_inputs) == 1 else f"{request_id}-{idx}" + for idx in range(len(engine_inputs)) + ] list_result_generator = [] - for i, engine_input in enumerate(engine_inputs): - request_id_item = f"{request_id}_{i}" - - self._log_inputs( - request_id_item, - engine_input, - params=sampling_params, - lora_request=lora_request, - ) - - trace_headers = ( - None - if raw_request is None - else await self._get_trace_headers(raw_request.headers) - ) - - if isinstance(sampling_params, BeamSearchParams): - generator = self.beam_search( - prompt=engine_input, - params=sampling_params, - request_id=request_id_item, - lora_request=lora_request, - trace_headers=trace_headers, - ) - else: - generator = self.engine_client.generate( - engine_input, - sampling_params, + try: + for request_id_item, engine_input in zip(engine_request_ids, engine_inputs): + self._log_inputs( request_id_item, + engine_input, + params=sampling_params, lora_request=lora_request, - trace_headers=trace_headers, ) - list_result_generator.append(generator) + trace_headers = ( + None + if raw_request is None + else await self._get_trace_headers(raw_request.headers) + ) + + if isinstance(sampling_params, BeamSearchParams): + generator = self.beam_search( + prompt=engine_input, + params=sampling_params, + request_id=request_id_item, + lora_request=lora_request, + trace_headers=trace_headers, + ) + else: + generator = self.engine_client.generate( + engine_input, + sampling_params, + request_id_item, + lora_request=lora_request, + trace_headers=trace_headers, + ) + + list_result_generator.append(generator) + except asyncio.CancelledError: + logger.info( + "Request %s cancelled; aborting %d transcription engine request(s).", + request_id, + len(engine_request_ids), + ) + await asyncio.gather( + self.engine_client.abort(engine_request_ids), + return_exceptions=True, + ) + raise separator = asr_inter_chunk_separator( request.language, self.model_cls.no_space_languages @@ -508,10 +530,12 @@ class OpenAISpeechToText(OpenAIServing): separator, ) # Non-streaming response. - total_segments = [] - text_parts = [] try: assert list_result_generator is not None + chunk_segment_parts: list[list[SpeechToTextSegment]] = [ + [] for _ in list_result_generator + ] + chunk_text_parts: list[list[str]] = [[] for _ in list_result_generator] segments_types: dict[str, type[SpeechToTextSegment]] = { "transcribe": TranscriptionSegment, "translate": TranslationSegment, @@ -522,28 +546,34 @@ class OpenAISpeechToText(OpenAIServing): assert len(list_result_generator) == 1, ( "`max_audio_clip_s` is set to None, audio cannot be chunked" ) - for idx, result_generator in enumerate(list_result_generator): + result_generator = merge_async_iterators(*list_result_generator) + async for idx, op in result_generator: start_time = ( float(idx * chunk_size_in_s) if chunk_size_in_s is not None else 0.0 ) - async for op in result_generator: - if request.response_format == "verbose_json": - assert op.outputs[0].logprobs - segments: list[SpeechToTextSegment] = ( - self._get_verbose_segments( - tokens=tuple(op.outputs[0].token_ids), - segment_class=segment_class, - request=request, - start_time=start_time, - log_probs=op.outputs[0].logprobs, - ) - ) + if request.response_format == "verbose_json": + assert op.outputs[0].logprobs + segments: list[SpeechToTextSegment] = self._get_verbose_segments( + tokens=tuple(op.outputs[0].token_ids), + segment_class=segment_class, + request=request, + start_time=start_time, + log_probs=op.outputs[0].logprobs, + ) - total_segments.extend(segments) - text_parts.extend([seg.text for seg in segments]) - else: - raw_text = op.outputs[0].text - text_parts.append(self.model_cls.post_process_output(raw_text)) + chunk_segment_parts[idx].extend(segments) + chunk_text_parts[idx].extend([seg.text for seg in segments]) + else: + raw_text = op.outputs[0].text + chunk_text_parts[idx].append( + self.model_cls.post_process_output(raw_text) + ) + total_segments = [ + segment + for segment_parts in chunk_segment_parts + for segment in segment_parts + ] + text_parts = [text for text_part in chunk_text_parts for text in text_part] text = separator.join(text_parts) if self.task_type == "transcribe": final_response: ResponseType @@ -583,7 +613,16 @@ class OpenAISpeechToText(OpenAIServing): ) return final_response except asyncio.CancelledError: - return self.create_error_response("Client disconnected") + logger.info( + "Request %s cancelled; aborting %d transcription engine request(s).", + request_id, + len(engine_request_ids), + ) + await asyncio.gather( + self.engine_client.abort(engine_request_ids), + return_exceptions=True, + ) + raise async def _speech_to_text_stream_generator( self,