[Frontend][Bugfix] Abort ASR engine requests on cancellation (#41266)
Signed-off-by: abdulrahman-cohere <abdulrahman.abdulrazzag@cohere.com> Signed-off-by: <> Co-authored-by: Cursor Agent <cursor-agent@cursor.com> Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
This commit is contained in:
co-authored by
Cursor Agent
Nicolò Lucchesi
parent
f2840120f6
commit
a5d0a5afba
@@ -49,6 +49,7 @@ from vllm.renderers.inputs import DictPrompt, EncoderDecoderDictPrompt
|
||||
from vllm.renderers.inputs.preprocess import parse_enc_dec_prompt, parse_model_prompt
|
||||
from vllm.sampling_params import BeamSearchParams, SamplingParams
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
from vllm.utils.async_utils import merge_async_iterators
|
||||
|
||||
SpeechToTextResponse: TypeAlias = TranscriptionResponse | TranslationResponse
|
||||
SpeechToTextResponseVerbose: TypeAlias = (
|
||||
@@ -163,10 +164,17 @@ class OpenAISpeechToText(OpenAIServing):
|
||||
request_id,
|
||||
)
|
||||
|
||||
final_output: RequestOutput
|
||||
async for final_output in result_generator:
|
||||
if final_output.finished:
|
||||
break
|
||||
try:
|
||||
final_output: RequestOutput
|
||||
async for final_output in result_generator:
|
||||
if final_output.finished:
|
||||
break
|
||||
except asyncio.CancelledError:
|
||||
await asyncio.gather(
|
||||
self.engine_client.abort(request_id),
|
||||
return_exceptions=True,
|
||||
)
|
||||
raise
|
||||
|
||||
token_ids = list(final_output.outputs[0].token_ids)
|
||||
lang = self.model_cls.parse_language_detection_output(
|
||||
@@ -458,41 +466,55 @@ class OpenAISpeechToText(OpenAIServing):
|
||||
if request.response_format == "verbose_json":
|
||||
sampling_params.logprobs = 1
|
||||
|
||||
engine_request_ids = [
|
||||
request_id if len(engine_inputs) == 1 else f"{request_id}-{idx}"
|
||||
for idx in range(len(engine_inputs))
|
||||
]
|
||||
list_result_generator = []
|
||||
for i, engine_input in enumerate(engine_inputs):
|
||||
request_id_item = f"{request_id}_{i}"
|
||||
|
||||
self._log_inputs(
|
||||
request_id_item,
|
||||
engine_input,
|
||||
params=sampling_params,
|
||||
lora_request=lora_request,
|
||||
)
|
||||
|
||||
trace_headers = (
|
||||
None
|
||||
if raw_request is None
|
||||
else await self._get_trace_headers(raw_request.headers)
|
||||
)
|
||||
|
||||
if isinstance(sampling_params, BeamSearchParams):
|
||||
generator = self.beam_search(
|
||||
prompt=engine_input,
|
||||
params=sampling_params,
|
||||
request_id=request_id_item,
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
)
|
||||
else:
|
||||
generator = self.engine_client.generate(
|
||||
engine_input,
|
||||
sampling_params,
|
||||
try:
|
||||
for request_id_item, engine_input in zip(engine_request_ids, engine_inputs):
|
||||
self._log_inputs(
|
||||
request_id_item,
|
||||
engine_input,
|
||||
params=sampling_params,
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
)
|
||||
|
||||
list_result_generator.append(generator)
|
||||
trace_headers = (
|
||||
None
|
||||
if raw_request is None
|
||||
else await self._get_trace_headers(raw_request.headers)
|
||||
)
|
||||
|
||||
if isinstance(sampling_params, BeamSearchParams):
|
||||
generator = self.beam_search(
|
||||
prompt=engine_input,
|
||||
params=sampling_params,
|
||||
request_id=request_id_item,
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
)
|
||||
else:
|
||||
generator = self.engine_client.generate(
|
||||
engine_input,
|
||||
sampling_params,
|
||||
request_id_item,
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
)
|
||||
|
||||
list_result_generator.append(generator)
|
||||
except asyncio.CancelledError:
|
||||
logger.info(
|
||||
"Request %s cancelled; aborting %d transcription engine request(s).",
|
||||
request_id,
|
||||
len(engine_request_ids),
|
||||
)
|
||||
await asyncio.gather(
|
||||
self.engine_client.abort(engine_request_ids),
|
||||
return_exceptions=True,
|
||||
)
|
||||
raise
|
||||
|
||||
separator = asr_inter_chunk_separator(
|
||||
request.language, self.model_cls.no_space_languages
|
||||
@@ -508,10 +530,12 @@ class OpenAISpeechToText(OpenAIServing):
|
||||
separator,
|
||||
)
|
||||
# Non-streaming response.
|
||||
total_segments = []
|
||||
text_parts = []
|
||||
try:
|
||||
assert list_result_generator is not None
|
||||
chunk_segment_parts: list[list[SpeechToTextSegment]] = [
|
||||
[] for _ in list_result_generator
|
||||
]
|
||||
chunk_text_parts: list[list[str]] = [[] for _ in list_result_generator]
|
||||
segments_types: dict[str, type[SpeechToTextSegment]] = {
|
||||
"transcribe": TranscriptionSegment,
|
||||
"translate": TranslationSegment,
|
||||
@@ -522,28 +546,34 @@ class OpenAISpeechToText(OpenAIServing):
|
||||
assert len(list_result_generator) == 1, (
|
||||
"`max_audio_clip_s` is set to None, audio cannot be chunked"
|
||||
)
|
||||
for idx, result_generator in enumerate(list_result_generator):
|
||||
result_generator = merge_async_iterators(*list_result_generator)
|
||||
async for idx, op in result_generator:
|
||||
start_time = (
|
||||
float(idx * chunk_size_in_s) if chunk_size_in_s is not None else 0.0
|
||||
)
|
||||
async for op in result_generator:
|
||||
if request.response_format == "verbose_json":
|
||||
assert op.outputs[0].logprobs
|
||||
segments: list[SpeechToTextSegment] = (
|
||||
self._get_verbose_segments(
|
||||
tokens=tuple(op.outputs[0].token_ids),
|
||||
segment_class=segment_class,
|
||||
request=request,
|
||||
start_time=start_time,
|
||||
log_probs=op.outputs[0].logprobs,
|
||||
)
|
||||
)
|
||||
if request.response_format == "verbose_json":
|
||||
assert op.outputs[0].logprobs
|
||||
segments: list[SpeechToTextSegment] = self._get_verbose_segments(
|
||||
tokens=tuple(op.outputs[0].token_ids),
|
||||
segment_class=segment_class,
|
||||
request=request,
|
||||
start_time=start_time,
|
||||
log_probs=op.outputs[0].logprobs,
|
||||
)
|
||||
|
||||
total_segments.extend(segments)
|
||||
text_parts.extend([seg.text for seg in segments])
|
||||
else:
|
||||
raw_text = op.outputs[0].text
|
||||
text_parts.append(self.model_cls.post_process_output(raw_text))
|
||||
chunk_segment_parts[idx].extend(segments)
|
||||
chunk_text_parts[idx].extend([seg.text for seg in segments])
|
||||
else:
|
||||
raw_text = op.outputs[0].text
|
||||
chunk_text_parts[idx].append(
|
||||
self.model_cls.post_process_output(raw_text)
|
||||
)
|
||||
total_segments = [
|
||||
segment
|
||||
for segment_parts in chunk_segment_parts
|
||||
for segment in segment_parts
|
||||
]
|
||||
text_parts = [text for text_part in chunk_text_parts for text in text_part]
|
||||
text = separator.join(text_parts)
|
||||
if self.task_type == "transcribe":
|
||||
final_response: ResponseType
|
||||
@@ -583,7 +613,16 @@ class OpenAISpeechToText(OpenAIServing):
|
||||
)
|
||||
return final_response
|
||||
except asyncio.CancelledError:
|
||||
return self.create_error_response("Client disconnected")
|
||||
logger.info(
|
||||
"Request %s cancelled; aborting %d transcription engine request(s).",
|
||||
request_id,
|
||||
len(engine_request_ids),
|
||||
)
|
||||
await asyncio.gather(
|
||||
self.engine_client.abort(engine_request_ids),
|
||||
return_exceptions=True,
|
||||
)
|
||||
raise
|
||||
|
||||
async def _speech_to_text_stream_generator(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user