[Bugfix][Model] Voxtral realtime: fix boot OOM and engine crash at max_model_len (#45022)
Signed-off-by: Damien Laine <damien.laine@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Cyrus Leung
parent
2c99ee9333
commit
2186b89a9a
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import json
|
||||
from collections.abc import AsyncGenerator
|
||||
from http import HTTPStatus
|
||||
@@ -48,6 +49,7 @@ class RealtimeConnection:
|
||||
self.serving = serving
|
||||
self.audio_queue: asyncio.Queue[np.ndarray | None] = asyncio.Queue()
|
||||
self.generation_task: asyncio.Task | None = None
|
||||
self.request_id: str | None = None
|
||||
|
||||
self._is_connected = False
|
||||
self._is_model_validated = False
|
||||
@@ -204,6 +206,7 @@ class RealtimeConnection:
|
||||
6. Cleans up the audio queue
|
||||
"""
|
||||
request_id = f"rt-{self.connection_id}-{uuid4()}"
|
||||
self.request_id = request_id
|
||||
full_text = ""
|
||||
|
||||
prompt_token_ids_len: int = 0
|
||||
@@ -263,7 +266,9 @@ class RealtimeConnection:
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("Error in generation: %s", e)
|
||||
await self.send_error(sanitize_message(str(e)), "processing_error")
|
||||
# The socket may already be closed; nothing more we can do.
|
||||
with contextlib.suppress(Exception):
|
||||
await self.send_error(sanitize_message(str(e)), "processing_error")
|
||||
|
||||
async def send(
|
||||
self, event: SessionCreated | TranscriptionDelta | TranscriptionDone
|
||||
@@ -282,8 +287,28 @@ class RealtimeConnection:
|
||||
# Signal audio stream to stop
|
||||
self.audio_queue.put_nowait(None)
|
||||
|
||||
# Cancel generation task if running
|
||||
# Abort the engine request. Cancelling the asyncio task alone does NOT
|
||||
# abort the underlying engine request: a request parked in the
|
||||
# VAD-waiting state keeps occupying a slot and generating into a now
|
||||
# dead websocket (an orphaned request that never frees and starves
|
||||
# other streams). Aborting also unblocks the generation task, which is
|
||||
# otherwise stuck awaiting the next engine output.
|
||||
if self.request_id is not None:
|
||||
try:
|
||||
await self.serving.engine_client.abort(self.request_id)
|
||||
except Exception:
|
||||
logger.exception("Failed to abort request %s", self.request_id)
|
||||
self.request_id = None
|
||||
|
||||
# Cancel the generation task if still running, and await it so the
|
||||
# cancellation (and any generator cleanup) actually completes.
|
||||
if self.generation_task and not self.generation_task.done():
|
||||
self.generation_task.cancel()
|
||||
try:
|
||||
await self.generation_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception:
|
||||
logger.exception("Error awaiting cancelled generation task")
|
||||
|
||||
logger.debug("Connection cleanup complete: %s", self.connection_id)
|
||||
|
||||
Reference in New Issue
Block a user