[Bugfix][Model] Voxtral realtime: fix boot OOM and engine crash at max_model_len (#45022)
Signed-off-by: Damien Laine <damien.laine@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Cyrus Leung
parent
2c99ee9333
commit
2186b89a9a
@@ -106,7 +106,8 @@ def tokenizer() -> MistralTokenizer:
|
|||||||
|
|
||||||
|
|
||||||
@pytest_asyncio.fixture
|
@pytest_asyncio.fixture
|
||||||
async def async_engine():
|
async def async_engine(request):
|
||||||
|
engine_config = {**ENGINE_CONFIG, **getattr(request, "param", {})}
|
||||||
gpu_memory_utilization = ENGINE_CONFIG.get("gpu_memory_utilization", 0.9)
|
gpu_memory_utilization = ENGINE_CONFIG.get("gpu_memory_utilization", 0.9)
|
||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
|
|
||||||
@@ -115,7 +116,7 @@ async def async_engine():
|
|||||||
|
|
||||||
wait_for_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
|
wait_for_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
|
||||||
|
|
||||||
engine_args = AsyncEngineArgs(**ENGINE_CONFIG)
|
engine_args = AsyncEngineArgs(**engine_config)
|
||||||
llm = AsyncLLM.from_engine_args(engine_args)
|
llm = AsyncLLM.from_engine_args(engine_args)
|
||||||
try:
|
try:
|
||||||
yield llm
|
yield llm
|
||||||
@@ -275,3 +276,82 @@ async def test_voxtral_realtime_generator(audio_assets, tokenizer, async_engine)
|
|||||||
f" got: {got!r}\n"
|
f" got: {got!r}\n"
|
||||||
f" expected: {expected!r}"
|
f" expected: {expected!r}"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_voxtral_realtime_buffer_stops_at_max_model_len(tokenizer):
|
||||||
|
from vllm.model_executor.models.voxtral_realtime import VoxtralRealtimeBuffer
|
||||||
|
|
||||||
|
audio_config = tokenizer.instruct_tokenizer.audio_encoder.audio_config
|
||||||
|
audio = Audio.from_file(AudioAsset("winning_call").get_local_path(), strict=False)
|
||||||
|
req = TranscriptionRequest(
|
||||||
|
streaming=StreamingMode.OFFLINE,
|
||||||
|
audio=audio.to_base64(audio.format),
|
||||||
|
language=None,
|
||||||
|
)
|
||||||
|
audio_enc = tokenizer.encode_transcription(req)
|
||||||
|
|
||||||
|
max_model_len = 64
|
||||||
|
buffer = VoxtralRealtimeBuffer(
|
||||||
|
audio_config, audio_enc.tokens, max_model_len=max_model_len
|
||||||
|
)
|
||||||
|
await buffer.append_audio(audio_enc.audios[0].audio_array)
|
||||||
|
await buffer.append_audio(None)
|
||||||
|
# Unlimited feedback tokens so only the length cap can stop the stream.
|
||||||
|
await buffer.append_tokens([0] * max_model_len)
|
||||||
|
|
||||||
|
n_tokens = 0
|
||||||
|
async for prompt in buffer.get_input_stream():
|
||||||
|
n_tokens += len(prompt["prompt_token_ids"])
|
||||||
|
|
||||||
|
assert len(audio_enc.tokens) < n_tokens < max_model_len
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.parametrize("async_engine", [{"max_model_len": 128}], indirect=True)
|
||||||
|
async def test_voxtral_realtime_generator_max_model_len(
|
||||||
|
audio_assets, tokenizer, async_engine
|
||||||
|
):
|
||||||
|
# Lazy import to avoid CUDA-reinitialization error
|
||||||
|
from vllm.model_executor.models.voxtral_realtime import VoxtralRealtimeBuffer
|
||||||
|
|
||||||
|
max_model_len = async_engine.model_config.max_model_len
|
||||||
|
sampling_params = SamplingParams(temperature=0.0, max_tokens=1)
|
||||||
|
audio_config = tokenizer.instruct_tokenizer.audio_encoder.audio_config
|
||||||
|
|
||||||
|
async def input_stream(audio_buffer):
|
||||||
|
async for prompt in audio_buffer.get_input_stream():
|
||||||
|
parsed_prompt = parse_model_prompt(async_engine.model_config, prompt)
|
||||||
|
(engine_input,) = await async_engine.renderer.render_cmpl_async(
|
||||||
|
[parsed_prompt]
|
||||||
|
)
|
||||||
|
yield StreamingInput(prompt=engine_input)
|
||||||
|
|
||||||
|
# winning_call is ~300 tokens long, well past max_model_len.
|
||||||
|
audio = Audio.from_file(audio_assets[1].get_local_path(), strict=False)
|
||||||
|
req = TranscriptionRequest(
|
||||||
|
streaming=StreamingMode.OFFLINE,
|
||||||
|
audio=audio.to_base64(audio.format),
|
||||||
|
language=None,
|
||||||
|
)
|
||||||
|
audio_enc = tokenizer.encode_transcription(req)
|
||||||
|
buffer = VoxtralRealtimeBuffer(
|
||||||
|
audio_config, audio_enc.tokens, max_model_len=max_model_len
|
||||||
|
)
|
||||||
|
await buffer.append_audio(audio_enc.audios[0].audio_array)
|
||||||
|
await buffer.append_audio(None)
|
||||||
|
|
||||||
|
output_tokens = []
|
||||||
|
async for resp in async_engine.generate(
|
||||||
|
prompt=input_stream(buffer),
|
||||||
|
sampling_params=sampling_params,
|
||||||
|
request_id="session-max-len",
|
||||||
|
):
|
||||||
|
tokens = resp.outputs[0].token_ids[-1:]
|
||||||
|
output_tokens.extend(tokens)
|
||||||
|
await buffer.append_tokens(tokens)
|
||||||
|
|
||||||
|
assert resp.finished
|
||||||
|
assert 0 < len(output_tokens) < max_model_len
|
||||||
|
# The engine is still healthy after the session hit the cap.
|
||||||
|
assert not async_engine.errored
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import contextlib
|
||||||
import json
|
import json
|
||||||
from collections.abc import AsyncGenerator
|
from collections.abc import AsyncGenerator
|
||||||
from http import HTTPStatus
|
from http import HTTPStatus
|
||||||
@@ -48,6 +49,7 @@ class RealtimeConnection:
|
|||||||
self.serving = serving
|
self.serving = serving
|
||||||
self.audio_queue: asyncio.Queue[np.ndarray | None] = asyncio.Queue()
|
self.audio_queue: asyncio.Queue[np.ndarray | None] = asyncio.Queue()
|
||||||
self.generation_task: asyncio.Task | None = None
|
self.generation_task: asyncio.Task | None = None
|
||||||
|
self.request_id: str | None = None
|
||||||
|
|
||||||
self._is_connected = False
|
self._is_connected = False
|
||||||
self._is_model_validated = False
|
self._is_model_validated = False
|
||||||
@@ -204,6 +206,7 @@ class RealtimeConnection:
|
|||||||
6. Cleans up the audio queue
|
6. Cleans up the audio queue
|
||||||
"""
|
"""
|
||||||
request_id = f"rt-{self.connection_id}-{uuid4()}"
|
request_id = f"rt-{self.connection_id}-{uuid4()}"
|
||||||
|
self.request_id = request_id
|
||||||
full_text = ""
|
full_text = ""
|
||||||
|
|
||||||
prompt_token_ids_len: int = 0
|
prompt_token_ids_len: int = 0
|
||||||
@@ -263,7 +266,9 @@ class RealtimeConnection:
|
|||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.exception("Error in generation: %s", e)
|
logger.exception("Error in generation: %s", e)
|
||||||
await self.send_error(sanitize_message(str(e)), "processing_error")
|
# The socket may already be closed; nothing more we can do.
|
||||||
|
with contextlib.suppress(Exception):
|
||||||
|
await self.send_error(sanitize_message(str(e)), "processing_error")
|
||||||
|
|
||||||
async def send(
|
async def send(
|
||||||
self, event: SessionCreated | TranscriptionDelta | TranscriptionDone
|
self, event: SessionCreated | TranscriptionDelta | TranscriptionDone
|
||||||
@@ -282,8 +287,28 @@ class RealtimeConnection:
|
|||||||
# Signal audio stream to stop
|
# Signal audio stream to stop
|
||||||
self.audio_queue.put_nowait(None)
|
self.audio_queue.put_nowait(None)
|
||||||
|
|
||||||
# Cancel generation task if running
|
# Abort the engine request. Cancelling the asyncio task alone does NOT
|
||||||
|
# abort the underlying engine request: a request parked in the
|
||||||
|
# VAD-waiting state keeps occupying a slot and generating into a now
|
||||||
|
# dead websocket (an orphaned request that never frees and starves
|
||||||
|
# other streams). Aborting also unblocks the generation task, which is
|
||||||
|
# otherwise stuck awaiting the next engine output.
|
||||||
|
if self.request_id is not None:
|
||||||
|
try:
|
||||||
|
await self.serving.engine_client.abort(self.request_id)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Failed to abort request %s", self.request_id)
|
||||||
|
self.request_id = None
|
||||||
|
|
||||||
|
# Cancel the generation task if still running, and await it so the
|
||||||
|
# cancellation (and any generator cleanup) actually completes.
|
||||||
if self.generation_task and not self.generation_task.done():
|
if self.generation_task and not self.generation_task.done():
|
||||||
self.generation_task.cancel()
|
self.generation_task.cancel()
|
||||||
|
try:
|
||||||
|
await self.generation_task
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
pass
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Error awaiting cancelled generation task")
|
||||||
|
|
||||||
logger.debug("Connection cleanup complete: %s", self.connection_id)
|
logger.debug("Connection cleanup complete: %s", self.connection_id)
|
||||||
|
|||||||
@@ -120,9 +120,35 @@ def _expand_tensor(input_tensor: torch.Tensor, scaling: int) -> torch.Tensor:
|
|||||||
return (base.unsqueeze(1) + offsets).view(-1)
|
return (base.unsqueeze(1) + offsets).view(-1)
|
||||||
|
|
||||||
|
|
||||||
|
class VoxtralRealtimeProcessingInfo(VoxtralProcessingInfo):
|
||||||
|
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||||
|
# Each streaming update carries a single audio frame.
|
||||||
|
return {"audio": 1}
|
||||||
|
|
||||||
|
def get_max_audio_array_len(self) -> int:
|
||||||
|
# Profile with the largest streaming chunk, the first one: it covers
|
||||||
|
# the prompt prefix (BOS + left padding + delay), later chunks are
|
||||||
|
# ~1 token. get_max_audio_tokens keeps max_model_len so the encoder
|
||||||
|
# cache stays as large as before: each paused session keeps its last
|
||||||
|
# chunk cached until it resumes, and a cache sized to one chunk
|
||||||
|
# deadlocks a few concurrent sessions.
|
||||||
|
tokenizer = self.get_tokenizer()
|
||||||
|
audio_encoder = tokenizer.instruct.audio_encoder
|
||||||
|
prompt_tokens = (
|
||||||
|
tokenizer.instruct.start() + audio_encoder.encode_streaming_tokens()
|
||||||
|
)
|
||||||
|
return len(prompt_tokens) * audio_encoder.audio_config.raw_audio_length_per_tok
|
||||||
|
|
||||||
|
|
||||||
class VoxtralRealtimeBuffer:
|
class VoxtralRealtimeBuffer:
|
||||||
def __init__(self, config: AudioConfig, prompt_tokens: list[int]) -> None:
|
def __init__(
|
||||||
|
self,
|
||||||
|
config: AudioConfig,
|
||||||
|
prompt_tokens: list[int],
|
||||||
|
max_model_len: int | None = None,
|
||||||
|
) -> None:
|
||||||
self._config = config
|
self._config = config
|
||||||
|
self._max_model_len = max_model_len
|
||||||
|
|
||||||
_look_ahead_in_ms = self._config.streaming_look_ahead_ms
|
_look_ahead_in_ms = self._config.streaming_look_ahead_ms
|
||||||
_look_back_in_ms = self._config.streaming_look_back_ms
|
_look_back_in_ms = self._config.streaming_look_back_ms
|
||||||
@@ -165,7 +191,16 @@ class VoxtralRealtimeBuffer:
|
|||||||
await self._token_queue.put(token)
|
await self._token_queue.put(token)
|
||||||
|
|
||||||
async def get_input_stream(self) -> AsyncGenerator[TokensPrompt]:
|
async def get_input_stream(self) -> AsyncGenerator[TokensPrompt]:
|
||||||
|
n_tokens = 0
|
||||||
for frame_size, num_tokens in self._generate_frame_size_and_num_tokens():
|
for frame_size, num_tokens in self._generate_frame_size_and_num_tokens():
|
||||||
|
n_tokens += num_tokens
|
||||||
|
if self._max_model_len is not None and n_tokens >= self._max_model_len:
|
||||||
|
logger.warning(
|
||||||
|
"Stopping realtime audio stream: reached max_model_len (%d).",
|
||||||
|
self._max_model_len,
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
next_tokens = [await self._token_queue.get() for _ in range(num_tokens)]
|
next_tokens = [await self._token_queue.get() for _ in range(num_tokens)]
|
||||||
|
|
||||||
audio_arrays: list[np.ndarray] = (
|
audio_arrays: list[np.ndarray] = (
|
||||||
@@ -198,7 +233,7 @@ class VoxtralRealtimeBuffer:
|
|||||||
|
|
||||||
@MULTIMODAL_REGISTRY.register_processor(
|
@MULTIMODAL_REGISTRY.register_processor(
|
||||||
VoxtralRealtimeMultiModalProcessor,
|
VoxtralRealtimeMultiModalProcessor,
|
||||||
info=VoxtralProcessingInfo,
|
info=VoxtralRealtimeProcessingInfo,
|
||||||
dummy_inputs=VoxtralDummyInputsBuilder,
|
dummy_inputs=VoxtralDummyInputsBuilder,
|
||||||
)
|
)
|
||||||
@support_torch_compile
|
@support_torch_compile
|
||||||
@@ -248,7 +283,9 @@ class VoxtralRealtimeGeneration(VoxtralForConditionalGeneration, SupportsRealtim
|
|||||||
# Get left/right padding audio
|
# Get left/right padding audio
|
||||||
left_pad, right_pad = audio_encoder.get_padding_audio()
|
left_pad, right_pad = audio_encoder.get_padding_audio()
|
||||||
|
|
||||||
buffer = VoxtralRealtimeBuffer(config, prompt_tokens)
|
buffer = VoxtralRealtimeBuffer(
|
||||||
|
config, prompt_tokens, max_model_len=model_config.max_model_len
|
||||||
|
)
|
||||||
|
|
||||||
# Feed audio with padding into buffer in background
|
# Feed audio with padding into buffer in background
|
||||||
async def feed_audio():
|
async def feed_audio():
|
||||||
|
|||||||
Reference in New Issue
Block a user