[Bugfix][Model] Voxtral realtime: fix boot OOM and engine crash at max_model_len (#45022)

Signed-off-by: Damien Laine <damien.laine@gmail.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Damien Laine
2026-10-09 17:40:29 +08:00
committed by GitHub
co-authored by Cyrus Leung
parent 2c99ee9333
commit 2186b89a9a
3 changed files with 149 additions and 7 deletions
+40 -3
View File
@@ -120,9 +120,35 @@ def _expand_tensor(input_tensor: torch.Tensor, scaling: int) -> torch.Tensor:
return (base.unsqueeze(1) + offsets).view(-1)
class VoxtralRealtimeProcessingInfo(VoxtralProcessingInfo):
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
# Each streaming update carries a single audio frame.
return {"audio": 1}
def get_max_audio_array_len(self) -> int:
# Profile with the largest streaming chunk, the first one: it covers
# the prompt prefix (BOS + left padding + delay), later chunks are
# ~1 token. get_max_audio_tokens keeps max_model_len so the encoder
# cache stays as large as before: each paused session keeps its last
# chunk cached until it resumes, and a cache sized to one chunk
# deadlocks a few concurrent sessions.
tokenizer = self.get_tokenizer()
audio_encoder = tokenizer.instruct.audio_encoder
prompt_tokens = (
tokenizer.instruct.start() + audio_encoder.encode_streaming_tokens()
)
return len(prompt_tokens) * audio_encoder.audio_config.raw_audio_length_per_tok
class VoxtralRealtimeBuffer:
def __init__(self, config: AudioConfig, prompt_tokens: list[int]) -> None:
def __init__(
self,
config: AudioConfig,
prompt_tokens: list[int],
max_model_len: int | None = None,
) -> None:
self._config = config
self._max_model_len = max_model_len
_look_ahead_in_ms = self._config.streaming_look_ahead_ms
_look_back_in_ms = self._config.streaming_look_back_ms
@@ -165,7 +191,16 @@ class VoxtralRealtimeBuffer:
await self._token_queue.put(token)
async def get_input_stream(self) -> AsyncGenerator[TokensPrompt]:
n_tokens = 0
for frame_size, num_tokens in self._generate_frame_size_and_num_tokens():
n_tokens += num_tokens
if self._max_model_len is not None and n_tokens >= self._max_model_len:
logger.warning(
"Stopping realtime audio stream: reached max_model_len (%d).",
self._max_model_len,
)
return
next_tokens = [await self._token_queue.get() for _ in range(num_tokens)]
audio_arrays: list[np.ndarray] = (
@@ -198,7 +233,7 @@ class VoxtralRealtimeBuffer:
@MULTIMODAL_REGISTRY.register_processor(
VoxtralRealtimeMultiModalProcessor,
info=VoxtralProcessingInfo,
info=VoxtralRealtimeProcessingInfo,
dummy_inputs=VoxtralDummyInputsBuilder,
)
@support_torch_compile
@@ -248,7 +283,9 @@ class VoxtralRealtimeGeneration(VoxtralForConditionalGeneration, SupportsRealtim
# Get left/right padding audio
left_pad, right_pad = audio_encoder.get_padding_audio()
buffer = VoxtralRealtimeBuffer(config, prompt_tokens)
buffer = VoxtralRealtimeBuffer(
config, prompt_tokens, max_model_len=model_config.max_model_len
)
# Feed audio with padding into buffer in background
async def feed_audio():