[Bugfix][Model] Voxtral realtime: fix boot OOM and engine crash at max_model_len (#45022)
Signed-off-by: Damien Laine <damien.laine@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Cyrus Leung
parent
2c99ee9333
commit
2186b89a9a
@@ -120,9 +120,35 @@ def _expand_tensor(input_tensor: torch.Tensor, scaling: int) -> torch.Tensor:
|
||||
return (base.unsqueeze(1) + offsets).view(-1)
|
||||
|
||||
|
||||
class VoxtralRealtimeProcessingInfo(VoxtralProcessingInfo):
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
# Each streaming update carries a single audio frame.
|
||||
return {"audio": 1}
|
||||
|
||||
def get_max_audio_array_len(self) -> int:
|
||||
# Profile with the largest streaming chunk, the first one: it covers
|
||||
# the prompt prefix (BOS + left padding + delay), later chunks are
|
||||
# ~1 token. get_max_audio_tokens keeps max_model_len so the encoder
|
||||
# cache stays as large as before: each paused session keeps its last
|
||||
# chunk cached until it resumes, and a cache sized to one chunk
|
||||
# deadlocks a few concurrent sessions.
|
||||
tokenizer = self.get_tokenizer()
|
||||
audio_encoder = tokenizer.instruct.audio_encoder
|
||||
prompt_tokens = (
|
||||
tokenizer.instruct.start() + audio_encoder.encode_streaming_tokens()
|
||||
)
|
||||
return len(prompt_tokens) * audio_encoder.audio_config.raw_audio_length_per_tok
|
||||
|
||||
|
||||
class VoxtralRealtimeBuffer:
|
||||
def __init__(self, config: AudioConfig, prompt_tokens: list[int]) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
config: AudioConfig,
|
||||
prompt_tokens: list[int],
|
||||
max_model_len: int | None = None,
|
||||
) -> None:
|
||||
self._config = config
|
||||
self._max_model_len = max_model_len
|
||||
|
||||
_look_ahead_in_ms = self._config.streaming_look_ahead_ms
|
||||
_look_back_in_ms = self._config.streaming_look_back_ms
|
||||
@@ -165,7 +191,16 @@ class VoxtralRealtimeBuffer:
|
||||
await self._token_queue.put(token)
|
||||
|
||||
async def get_input_stream(self) -> AsyncGenerator[TokensPrompt]:
|
||||
n_tokens = 0
|
||||
for frame_size, num_tokens in self._generate_frame_size_and_num_tokens():
|
||||
n_tokens += num_tokens
|
||||
if self._max_model_len is not None and n_tokens >= self._max_model_len:
|
||||
logger.warning(
|
||||
"Stopping realtime audio stream: reached max_model_len (%d).",
|
||||
self._max_model_len,
|
||||
)
|
||||
return
|
||||
|
||||
next_tokens = [await self._token_queue.get() for _ in range(num_tokens)]
|
||||
|
||||
audio_arrays: list[np.ndarray] = (
|
||||
@@ -198,7 +233,7 @@ class VoxtralRealtimeBuffer:
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
VoxtralRealtimeMultiModalProcessor,
|
||||
info=VoxtralProcessingInfo,
|
||||
info=VoxtralRealtimeProcessingInfo,
|
||||
dummy_inputs=VoxtralDummyInputsBuilder,
|
||||
)
|
||||
@support_torch_compile
|
||||
@@ -248,7 +283,9 @@ class VoxtralRealtimeGeneration(VoxtralForConditionalGeneration, SupportsRealtim
|
||||
# Get left/right padding audio
|
||||
left_pad, right_pad = audio_encoder.get_padding_audio()
|
||||
|
||||
buffer = VoxtralRealtimeBuffer(config, prompt_tokens)
|
||||
buffer = VoxtralRealtimeBuffer(
|
||||
config, prompt_tokens, max_model_len=model_config.max_model_len
|
||||
)
|
||||
|
||||
# Feed audio with padding into buffer in background
|
||||
async def feed_audio():
|
||||
|
||||
Reference in New Issue
Block a user