[Model] Voxtral Realtime: add support for CUDAGraphMode.FULL_DECODE_ONLY (#51167)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
Co-authored-by: Jorge Gallego Feliciano <jorge.gallego@mistral.ai>
This commit is contained in:
Nicolò Lucchesi
2026-09-14 19:16:39 +00:00
committed by GitHub
co-authored by Jorge Gallego Feliciano
parent 0a5747d410
commit 995e8581f4
3 changed files with 174 additions and 62 deletions
@@ -14,7 +14,9 @@ from mistral_common.tokens.tokenizers.tekken import SpecialTokenPolicy
from vllm import LLM, SamplingParams
from vllm.assets.audio import AudioAsset
from vllm.config import CUDAGraphMode
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.platforms import current_platform
from vllm.utils.math_utils import cdiv
from vllm.v1.engine.async_llm import AsyncLLM
from vllm.v1.kv_cache_interface import SlidingWindowSpec
@@ -130,6 +132,45 @@ async def async_engine():
wait_for_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
def _build_inputs(
audio_assets: list[AudioAsset], tokenizer: MistralTokenizer
) -> tuple[list[dict], list[SamplingParams]]:
audio_config = tokenizer.instruct_tokenizer.tokenizer.audio
inputs = []
sampling_params = []
for audio_asset in audio_assets:
audio = Audio.from_file(audio_asset.get_local_path(), strict=False)
req = TranscriptionRequest(
audio=audio.to_base64(audio.format),
streaming=StreamingMode.OFFLINE,
language=None,
)
tokenized = tokenizer.instruct_tokenizer.encode_transcription(req)
tokens = tokenized.tokens
audio_array = tokenized.audios[0].audio_array
num_samples = audio_array.shape[0]
max_tokens = audio_config.num_audio_tokens(num_samples) - len(tokens) - 1
sampling_params.append(SamplingParams(temperature=0.0, max_tokens=max_tokens))
inputs.append(
{
"multi_modal_data": {"audio": [(audio_array, None)]},
"prompt_token_ids": tokens,
}
)
return inputs, sampling_params
def _assert_expected_text(outputs) -> None:
texts = _normalize([out.outputs[0].text for out in outputs])
for i, (got, expected) in enumerate(zip(texts, EXPECTED_TEXT)):
assert got == expected, (
f"Output mismatch at index {i}:\n"
f" got: {got!r}\n"
f" expected: {expected!r}"
)
def test_voxtral_realtime_forward(audio_assets, tokenizer, vllm_runner, monkeypatch):
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
@@ -138,51 +179,38 @@ def test_voxtral_realtime_forward(audio_assets, tokenizer, vllm_runner, monkeypa
with vllm_runner(**vllm_kwargs) as vllm_model:
assert_encoder_kv_cache_spec(vllm_model.llm)
audio_config = tokenizer.instruct_tokenizer.tokenizer.audio
inputs, sampling_params = _build_inputs(audio_assets, tokenizer)
def from_file(file_path: str):
audio = Audio.from_file(file_path, strict=False)
req = TranscriptionRequest(
audio=audio.to_base64(audio.format),
streaming=StreamingMode.OFFLINE,
language=None,
)
tokenized = tokenizer.instruct_tokenizer.encode_transcription(req)
outputs = vllm_model.llm.generate(inputs, sampling_params=sampling_params)
_assert_expected_text(outputs)
return (tokenized.tokens, tokenized.audios[0].audio_array)
tokenized_list = [
from_file(audio_asset.get_local_path()) for audio_asset in audio_assets
]
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="Full cudagraphs require CUDA"
)
@pytest.mark.parametrize(
"cudagraph_mode",
[CUDAGraphMode.FULL_DECODE_ONLY, CUDAGraphMode.FULL_AND_PIECEWISE],
)
def test_voxtral_realtime_cudagraph(
audio_assets, tokenizer, vllm_runner, monkeypatch, cudagraph_mode
):
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
inputs = []
sampling_params = []
vllm_kwargs = {**ENGINE_CONFIG}
vllm_kwargs["model_name"] = vllm_kwargs.pop("model")
vllm_kwargs["enforce_eager"] = False
vllm_kwargs["compilation_config"] = {
"cudagraph_mode": cudagraph_mode,
# max_num_seqs is 4, so this is the only decode size worth capturing.
"cudagraph_capture_sizes": [4],
}
for tokens, audio_array in tokenized_list:
num_samples = audio_array.shape[0]
max_tokens = audio_config.num_audio_tokens(num_samples) - len(tokens) - 1
sampling_params.append(
SamplingParams(temperature=0.0, max_tokens=max_tokens)
)
with vllm_runner(**vllm_kwargs) as vllm_model:
inputs, sampling_params = _build_inputs(audio_assets, tokenizer)
input_dict = {
"multi_modal_data": {"audio": [(audio_array, None)]},
"prompt_token_ids": tokens,
}
inputs.append(input_dict)
outputs = vllm_model.llm.generate(
inputs,
sampling_params=sampling_params,
)
texts = _normalize([out.outputs[0].text for out in outputs])
for i, (got, expected) in enumerate(zip(texts, EXPECTED_TEXT)):
assert got == expected, (
f"Output mismatch at index {i}:\n"
f" got: {got!r}\n"
f" expected: {expected!r}"
)
outputs = vllm_model.llm.generate(inputs, sampling_params=sampling_params)
_assert_expected_text(outputs)
@pytest.mark.asyncio
@@ -217,9 +217,15 @@ class VoxtralRealtimeGeneration(VoxtralForConditionalGeneration, SupportsRealtim
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__(vllm_config=vllm_config, prefix=prefix)
assert (
not vllm_config.compilation_config.cudagraph_mode.has_full_cudagraphs()
), "Voxtral realtime doesn't support full cudagraphs yet. Please use PIECEWISE."
# Full cudagraphs are supported for decode-only batches (the encoder's
# block-pooling attention builder is capture-safe for uniform
# single-token decode). Mixed/prefill batches are not, so block only
# pure FULL; FULL_DECODE_ONLY and FULL_AND_PIECEWISE are allowed.
cudagraph_mode = vllm_config.compilation_config.cudagraph_mode
assert not cudagraph_mode.mixed_mode().has_full_cudagraphs(), (
"Voxtral realtime supports full cudagraphs for decode-only batches. "
"Use cudagraph_mode=FULL_DECODE_ONLY or FULL_AND_PIECEWISE (not FULL)."
)
self.time_embedding: TimeEmbedding = TimeEmbedding(
dim=self.config.text_config.hidden_size
+98 -20
View File
@@ -1,12 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
import functools
import logging
import math
from dataclasses import replace
from fractions import Fraction
from functools import partial
from typing import ClassVar
import torch
import torch.nn.functional as F
@@ -27,6 +27,7 @@ from vllm.model_executor.models.whisper import WhisperPosEmbedType
from vllm.utils.math_utils import cdiv
from vllm.v1.attention.backend import (
AttentionBackend,
AttentionCGSupport,
AttentionMetadata,
AttentionType,
CommonAttentionMetadata,
@@ -123,6 +124,16 @@ def create_whisper_attention_backend_with_block_pooling(
underlying_impl = underlying_attn_backend.get_impl_cls()
class WhisperCausalAttentionWithBlockPoolingBuilder(underlying_builder): # type: ignore
# Full cudagraphs only for uniform single-token decode: capture bakes in
# tensor addresses, so `build` writes metadata into persistent buffers
# (below). Prefill/mixed batches fall back to piecewise.
_cudagraph_support: ClassVar[AttentionCGSupport] = (
AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
)
# Disable the update_block_table fast path: it would bypass block pooling
# by splicing the raw slot_mapping/block_table into cached metadata.
supports_update_block_table: bool = False
def __init__(
self,
kv_cache_spec: AttentionSpec,
@@ -150,6 +161,38 @@ def create_whisper_attention_backend_with_block_pooling(
# num_heads_q for the encoder is the same as num_kv_heads
# (no GQA in whisper encoder)
self.num_heads_q = kv_cache_spec.num_kv_heads
# Persistent capture-safe buffers for pooled metadata; lazily sized
# on first `build` to match the incoming tensors' dtype/device.
self._pool_buffers_ready = False
def _maybe_init_pool_buffers(
self, common_attn_metadata: CommonAttentionMetadata
) -> None:
if self._pool_buffers_ready:
return
device = common_attn_metadata.slot_mapping.device
sched = self.vllm_config.scheduler_config
self._qsl_buf = torch.empty(
sched.max_num_seqs + 1,
dtype=common_attn_metadata.query_start_loc.dtype,
device=device,
)
self._seq_buf = torch.empty(
sched.max_num_seqs,
dtype=common_attn_metadata.seq_lens.dtype,
device=device,
)
self._slot_buf = torch.empty(
sched.max_num_batched_tokens * block_pool_size,
dtype=common_attn_metadata.slot_mapping.dtype,
device=device,
)
self._pool_arange = torch.arange(
block_pool_size,
dtype=common_attn_metadata.slot_mapping.dtype,
device=device,
)
self._pool_buffers_ready = True
def build(
self,
@@ -157,27 +200,62 @@ def create_whisper_attention_backend_with_block_pooling(
common_attn_metadata: CommonAttentionMetadata,
fast_build: bool = False,
) -> AttentionMetadata:
new_common_attn_metadata = copy.deepcopy(common_attn_metadata)
new_common_attn_metadata.query_start_loc *= block_pool_size
new_common_attn_metadata.query_start_loc_cpu *= block_pool_size
new_common_attn_metadata.seq_lens *= block_pool_size
if new_common_attn_metadata.seq_lens_cpu_upper_bound is not None:
new_common_attn_metadata.seq_lens_cpu_upper_bound *= block_pool_size
new_common_attn_metadata.num_actual_tokens *= block_pool_size
new_common_attn_metadata.max_query_len *= block_pool_size
new_common_attn_metadata.max_seq_len *= block_pool_size
original_slot_mapping = common_attn_metadata.slot_mapping
common_prefix_len *= block_pool_size
new_common_attn_metadata.slot_mapping = (
(
original_slot_mapping.unsqueeze(1) * block_pool_size
+ torch.arange(block_pool_size, device=original_slot_mapping.device)
)
.flatten()
.clamp(min=-1)
cm = common_attn_metadata
self._maybe_init_pool_buffers(cm)
# Scale query_start_loc / seq_lens into the persistent buffers.
n = cm.query_start_loc.numel()
assert n <= self._qsl_buf.numel(), (
f"{n=} exceeds pooled query_start_loc buffer {self._qsl_buf.numel()}"
)
qsl_buf = self._qsl_buf[:n]
qsl_buf.copy_(cm.query_start_loc)
qsl_buf.mul_(block_pool_size)
r = cm.seq_lens.numel()
assert r <= self._seq_buf.numel(), (
f"{r=} exceeds pooled seq_lens buffer {self._seq_buf.numel()}"
)
seq_buf = self._seq_buf[:r]
seq_buf.copy_(cm.seq_lens)
seq_buf.mul_(block_pool_size)
# Expand each slot into block_pool_size contiguous slots. Padding
# slots (-1) stay -1 after clamp so the KV write is skipped.
m = cm.slot_mapping.numel()
assert m * block_pool_size <= self._slot_buf.numel(), (
f"pooled slot_mapping {m * block_pool_size} exceeds buffer "
f"{self._slot_buf.numel()}"
)
slot_buf = self._slot_buf[: m * block_pool_size]
slot_buf.view(m, block_pool_size).copy_(
cm.slot_mapping.unsqueeze(1) * block_pool_size + self._pool_arange
)
slot_buf.clamp_(min=-1)
# CPU tensors aren't captured; fresh scaled copies avoid mutating
# the runner's shared buffers.
qsl_cpu = cm.query_start_loc_cpu * block_pool_size
seq_lens_cpu_upper_bound = (
cm.seq_lens_cpu_upper_bound * block_pool_size
if cm.seq_lens_cpu_upper_bound is not None
else None
)
new_common_attn_metadata = cm.replace(
query_start_loc=qsl_buf,
query_start_loc_cpu=qsl_cpu,
seq_lens=seq_buf,
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
num_actual_tokens=cm.num_actual_tokens * block_pool_size,
max_query_len=cm.max_query_len * block_pool_size,
max_seq_len=cm.max_seq_len * block_pool_size,
slot_mapping=slot_buf,
)
return super().build(
common_prefix_len, new_common_attn_metadata, fast_build
common_prefix_len * block_pool_size,
new_common_attn_metadata,
fast_build,
)
# NOTE: We need a custom impl so we can use the transformed slot_mapping