From 995e8581f462a13e32f30cfba946c63d48cf31d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Lucchesi?= Date: Mon, 14 Sep 2026 21:16:39 +0200 Subject: [PATCH] [Model] Voxtral Realtime: add support for `CUDAGraphMode.FULL_DECODE_ONLY` (#51167) Signed-off-by: NickLucche Co-authored-by: Jorge Gallego Feliciano --- .../generation/test_voxtral_realtime.py | 106 ++++++++++------ .../model_executor/models/voxtral_realtime.py | 12 +- vllm/model_executor/models/whisper_causal.py | 118 +++++++++++++++--- 3 files changed, 174 insertions(+), 62 deletions(-) diff --git a/tests/models/multimodal/generation/test_voxtral_realtime.py b/tests/models/multimodal/generation/test_voxtral_realtime.py index 0ef3c33aeb..1bd9f62303 100644 --- a/tests/models/multimodal/generation/test_voxtral_realtime.py +++ b/tests/models/multimodal/generation/test_voxtral_realtime.py @@ -14,7 +14,9 @@ from mistral_common.tokens.tokenizers.tekken import SpecialTokenPolicy from vllm import LLM, SamplingParams from vllm.assets.audio import AudioAsset +from vllm.config import CUDAGraphMode from vllm.engine.arg_utils import AsyncEngineArgs +from vllm.platforms import current_platform from vllm.utils.math_utils import cdiv from vllm.v1.engine.async_llm import AsyncLLM from vllm.v1.kv_cache_interface import SlidingWindowSpec @@ -130,6 +132,45 @@ async def async_engine(): wait_for_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization) +def _build_inputs( + audio_assets: list[AudioAsset], tokenizer: MistralTokenizer +) -> tuple[list[dict], list[SamplingParams]]: + audio_config = tokenizer.instruct_tokenizer.tokenizer.audio + inputs = [] + sampling_params = [] + for audio_asset in audio_assets: + audio = Audio.from_file(audio_asset.get_local_path(), strict=False) + req = TranscriptionRequest( + audio=audio.to_base64(audio.format), + streaming=StreamingMode.OFFLINE, + language=None, + ) + tokenized = tokenizer.instruct_tokenizer.encode_transcription(req) + tokens = tokenized.tokens + audio_array = tokenized.audios[0].audio_array + + num_samples = audio_array.shape[0] + max_tokens = audio_config.num_audio_tokens(num_samples) - len(tokens) - 1 + sampling_params.append(SamplingParams(temperature=0.0, max_tokens=max_tokens)) + inputs.append( + { + "multi_modal_data": {"audio": [(audio_array, None)]}, + "prompt_token_ids": tokens, + } + ) + return inputs, sampling_params + + +def _assert_expected_text(outputs) -> None: + texts = _normalize([out.outputs[0].text for out in outputs]) + for i, (got, expected) in enumerate(zip(texts, EXPECTED_TEXT)): + assert got == expected, ( + f"Output mismatch at index {i}:\n" + f" got: {got!r}\n" + f" expected: {expected!r}" + ) + + def test_voxtral_realtime_forward(audio_assets, tokenizer, vllm_runner, monkeypatch): monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0") @@ -138,51 +179,38 @@ def test_voxtral_realtime_forward(audio_assets, tokenizer, vllm_runner, monkeypa with vllm_runner(**vllm_kwargs) as vllm_model: assert_encoder_kv_cache_spec(vllm_model.llm) - audio_config = tokenizer.instruct_tokenizer.tokenizer.audio + inputs, sampling_params = _build_inputs(audio_assets, tokenizer) - def from_file(file_path: str): - audio = Audio.from_file(file_path, strict=False) - req = TranscriptionRequest( - audio=audio.to_base64(audio.format), - streaming=StreamingMode.OFFLINE, - language=None, - ) - tokenized = tokenizer.instruct_tokenizer.encode_transcription(req) + outputs = vllm_model.llm.generate(inputs, sampling_params=sampling_params) + _assert_expected_text(outputs) - return (tokenized.tokens, tokenized.audios[0].audio_array) - tokenized_list = [ - from_file(audio_asset.get_local_path()) for audio_asset in audio_assets - ] +@pytest.mark.skipif( + not current_platform.is_cuda(), reason="Full cudagraphs require CUDA" +) +@pytest.mark.parametrize( + "cudagraph_mode", + [CUDAGraphMode.FULL_DECODE_ONLY, CUDAGraphMode.FULL_AND_PIECEWISE], +) +def test_voxtral_realtime_cudagraph( + audio_assets, tokenizer, vllm_runner, monkeypatch, cudagraph_mode +): + monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0") - inputs = [] - sampling_params = [] + vllm_kwargs = {**ENGINE_CONFIG} + vllm_kwargs["model_name"] = vllm_kwargs.pop("model") + vllm_kwargs["enforce_eager"] = False + vllm_kwargs["compilation_config"] = { + "cudagraph_mode": cudagraph_mode, + # max_num_seqs is 4, so this is the only decode size worth capturing. + "cudagraph_capture_sizes": [4], + } - for tokens, audio_array in tokenized_list: - num_samples = audio_array.shape[0] - max_tokens = audio_config.num_audio_tokens(num_samples) - len(tokens) - 1 - sampling_params.append( - SamplingParams(temperature=0.0, max_tokens=max_tokens) - ) + with vllm_runner(**vllm_kwargs) as vllm_model: + inputs, sampling_params = _build_inputs(audio_assets, tokenizer) - input_dict = { - "multi_modal_data": {"audio": [(audio_array, None)]}, - "prompt_token_ids": tokens, - } - inputs.append(input_dict) - - outputs = vllm_model.llm.generate( - inputs, - sampling_params=sampling_params, - ) - - texts = _normalize([out.outputs[0].text for out in outputs]) - for i, (got, expected) in enumerate(zip(texts, EXPECTED_TEXT)): - assert got == expected, ( - f"Output mismatch at index {i}:\n" - f" got: {got!r}\n" - f" expected: {expected!r}" - ) + outputs = vllm_model.llm.generate(inputs, sampling_params=sampling_params) + _assert_expected_text(outputs) @pytest.mark.asyncio diff --git a/vllm/model_executor/models/voxtral_realtime.py b/vllm/model_executor/models/voxtral_realtime.py index 738185a7ae..961aa3af9c 100644 --- a/vllm/model_executor/models/voxtral_realtime.py +++ b/vllm/model_executor/models/voxtral_realtime.py @@ -217,9 +217,15 @@ class VoxtralRealtimeGeneration(VoxtralForConditionalGeneration, SupportsRealtim def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__(vllm_config=vllm_config, prefix=prefix) - assert ( - not vllm_config.compilation_config.cudagraph_mode.has_full_cudagraphs() - ), "Voxtral realtime doesn't support full cudagraphs yet. Please use PIECEWISE." + # Full cudagraphs are supported for decode-only batches (the encoder's + # block-pooling attention builder is capture-safe for uniform + # single-token decode). Mixed/prefill batches are not, so block only + # pure FULL; FULL_DECODE_ONLY and FULL_AND_PIECEWISE are allowed. + cudagraph_mode = vllm_config.compilation_config.cudagraph_mode + assert not cudagraph_mode.mixed_mode().has_full_cudagraphs(), ( + "Voxtral realtime supports full cudagraphs for decode-only batches. " + "Use cudagraph_mode=FULL_DECODE_ONLY or FULL_AND_PIECEWISE (not FULL)." + ) self.time_embedding: TimeEmbedding = TimeEmbedding( dim=self.config.text_config.hidden_size diff --git a/vllm/model_executor/models/whisper_causal.py b/vllm/model_executor/models/whisper_causal.py index 7004bba283..ec84cae8cf 100644 --- a/vllm/model_executor/models/whisper_causal.py +++ b/vllm/model_executor/models/whisper_causal.py @@ -1,12 +1,12 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import copy import functools import logging import math from dataclasses import replace from fractions import Fraction from functools import partial +from typing import ClassVar import torch import torch.nn.functional as F @@ -27,6 +27,7 @@ from vllm.model_executor.models.whisper import WhisperPosEmbedType from vllm.utils.math_utils import cdiv from vllm.v1.attention.backend import ( AttentionBackend, + AttentionCGSupport, AttentionMetadata, AttentionType, CommonAttentionMetadata, @@ -123,6 +124,16 @@ def create_whisper_attention_backend_with_block_pooling( underlying_impl = underlying_attn_backend.get_impl_cls() class WhisperCausalAttentionWithBlockPoolingBuilder(underlying_builder): # type: ignore + # Full cudagraphs only for uniform single-token decode: capture bakes in + # tensor addresses, so `build` writes metadata into persistent buffers + # (below). Prefill/mixed batches fall back to piecewise. + _cudagraph_support: ClassVar[AttentionCGSupport] = ( + AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE + ) + # Disable the update_block_table fast path: it would bypass block pooling + # by splicing the raw slot_mapping/block_table into cached metadata. + supports_update_block_table: bool = False + def __init__( self, kv_cache_spec: AttentionSpec, @@ -150,6 +161,38 @@ def create_whisper_attention_backend_with_block_pooling( # num_heads_q for the encoder is the same as num_kv_heads # (no GQA in whisper encoder) self.num_heads_q = kv_cache_spec.num_kv_heads + # Persistent capture-safe buffers for pooled metadata; lazily sized + # on first `build` to match the incoming tensors' dtype/device. + self._pool_buffers_ready = False + + def _maybe_init_pool_buffers( + self, common_attn_metadata: CommonAttentionMetadata + ) -> None: + if self._pool_buffers_ready: + return + device = common_attn_metadata.slot_mapping.device + sched = self.vllm_config.scheduler_config + self._qsl_buf = torch.empty( + sched.max_num_seqs + 1, + dtype=common_attn_metadata.query_start_loc.dtype, + device=device, + ) + self._seq_buf = torch.empty( + sched.max_num_seqs, + dtype=common_attn_metadata.seq_lens.dtype, + device=device, + ) + self._slot_buf = torch.empty( + sched.max_num_batched_tokens * block_pool_size, + dtype=common_attn_metadata.slot_mapping.dtype, + device=device, + ) + self._pool_arange = torch.arange( + block_pool_size, + dtype=common_attn_metadata.slot_mapping.dtype, + device=device, + ) + self._pool_buffers_ready = True def build( self, @@ -157,27 +200,62 @@ def create_whisper_attention_backend_with_block_pooling( common_attn_metadata: CommonAttentionMetadata, fast_build: bool = False, ) -> AttentionMetadata: - new_common_attn_metadata = copy.deepcopy(common_attn_metadata) - new_common_attn_metadata.query_start_loc *= block_pool_size - new_common_attn_metadata.query_start_loc_cpu *= block_pool_size - new_common_attn_metadata.seq_lens *= block_pool_size - if new_common_attn_metadata.seq_lens_cpu_upper_bound is not None: - new_common_attn_metadata.seq_lens_cpu_upper_bound *= block_pool_size - new_common_attn_metadata.num_actual_tokens *= block_pool_size - new_common_attn_metadata.max_query_len *= block_pool_size - new_common_attn_metadata.max_seq_len *= block_pool_size - original_slot_mapping = common_attn_metadata.slot_mapping - common_prefix_len *= block_pool_size - new_common_attn_metadata.slot_mapping = ( - ( - original_slot_mapping.unsqueeze(1) * block_pool_size - + torch.arange(block_pool_size, device=original_slot_mapping.device) - ) - .flatten() - .clamp(min=-1) + cm = common_attn_metadata + self._maybe_init_pool_buffers(cm) + + # Scale query_start_loc / seq_lens into the persistent buffers. + n = cm.query_start_loc.numel() + assert n <= self._qsl_buf.numel(), ( + f"{n=} exceeds pooled query_start_loc buffer {self._qsl_buf.numel()}" + ) + qsl_buf = self._qsl_buf[:n] + qsl_buf.copy_(cm.query_start_loc) + qsl_buf.mul_(block_pool_size) + + r = cm.seq_lens.numel() + assert r <= self._seq_buf.numel(), ( + f"{r=} exceeds pooled seq_lens buffer {self._seq_buf.numel()}" + ) + seq_buf = self._seq_buf[:r] + seq_buf.copy_(cm.seq_lens) + seq_buf.mul_(block_pool_size) + + # Expand each slot into block_pool_size contiguous slots. Padding + # slots (-1) stay -1 after clamp so the KV write is skipped. + m = cm.slot_mapping.numel() + assert m * block_pool_size <= self._slot_buf.numel(), ( + f"pooled slot_mapping {m * block_pool_size} exceeds buffer " + f"{self._slot_buf.numel()}" + ) + slot_buf = self._slot_buf[: m * block_pool_size] + slot_buf.view(m, block_pool_size).copy_( + cm.slot_mapping.unsqueeze(1) * block_pool_size + self._pool_arange + ) + slot_buf.clamp_(min=-1) + + # CPU tensors aren't captured; fresh scaled copies avoid mutating + # the runner's shared buffers. + qsl_cpu = cm.query_start_loc_cpu * block_pool_size + seq_lens_cpu_upper_bound = ( + cm.seq_lens_cpu_upper_bound * block_pool_size + if cm.seq_lens_cpu_upper_bound is not None + else None + ) + + new_common_attn_metadata = cm.replace( + query_start_loc=qsl_buf, + query_start_loc_cpu=qsl_cpu, + seq_lens=seq_buf, + seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound, + num_actual_tokens=cm.num_actual_tokens * block_pool_size, + max_query_len=cm.max_query_len * block_pool_size, + max_seq_len=cm.max_seq_len * block_pool_size, + slot_mapping=slot_buf, ) return super().build( - common_prefix_len, new_common_attn_metadata, fast_build + common_prefix_len * block_pool_size, + new_common_attn_metadata, + fast_build, ) # NOTE: We need a custom impl so we can use the transformed slot_mapping