[Model] Voxtral Realtime: add support for CUDAGraphMode.FULL_DECODE_ONLY (#51167)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai> Co-authored-by: Jorge Gallego Feliciano <jorge.gallego@mistral.ai>
This commit is contained in:
co-authored by
Jorge Gallego Feliciano
parent
0a5747d410
commit
995e8581f4
@@ -14,7 +14,9 @@ from mistral_common.tokens.tokenizers.tekken import SpecialTokenPolicy
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
from vllm.v1.kv_cache_interface import SlidingWindowSpec
|
||||
@@ -130,6 +132,45 @@ async def async_engine():
|
||||
wait_for_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
|
||||
|
||||
|
||||
def _build_inputs(
|
||||
audio_assets: list[AudioAsset], tokenizer: MistralTokenizer
|
||||
) -> tuple[list[dict], list[SamplingParams]]:
|
||||
audio_config = tokenizer.instruct_tokenizer.tokenizer.audio
|
||||
inputs = []
|
||||
sampling_params = []
|
||||
for audio_asset in audio_assets:
|
||||
audio = Audio.from_file(audio_asset.get_local_path(), strict=False)
|
||||
req = TranscriptionRequest(
|
||||
audio=audio.to_base64(audio.format),
|
||||
streaming=StreamingMode.OFFLINE,
|
||||
language=None,
|
||||
)
|
||||
tokenized = tokenizer.instruct_tokenizer.encode_transcription(req)
|
||||
tokens = tokenized.tokens
|
||||
audio_array = tokenized.audios[0].audio_array
|
||||
|
||||
num_samples = audio_array.shape[0]
|
||||
max_tokens = audio_config.num_audio_tokens(num_samples) - len(tokens) - 1
|
||||
sampling_params.append(SamplingParams(temperature=0.0, max_tokens=max_tokens))
|
||||
inputs.append(
|
||||
{
|
||||
"multi_modal_data": {"audio": [(audio_array, None)]},
|
||||
"prompt_token_ids": tokens,
|
||||
}
|
||||
)
|
||||
return inputs, sampling_params
|
||||
|
||||
|
||||
def _assert_expected_text(outputs) -> None:
|
||||
texts = _normalize([out.outputs[0].text for out in outputs])
|
||||
for i, (got, expected) in enumerate(zip(texts, EXPECTED_TEXT)):
|
||||
assert got == expected, (
|
||||
f"Output mismatch at index {i}:\n"
|
||||
f" got: {got!r}\n"
|
||||
f" expected: {expected!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_voxtral_realtime_forward(audio_assets, tokenizer, vllm_runner, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
|
||||
@@ -138,51 +179,38 @@ def test_voxtral_realtime_forward(audio_assets, tokenizer, vllm_runner, monkeypa
|
||||
|
||||
with vllm_runner(**vllm_kwargs) as vllm_model:
|
||||
assert_encoder_kv_cache_spec(vllm_model.llm)
|
||||
audio_config = tokenizer.instruct_tokenizer.tokenizer.audio
|
||||
inputs, sampling_params = _build_inputs(audio_assets, tokenizer)
|
||||
|
||||
def from_file(file_path: str):
|
||||
audio = Audio.from_file(file_path, strict=False)
|
||||
req = TranscriptionRequest(
|
||||
audio=audio.to_base64(audio.format),
|
||||
streaming=StreamingMode.OFFLINE,
|
||||
language=None,
|
||||
)
|
||||
tokenized = tokenizer.instruct_tokenizer.encode_transcription(req)
|
||||
outputs = vllm_model.llm.generate(inputs, sampling_params=sampling_params)
|
||||
_assert_expected_text(outputs)
|
||||
|
||||
return (tokenized.tokens, tokenized.audios[0].audio_array)
|
||||
|
||||
tokenized_list = [
|
||||
from_file(audio_asset.get_local_path()) for audio_asset in audio_assets
|
||||
]
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="Full cudagraphs require CUDA"
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"cudagraph_mode",
|
||||
[CUDAGraphMode.FULL_DECODE_ONLY, CUDAGraphMode.FULL_AND_PIECEWISE],
|
||||
)
|
||||
def test_voxtral_realtime_cudagraph(
|
||||
audio_assets, tokenizer, vllm_runner, monkeypatch, cudagraph_mode
|
||||
):
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
|
||||
inputs = []
|
||||
sampling_params = []
|
||||
vllm_kwargs = {**ENGINE_CONFIG}
|
||||
vllm_kwargs["model_name"] = vllm_kwargs.pop("model")
|
||||
vllm_kwargs["enforce_eager"] = False
|
||||
vllm_kwargs["compilation_config"] = {
|
||||
"cudagraph_mode": cudagraph_mode,
|
||||
# max_num_seqs is 4, so this is the only decode size worth capturing.
|
||||
"cudagraph_capture_sizes": [4],
|
||||
}
|
||||
|
||||
for tokens, audio_array in tokenized_list:
|
||||
num_samples = audio_array.shape[0]
|
||||
max_tokens = audio_config.num_audio_tokens(num_samples) - len(tokens) - 1
|
||||
sampling_params.append(
|
||||
SamplingParams(temperature=0.0, max_tokens=max_tokens)
|
||||
)
|
||||
with vllm_runner(**vllm_kwargs) as vllm_model:
|
||||
inputs, sampling_params = _build_inputs(audio_assets, tokenizer)
|
||||
|
||||
input_dict = {
|
||||
"multi_modal_data": {"audio": [(audio_array, None)]},
|
||||
"prompt_token_ids": tokens,
|
||||
}
|
||||
inputs.append(input_dict)
|
||||
|
||||
outputs = vllm_model.llm.generate(
|
||||
inputs,
|
||||
sampling_params=sampling_params,
|
||||
)
|
||||
|
||||
texts = _normalize([out.outputs[0].text for out in outputs])
|
||||
for i, (got, expected) in enumerate(zip(texts, EXPECTED_TEXT)):
|
||||
assert got == expected, (
|
||||
f"Output mismatch at index {i}:\n"
|
||||
f" got: {got!r}\n"
|
||||
f" expected: {expected!r}"
|
||||
)
|
||||
outputs = vllm_model.llm.generate(inputs, sampling_params=sampling_params)
|
||||
_assert_expected_text(outputs)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -217,9 +217,15 @@ class VoxtralRealtimeGeneration(VoxtralForConditionalGeneration, SupportsRealtim
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
|
||||
assert (
|
||||
not vllm_config.compilation_config.cudagraph_mode.has_full_cudagraphs()
|
||||
), "Voxtral realtime doesn't support full cudagraphs yet. Please use PIECEWISE."
|
||||
# Full cudagraphs are supported for decode-only batches (the encoder's
|
||||
# block-pooling attention builder is capture-safe for uniform
|
||||
# single-token decode). Mixed/prefill batches are not, so block only
|
||||
# pure FULL; FULL_DECODE_ONLY and FULL_AND_PIECEWISE are allowed.
|
||||
cudagraph_mode = vllm_config.compilation_config.cudagraph_mode
|
||||
assert not cudagraph_mode.mixed_mode().has_full_cudagraphs(), (
|
||||
"Voxtral realtime supports full cudagraphs for decode-only batches. "
|
||||
"Use cudagraph_mode=FULL_DECODE_ONLY or FULL_AND_PIECEWISE (not FULL)."
|
||||
)
|
||||
|
||||
self.time_embedding: TimeEmbedding = TimeEmbedding(
|
||||
dim=self.config.text_config.hidden_size
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import copy
|
||||
import functools
|
||||
import logging
|
||||
import math
|
||||
from dataclasses import replace
|
||||
from fractions import Fraction
|
||||
from functools import partial
|
||||
from typing import ClassVar
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
@@ -27,6 +27,7 @@ from vllm.model_executor.models.whisper import WhisperPosEmbedType
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionCGSupport,
|
||||
AttentionMetadata,
|
||||
AttentionType,
|
||||
CommonAttentionMetadata,
|
||||
@@ -123,6 +124,16 @@ def create_whisper_attention_backend_with_block_pooling(
|
||||
underlying_impl = underlying_attn_backend.get_impl_cls()
|
||||
|
||||
class WhisperCausalAttentionWithBlockPoolingBuilder(underlying_builder): # type: ignore
|
||||
# Full cudagraphs only for uniform single-token decode: capture bakes in
|
||||
# tensor addresses, so `build` writes metadata into persistent buffers
|
||||
# (below). Prefill/mixed batches fall back to piecewise.
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = (
|
||||
AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
|
||||
)
|
||||
# Disable the update_block_table fast path: it would bypass block pooling
|
||||
# by splicing the raw slot_mapping/block_table into cached metadata.
|
||||
supports_update_block_table: bool = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_spec: AttentionSpec,
|
||||
@@ -150,6 +161,38 @@ def create_whisper_attention_backend_with_block_pooling(
|
||||
# num_heads_q for the encoder is the same as num_kv_heads
|
||||
# (no GQA in whisper encoder)
|
||||
self.num_heads_q = kv_cache_spec.num_kv_heads
|
||||
# Persistent capture-safe buffers for pooled metadata; lazily sized
|
||||
# on first `build` to match the incoming tensors' dtype/device.
|
||||
self._pool_buffers_ready = False
|
||||
|
||||
def _maybe_init_pool_buffers(
|
||||
self, common_attn_metadata: CommonAttentionMetadata
|
||||
) -> None:
|
||||
if self._pool_buffers_ready:
|
||||
return
|
||||
device = common_attn_metadata.slot_mapping.device
|
||||
sched = self.vllm_config.scheduler_config
|
||||
self._qsl_buf = torch.empty(
|
||||
sched.max_num_seqs + 1,
|
||||
dtype=common_attn_metadata.query_start_loc.dtype,
|
||||
device=device,
|
||||
)
|
||||
self._seq_buf = torch.empty(
|
||||
sched.max_num_seqs,
|
||||
dtype=common_attn_metadata.seq_lens.dtype,
|
||||
device=device,
|
||||
)
|
||||
self._slot_buf = torch.empty(
|
||||
sched.max_num_batched_tokens * block_pool_size,
|
||||
dtype=common_attn_metadata.slot_mapping.dtype,
|
||||
device=device,
|
||||
)
|
||||
self._pool_arange = torch.arange(
|
||||
block_pool_size,
|
||||
dtype=common_attn_metadata.slot_mapping.dtype,
|
||||
device=device,
|
||||
)
|
||||
self._pool_buffers_ready = True
|
||||
|
||||
def build(
|
||||
self,
|
||||
@@ -157,27 +200,62 @@ def create_whisper_attention_backend_with_block_pooling(
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
fast_build: bool = False,
|
||||
) -> AttentionMetadata:
|
||||
new_common_attn_metadata = copy.deepcopy(common_attn_metadata)
|
||||
new_common_attn_metadata.query_start_loc *= block_pool_size
|
||||
new_common_attn_metadata.query_start_loc_cpu *= block_pool_size
|
||||
new_common_attn_metadata.seq_lens *= block_pool_size
|
||||
if new_common_attn_metadata.seq_lens_cpu_upper_bound is not None:
|
||||
new_common_attn_metadata.seq_lens_cpu_upper_bound *= block_pool_size
|
||||
new_common_attn_metadata.num_actual_tokens *= block_pool_size
|
||||
new_common_attn_metadata.max_query_len *= block_pool_size
|
||||
new_common_attn_metadata.max_seq_len *= block_pool_size
|
||||
original_slot_mapping = common_attn_metadata.slot_mapping
|
||||
common_prefix_len *= block_pool_size
|
||||
new_common_attn_metadata.slot_mapping = (
|
||||
(
|
||||
original_slot_mapping.unsqueeze(1) * block_pool_size
|
||||
+ torch.arange(block_pool_size, device=original_slot_mapping.device)
|
||||
)
|
||||
.flatten()
|
||||
.clamp(min=-1)
|
||||
cm = common_attn_metadata
|
||||
self._maybe_init_pool_buffers(cm)
|
||||
|
||||
# Scale query_start_loc / seq_lens into the persistent buffers.
|
||||
n = cm.query_start_loc.numel()
|
||||
assert n <= self._qsl_buf.numel(), (
|
||||
f"{n=} exceeds pooled query_start_loc buffer {self._qsl_buf.numel()}"
|
||||
)
|
||||
qsl_buf = self._qsl_buf[:n]
|
||||
qsl_buf.copy_(cm.query_start_loc)
|
||||
qsl_buf.mul_(block_pool_size)
|
||||
|
||||
r = cm.seq_lens.numel()
|
||||
assert r <= self._seq_buf.numel(), (
|
||||
f"{r=} exceeds pooled seq_lens buffer {self._seq_buf.numel()}"
|
||||
)
|
||||
seq_buf = self._seq_buf[:r]
|
||||
seq_buf.copy_(cm.seq_lens)
|
||||
seq_buf.mul_(block_pool_size)
|
||||
|
||||
# Expand each slot into block_pool_size contiguous slots. Padding
|
||||
# slots (-1) stay -1 after clamp so the KV write is skipped.
|
||||
m = cm.slot_mapping.numel()
|
||||
assert m * block_pool_size <= self._slot_buf.numel(), (
|
||||
f"pooled slot_mapping {m * block_pool_size} exceeds buffer "
|
||||
f"{self._slot_buf.numel()}"
|
||||
)
|
||||
slot_buf = self._slot_buf[: m * block_pool_size]
|
||||
slot_buf.view(m, block_pool_size).copy_(
|
||||
cm.slot_mapping.unsqueeze(1) * block_pool_size + self._pool_arange
|
||||
)
|
||||
slot_buf.clamp_(min=-1)
|
||||
|
||||
# CPU tensors aren't captured; fresh scaled copies avoid mutating
|
||||
# the runner's shared buffers.
|
||||
qsl_cpu = cm.query_start_loc_cpu * block_pool_size
|
||||
seq_lens_cpu_upper_bound = (
|
||||
cm.seq_lens_cpu_upper_bound * block_pool_size
|
||||
if cm.seq_lens_cpu_upper_bound is not None
|
||||
else None
|
||||
)
|
||||
|
||||
new_common_attn_metadata = cm.replace(
|
||||
query_start_loc=qsl_buf,
|
||||
query_start_loc_cpu=qsl_cpu,
|
||||
seq_lens=seq_buf,
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||
num_actual_tokens=cm.num_actual_tokens * block_pool_size,
|
||||
max_query_len=cm.max_query_len * block_pool_size,
|
||||
max_seq_len=cm.max_seq_len * block_pool_size,
|
||||
slot_mapping=slot_buf,
|
||||
)
|
||||
return super().build(
|
||||
common_prefix_len, new_common_attn_metadata, fast_build
|
||||
common_prefix_len * block_pool_size,
|
||||
new_common_attn_metadata,
|
||||
fast_build,
|
||||
)
|
||||
|
||||
# NOTE: We need a custom impl so we can use the transformed slot_mapping
|
||||
|
||||
Reference in New Issue
Block a user