[Bugfix] Fix pooled Whisper sliding-window KV sizing (#47071)
Signed-off-by: Andy Lo <andy@mistral.ai>
This commit is contained in:
@@ -15,11 +15,14 @@ from mistral_common.tokens.tokenizers.tekken import SpecialTokenPolicy
|
||||
from vllm import LLM, EngineArgs, SamplingParams
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
from vllm.v1.kv_cache_interface import SlidingWindowSpec
|
||||
|
||||
from ....utils import ROCM_ENGINE_KWARGS
|
||||
|
||||
MODEL_NAME = "mistralai/Voxtral-Mini-4B-Realtime-2602"
|
||||
AUDIO_LAYER_NAME = "whisper_encoder.whisper_encoder.layers.0.layers.self_attn.attn"
|
||||
ENGINE_CONFIG = {
|
||||
"model": MODEL_NAME,
|
||||
"max_model_len": 8192,
|
||||
@@ -60,6 +63,31 @@ def _normalize(texts: list[str]) -> list[str]:
|
||||
return texts
|
||||
|
||||
|
||||
def assert_encoder_kv_cache_spec(engine: LLM) -> None:
|
||||
vllm_config = engine.llm_engine.vllm_config
|
||||
audio_config = vllm_config.model_config.hf_config.audio_config
|
||||
kv_cache_specs_per_rank = engine.llm_engine.model_executor.get_kv_cache_specs()
|
||||
|
||||
assert len(kv_cache_specs_per_rank) == 1
|
||||
kv_cache_specs = kv_cache_specs_per_rank[0]
|
||||
assert AUDIO_LAYER_NAME in kv_cache_specs, kv_cache_specs.keys()
|
||||
spec = kv_cache_specs[AUDIO_LAYER_NAME]
|
||||
|
||||
assert audio_config.sliding_window == 750
|
||||
assert audio_config.block_pool_size == 4
|
||||
assert isinstance(spec, SlidingWindowSpec)
|
||||
assert spec.block_size == 16
|
||||
assert spec.num_kv_heads == 128
|
||||
assert spec.sliding_window == cdiv(750, 4) == 188
|
||||
assert (
|
||||
spec.max_admission_blocks_per_request(
|
||||
max_num_batched_tokens=1,
|
||||
max_model_len=vllm_config.model_config.max_model_len,
|
||||
)
|
||||
== 13
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def audio_assets() -> list[AudioAsset]:
|
||||
return [AudioAsset("mary_had_lamb"), AudioAsset("winning_call")]
|
||||
@@ -71,7 +99,9 @@ def tokenizer() -> MistralTokenizer:
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def engine():
|
||||
def engine(monkeypatch: pytest.MonkeyPatch):
|
||||
# Disable multiprocessing allows us to access model executor from LLM engine
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
engine_args = EngineArgs(**ENGINE_CONFIG)
|
||||
llm = LLM.from_engine_args(engine_args)
|
||||
try:
|
||||
@@ -95,6 +125,7 @@ async def async_engine():
|
||||
|
||||
|
||||
def test_voxtral_realtime_forward(audio_assets, tokenizer, engine):
|
||||
assert_encoder_kv_cache_spec(engine)
|
||||
audio_config = tokenizer.instruct_tokenizer.tokenizer.audio
|
||||
|
||||
def from_file(file_path: str):
|
||||
|
||||
@@ -23,6 +23,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.models.mistral import MistralMLP
|
||||
from vllm.model_executor.models.whisper import WhisperPosEmbedType
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionMetadata,
|
||||
@@ -39,7 +40,7 @@ except ImportError:
|
||||
from vllm.v1.attention.backends.rocm_attn import RocmAttentionBackend
|
||||
from vllm.v1.attention.backends.triton_attn import TritonAttentionBackend
|
||||
from vllm.v1.attention.selector import get_attn_backend
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec, SlidingWindowSpec
|
||||
|
||||
from .utils import make_layers
|
||||
|
||||
@@ -329,6 +330,15 @@ class WhisperCausalAttentionWithBlockPooling(Attention):
|
||||
kv_cache_spec,
|
||||
num_kv_heads=self.block_pool_size * kv_cache_spec.num_kv_heads,
|
||||
)
|
||||
if isinstance(kv_cache_spec, SlidingWindowSpec):
|
||||
# The KV cache manager counts blocks in pooled units, so express the
|
||||
# window in pooled units too to avoid reserving `block_pool_size`x
|
||||
# too many blocks. The kernel window is unaffected because it comes
|
||||
# from the attention impl, not this spec.
|
||||
kv_cache_spec = replace(
|
||||
kv_cache_spec,
|
||||
sliding_window=cdiv(kv_cache_spec.sliding_window, self.block_pool_size),
|
||||
)
|
||||
return kv_cache_spec
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user