Merge remote-tracking branch 'origin/main' into sync/lesj/qwen4exp-bugfixes
Signed-off-by: lesj0610 <lesj0610@users.noreply.github.com>
This commit is contained in:
@@ -14,12 +14,14 @@ from mistral_common.tokens.tokenizers.tekken import SpecialTokenPolicy
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
from vllm.v1.kv_cache_interface import SlidingWindowSpec
|
||||
|
||||
from ....utils import ROCM_ENGINE_KWARGS
|
||||
from ....utils import ROCM_ENGINE_KWARGS, create_new_process_for_each_test
|
||||
|
||||
MODEL_NAME = "mistralai/Voxtral-Mini-4B-Realtime-2602"
|
||||
AUDIO_LAYER_NAME = "whisper_encoder.whisper_encoder.layers.0.layers.self_attn.attn"
|
||||
@@ -130,6 +132,45 @@ async def async_engine():
|
||||
wait_for_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
|
||||
|
||||
|
||||
def _build_inputs(
|
||||
audio_assets: list[AudioAsset], tokenizer: MistralTokenizer
|
||||
) -> tuple[list[dict], list[SamplingParams]]:
|
||||
audio_config = tokenizer.instruct_tokenizer.tokenizer.audio
|
||||
inputs = []
|
||||
sampling_params = []
|
||||
for audio_asset in audio_assets:
|
||||
audio = Audio.from_file(audio_asset.get_local_path(), strict=False)
|
||||
req = TranscriptionRequest(
|
||||
audio=audio.to_base64(audio.format),
|
||||
streaming=StreamingMode.OFFLINE,
|
||||
language=None,
|
||||
)
|
||||
tokenized = tokenizer.instruct_tokenizer.encode_transcription(req)
|
||||
tokens = tokenized.tokens
|
||||
audio_array = tokenized.audios[0].audio_array
|
||||
|
||||
num_samples = audio_array.shape[0]
|
||||
max_tokens = audio_config.num_audio_tokens(num_samples) - len(tokens) - 1
|
||||
sampling_params.append(SamplingParams(temperature=0.0, max_tokens=max_tokens))
|
||||
inputs.append(
|
||||
{
|
||||
"multi_modal_data": {"audio": [(audio_array, None)]},
|
||||
"prompt_token_ids": tokens,
|
||||
}
|
||||
)
|
||||
return inputs, sampling_params
|
||||
|
||||
|
||||
def _assert_expected_text(outputs) -> None:
|
||||
texts = _normalize([out.outputs[0].text for out in outputs])
|
||||
for i, (got, expected) in enumerate(zip(texts, EXPECTED_TEXT)):
|
||||
assert got == expected, (
|
||||
f"Output mismatch at index {i}:\n"
|
||||
f" got: {got!r}\n"
|
||||
f" expected: {expected!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_voxtral_realtime_forward(audio_assets, tokenizer, vllm_runner, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
|
||||
@@ -138,51 +179,39 @@ def test_voxtral_realtime_forward(audio_assets, tokenizer, vllm_runner, monkeypa
|
||||
|
||||
with vllm_runner(**vllm_kwargs) as vllm_model:
|
||||
assert_encoder_kv_cache_spec(vllm_model.llm)
|
||||
audio_config = tokenizer.instruct_tokenizer.tokenizer.audio
|
||||
inputs, sampling_params = _build_inputs(audio_assets, tokenizer)
|
||||
|
||||
def from_file(file_path: str):
|
||||
audio = Audio.from_file(file_path, strict=False)
|
||||
req = TranscriptionRequest(
|
||||
audio=audio.to_base64(audio.format),
|
||||
streaming=StreamingMode.OFFLINE,
|
||||
language=None,
|
||||
)
|
||||
tokenized = tokenizer.instruct_tokenizer.encode_transcription(req)
|
||||
outputs = vllm_model.llm.generate(inputs, sampling_params=sampling_params)
|
||||
_assert_expected_text(outputs)
|
||||
|
||||
return (tokenized.tokens, tokenized.audios[0].audio_array)
|
||||
|
||||
tokenized_list = [
|
||||
from_file(audio_asset.get_local_path()) for audio_asset in audio_assets
|
||||
]
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="Full cudagraphs require CUDA"
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"cudagraph_mode",
|
||||
[CUDAGraphMode.FULL_DECODE_ONLY, CUDAGraphMode.FULL_AND_PIECEWISE],
|
||||
)
|
||||
@create_new_process_for_each_test(method="spawn")
|
||||
def test_voxtral_realtime_cudagraph(
|
||||
audio_assets, tokenizer, vllm_runner, monkeypatch, cudagraph_mode
|
||||
):
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
|
||||
inputs = []
|
||||
sampling_params = []
|
||||
vllm_kwargs = {**ENGINE_CONFIG}
|
||||
vllm_kwargs["model_name"] = vllm_kwargs.pop("model")
|
||||
vllm_kwargs["enforce_eager"] = False
|
||||
vllm_kwargs["compilation_config"] = {
|
||||
"cudagraph_mode": cudagraph_mode,
|
||||
# max_num_seqs is 4, so this is the only decode size worth capturing.
|
||||
"cudagraph_capture_sizes": [4],
|
||||
}
|
||||
|
||||
for tokens, audio_array in tokenized_list:
|
||||
num_samples = audio_array.shape[0]
|
||||
max_tokens = audio_config.num_audio_tokens(num_samples) - len(tokens) - 1
|
||||
sampling_params.append(
|
||||
SamplingParams(temperature=0.0, max_tokens=max_tokens)
|
||||
)
|
||||
with vllm_runner(**vllm_kwargs) as vllm_model:
|
||||
inputs, sampling_params = _build_inputs(audio_assets, tokenizer)
|
||||
|
||||
input_dict = {
|
||||
"multi_modal_data": {"audio": [(audio_array, None)]},
|
||||
"prompt_token_ids": tokens,
|
||||
}
|
||||
inputs.append(input_dict)
|
||||
|
||||
outputs = vllm_model.llm.generate(
|
||||
inputs,
|
||||
sampling_params=sampling_params,
|
||||
)
|
||||
|
||||
texts = _normalize([out.outputs[0].text for out in outputs])
|
||||
for i, (got, expected) in enumerate(zip(texts, EXPECTED_TEXT)):
|
||||
assert got == expected, (
|
||||
f"Output mismatch at index {i}:\n"
|
||||
f" got: {got!r}\n"
|
||||
f" expected: {expected!r}"
|
||||
)
|
||||
outputs = vllm_model.llm.generate(inputs, sampling_params=sampling_params)
|
||||
_assert_expected_text(outputs)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -6,9 +6,13 @@ from collections.abc import Mapping
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image as PILImage
|
||||
from transformers.models.gemma4 import Gemma4AudioFeatureExtractor
|
||||
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.model_executor.models.gemma4_mm import Gemma4ImagePixelInputs
|
||||
from vllm.model_executor.models.gemma4_mm import (
|
||||
Gemma4ImagePixelInputs,
|
||||
_dummy_audio_num_samples,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalFieldConfig
|
||||
|
||||
@@ -204,3 +208,41 @@ def test_limit_mm_per_prompt(
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
|
||||
def test_dummy_audio_length_falls_back_for_the_unified_extractor():
|
||||
"""The unified audio feature extractor does not define ``fft_length``.
|
||||
|
||||
``Gemma4DummyInputsBuilder.get_dummy_mm_data`` read that attribute
|
||||
unconditionally, so memory profiling raised ``AttributeError`` for every
|
||||
unified Gemma 4 deployment in which audio is the modality with the largest
|
||||
per-item token count -- which is the case as soon as ``audio_seq_length``
|
||||
is raised above the video budget.
|
||||
"""
|
||||
|
||||
class _UnifiedFeatureExtractor:
|
||||
sampling_rate = 16000
|
||||
|
||||
class _UnifiedProcessor:
|
||||
feature_extractor = _UnifiedFeatureExtractor()
|
||||
audio_seq_length = 3000
|
||||
audio_ms_per_token = 40
|
||||
|
||||
# 3000 tokens * 40 ms = 120 s at 16 kHz
|
||||
assert _dummy_audio_num_samples(_UnifiedProcessor()) == 120 * 16000
|
||||
|
||||
|
||||
def test_dummy_audio_length_still_uses_fft_length_for_the_tower_extractor():
|
||||
"""The mel/tower extractor keeps its existing behaviour."""
|
||||
|
||||
class _TowerFeatureExtractor(Gemma4AudioFeatureExtractor):
|
||||
def __init__(self): # noqa: D107 - bypass the real feature-extractor init
|
||||
self.fft_length = 512
|
||||
self.sampling_rate = 16000
|
||||
|
||||
class _TowerProcessor:
|
||||
feature_extractor = _TowerFeatureExtractor()
|
||||
audio_seq_length = 750
|
||||
audio_ms_per_token = 40
|
||||
|
||||
assert _dummy_audio_num_samples(_TowerProcessor()) == 512
|
||||
|
||||
@@ -14,7 +14,7 @@ from ...utils import build_model_context
|
||||
|
||||
|
||||
class _StubModel:
|
||||
"""Carries only the two attributes the token helpers read from
|
||||
"""Carries only the attributes the token-count method reads from
|
||||
`self`, so the real methods can be exercised without constructing
|
||||
the full `nn.Module` (vision tower, language model, etc.)."""
|
||||
|
||||
@@ -22,18 +22,15 @@ class _StubModel:
|
||||
pooled_grid_length: int
|
||||
|
||||
|
||||
get_num_mm_encoder_tokens = (
|
||||
LlavaNextVideoForConditionalGeneration.get_num_mm_encoder_tokens
|
||||
)
|
||||
get_num_mm_connector_tokens = (
|
||||
LlavaNextVideoForConditionalGeneration.get_num_mm_connector_tokens
|
||||
get_mm_lora_token_counts = (
|
||||
LlavaNextVideoForConditionalGeneration.get_mm_lora_token_counts
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["llava-hf/LLaVA-NeXT-Video-7B-hf"])
|
||||
def test_num_mm_tokens_match_real_config(model_id):
|
||||
"""The stored grid lengths must match what `__init__` derives from
|
||||
the real HF config, and the two helpers must invert each other's
|
||||
the real HF config, and the LoRA token counts must preserve the
|
||||
frame-level scaling exactly."""
|
||||
ctx = build_model_context(model_id, limit_mm_per_prompt={"video": 1})
|
||||
config = ctx.model_config.hf_config
|
||||
@@ -49,10 +46,13 @@ def test_num_mm_tokens_match_real_config(model_id):
|
||||
for num_frames in (1, 2, 8, 16, 32):
|
||||
num_video_tokens = num_frames * pooled_grid_length**2
|
||||
|
||||
encoder_tokens = get_num_mm_encoder_tokens(stub, num_video_tokens)
|
||||
encoder_tokens, connector_tokens = get_mm_lora_token_counts(
|
||||
stub,
|
||||
modality="video",
|
||||
mm_kwargs=None,
|
||||
num_mm_embeds=num_video_tokens,
|
||||
)
|
||||
assert encoder_tokens == num_frames * patch_grid_length**2
|
||||
|
||||
connector_tokens = get_num_mm_connector_tokens(stub, encoder_tokens)
|
||||
assert connector_tokens == num_video_tokens
|
||||
|
||||
|
||||
@@ -72,10 +72,13 @@ def test_num_mm_tokens_roundtrip(patch_grid_length, pooled_grid_length, num_fram
|
||||
|
||||
num_video_tokens = num_frames * pooled_grid_length**2
|
||||
|
||||
encoder_tokens = get_num_mm_encoder_tokens(stub, num_video_tokens)
|
||||
encoder_tokens, connector_tokens = get_mm_lora_token_counts(
|
||||
stub,
|
||||
modality="video",
|
||||
mm_kwargs=None,
|
||||
num_mm_embeds=num_video_tokens,
|
||||
)
|
||||
assert encoder_tokens == num_frames * patch_grid_length**2
|
||||
|
||||
connector_tokens = get_num_mm_connector_tokens(stub, encoder_tokens)
|
||||
assert connector_tokens == num_video_tokens
|
||||
|
||||
|
||||
@@ -84,5 +87,9 @@ def test_num_mm_tokens_zero():
|
||||
stub.patch_grid_length = 24
|
||||
stub.pooled_grid_length = 12
|
||||
|
||||
assert get_num_mm_encoder_tokens(stub, 0) == 0
|
||||
assert get_num_mm_connector_tokens(stub, 0) == 0
|
||||
assert get_mm_lora_token_counts(
|
||||
stub,
|
||||
modality="video",
|
||||
mm_kwargs=None,
|
||||
num_mm_embeds=0,
|
||||
) == (0, 0)
|
||||
|
||||
@@ -1610,6 +1610,12 @@ def test_fused_conv_correctness(
|
||||
dtype=torch.bfloat16,
|
||||
generator=rng,
|
||||
)
|
||||
outer_residual = torch.randn(
|
||||
inputs.shape,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
generator=rng,
|
||||
)
|
||||
null_state = conv_state[NULL_BLOCK_ID].clone()
|
||||
residual_kernel = residual.clone()
|
||||
residual_reference = residual.clone()
|
||||
@@ -1617,6 +1623,7 @@ def test_fused_conv_correctness(
|
||||
module._short_conv_dilated_dispatch(
|
||||
inputs=inputs,
|
||||
residual=residual_kernel,
|
||||
outer_residual=outer_residual,
|
||||
metadata=metadata,
|
||||
conv_state=conv_state,
|
||||
conv_weights=weights,
|
||||
@@ -1630,15 +1637,17 @@ def test_fused_conv_correctness(
|
||||
conv_state_len=module.conv_state_len,
|
||||
dilation=module.short_conv_dilation,
|
||||
)
|
||||
residual_reference = outer_residual + residual_reference
|
||||
|
||||
torch.testing.assert_close(
|
||||
residual_kernel.float(), residual_reference.float(), atol=3e-2, rtol=3e-2
|
||||
assert torch.equal(
|
||||
residual_kernel[:num_real_tokens], residual_reference[:num_real_tokens]
|
||||
)
|
||||
assert torch.equal(conv_state, state_reference)
|
||||
assert torch.equal(conv_state[NULL_BLOCK_ID], null_state)
|
||||
if case.graph_padding:
|
||||
assert torch.equal(
|
||||
residual_kernel[num_real_tokens:], residual[num_real_tokens:]
|
||||
residual_kernel[num_real_tokens:],
|
||||
(outer_residual + residual)[num_real_tokens:],
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -944,6 +944,7 @@ def test_qsa_sparse_paged_attention_correctness(
|
||||
q = torch.randn(
|
||||
num_rows, num_query_heads, head_dim, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
output_gate = torch.randn_like(q)
|
||||
kv_cache = torch.randn(
|
||||
num_cache_blocks,
|
||||
page_size,
|
||||
@@ -1023,6 +1024,7 @@ def test_qsa_sparse_paged_attention_correctness(
|
||||
block_table,
|
||||
token_to_req,
|
||||
use_prefill_config=use_prefill_config,
|
||||
output_gate=output_gate,
|
||||
)
|
||||
expected = _qsa_sparse_paged_attention_reference(
|
||||
q,
|
||||
@@ -1033,6 +1035,7 @@ def test_qsa_sparse_paged_attention_correctness(
|
||||
token_to_req,
|
||||
scale,
|
||||
)
|
||||
expected = expected * torch.sigmoid(output_gate)
|
||||
|
||||
torch.testing.assert_close(actual, expected, rtol=2e-2, atol=2e-2)
|
||||
|
||||
|
||||
@@ -461,6 +461,10 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
trust_remote_code=True,
|
||||
is_available_online=False,
|
||||
),
|
||||
"NanbeigeForCausalLM": _HfExamplesInfo(
|
||||
"Nanbeige/Nanbeige4.2-3B",
|
||||
trust_remote_code=True,
|
||||
),
|
||||
"OlmoForCausalLM": _HfExamplesInfo("allenai/OLMo-1B-hf"),
|
||||
"Olmo2ForCausalLM": _HfExamplesInfo("allenai/OLMo-2-0425-1B"),
|
||||
"Olmo3ForCausalLM": _HfExamplesInfo("allenai/Olmo-3-7B-Instruct"),
|
||||
|
||||
@@ -18,8 +18,8 @@ from vllm.models.deepseek_v4.nvidia.model import (
|
||||
)
|
||||
from vllm.models.deepseek_v4.nvidia.mtp import DeepSeekV4MTP
|
||||
from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_inputs
|
||||
from vllm.models.deepseek_v4_1.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
|
||||
from vllm.models.deepseek_v4_1.nvidia.model import DeepseekV4MoE as DeepseekV41MoE
|
||||
from vllm.models.deepseek_v41.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
|
||||
from vllm.models.deepseek_v41.nvidia.model import DeepseekV4MoE as DeepseekV41MoE
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.configs.deepseek_v41 import DeepseekV41Config
|
||||
|
||||
|
||||
@@ -216,7 +216,7 @@ def test_v41_dspark_loads_linear_scales(
|
||||
"""Checkpoint ``.scale`` maps to the quant method's scale parameter and
|
||||
loads untouched. MXFP8 block-scale expansion lives in the KMxfp8Static
|
||||
loader (see tests/quantization/test_modelopt.py), not in load_weights."""
|
||||
from vllm.models.deepseek_v4_1.nvidia import dspark
|
||||
from vllm.models.deepseek_v41.nvidia import dspark
|
||||
|
||||
mxfp8 = scale_dtype != torch.float32
|
||||
scale_name = "weight_scale" if mxfp8 else "weight_scale_inv"
|
||||
|
||||
Reference in New Issue
Block a user