Merge remote-tracking branch 'upstream/main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-09-29 12:54:18 +07:00
264 changed files with 13095 additions and 1541 deletions
@@ -23,6 +23,7 @@ from vllm.distributed import get_tp_group
from vllm.model_executor.layers.fused_moe.runner import moe_runner
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.models.kimi_k3.amd import latent_moe_runner
from vllm.models.kimi_k3.amd.latent_moe_runner import ROCmLatentMoERunner
from vllm.models.kimi_k3.amd.linear import KimiRoutedOutputTransform
from vllm.platforms import current_platform
@@ -56,21 +57,26 @@ def _build_transform(device: torch.device) -> KimiRoutedOutputTransform:
def _tail_runner(
transform: KimiRoutedOutputTransform, tp_size: int
transform: KimiRoutedOutputTransform, tp_world: int, use_ep: bool = False
) -> ROCmLatentMoERunner:
"""A runner carrying only what the tail reads, so no engine is needed.
``_maybe_reduce_final_output`` is left as the real base-class method, so the
all-reduce that stitches the shards is the real collective.
``tp_world`` is the size of the TP process group, which is what the hidden
dim is split by. Under expert parallelism the MoE config reports
``tp_size=1``/``ep_size=tp_world`` for that same group, so ``use_ep`` pins
that the final reduce still fires.
"""
runner = object.__new__(ROCmLatentMoERunner)
attrs = {
"routed_output_transform": transform,
"_up_proj_shard_size": HIDDEN_SIZE // tp_size,
"_up_proj_shard_size": HIDDEN_SIZE // tp_world,
"_logged_sharded_tail": False,
"moe_config": SimpleNamespace(
tp_size=tp_size,
ep_size=1,
tp_size=1 if use_ep else tp_world,
ep_size=tp_world if use_ep else 1,
is_sequence_parallel=False,
skip_final_all_reduce=False,
),
@@ -94,9 +100,11 @@ def _rank_partials(
return routed.mul_(0.01), shared
def _check_matches_replicated(device: torch.device, tp_size: int, rank: int) -> None:
def _check_matches_replicated(
device: torch.device, tp_world: int, rank: int, use_ep: bool = False
) -> None:
transform = _build_transform(device)
runner = _tail_runner(transform, tp_size)
runner = _tail_runner(transform, tp_world, use_ep)
group = get_tp_group().device_group
for iteration, num_tokens in enumerate((1, 5, 8, 16, 5)):
@@ -119,13 +127,22 @@ def _check_matches_replicated(device: torch.device, tp_size: int, rank: int) ->
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
def _check_matches_replicated_under_ep(
device: torch.device, tp_world: int, rank: int
) -> None:
"""The served config: the routed experts are expert-parallel, so the MoE
config reports no tensor parallelism at all for a group that still splits
the hidden dim ``tp_world`` ways."""
_check_matches_replicated(device, tp_world, rank, use_ep=True)
def _check_writes_only_its_own_shard(
device: torch.device, tp_size: int, rank: int
device: torch.device, tp_world: int, rank: int
) -> None:
"""Two ranks that swapped both weight rows and write offsets still sum to
the right total, so inspect each slice before the final collective."""
transform = _build_transform(device)
runner = _tail_runner(transform, tp_size)
runner = _tail_runner(transform, tp_world)
group = get_tp_group().device_group
torch.manual_seed(rank + 1)
@@ -149,7 +166,7 @@ def _check_writes_only_its_own_shard(
assert captured["output_is_reduced"] is False
shard = HIDDEN_SIZE // tp_size
shard = HIDDEN_SIZE // tp_world
start, end = rank * shard, (rank + 1) * shard
local = captured["states"]
projected = F.linear(latent, transform.up_proj.weight[start:end])
@@ -163,6 +180,7 @@ def _check_writes_only_its_own_shard(
_CHECKS = {
"matches_replicated": _check_matches_replicated,
"matches_replicated_ep": _check_matches_replicated_under_ep,
"own_shard_only": _check_writes_only_its_own_shard,
}
@@ -196,6 +214,11 @@ def test_sharded_tail_tp8_matches_replicated_projection() -> None:
_run_ranks("matches_replicated", 8)
@multi_gpu_test(num_gpus=8)
def test_sharded_tail_tp8_ep8_matches_replicated_projection() -> None:
_run_ranks("matches_replicated_ep", 8)
@multi_gpu_test(num_gpus=4)
def test_sharded_tail_tp4_writes_only_its_own_shard() -> None:
_run_ranks("own_shard_only", 4)
@@ -241,12 +264,19 @@ def test_forward_shards_only_when_the_tail_is_valid(
@pytest.fixture
def build_runner(monkeypatch: pytest.MonkeyPatch):
"""Construct through the real subclass ``__init__``, stubbing only the base."""
"""Construct through the real subclass ``__init__``, stubbing only the base.
``tp_world`` is the size of the TP process group and is varied independently
of ``moe_config.tp_size``, because under expert parallelism those two are
different numbers for the same eight ranks.
"""
world = {"size": 8}
def _base_init(
self,
*,
tp_size: int = 8,
ep_size: int = 1,
hidden: int = HIDDEN_SIZE,
is_sequence_parallel: bool = False,
has_up_proj: bool = True,
@@ -254,7 +284,9 @@ def build_runner(monkeypatch: pytest.MonkeyPatch):
routed_scaling_factor: float = 1.0,
) -> None:
self.moe_config = SimpleNamespace(
tp_size=tp_size, is_sequence_parallel=is_sequence_parallel
tp_size=tp_size,
ep_size=ep_size,
is_sequence_parallel=is_sequence_parallel,
)
self.routed_output_transform = SimpleNamespace(
norm=None,
@@ -266,20 +298,63 @@ def build_runner(monkeypatch: pytest.MonkeyPatch):
self._shared_experts = object() if has_shared_experts else None
monkeypatch.setattr(moe_runner.MoERunner, "__init__", _base_init)
return ROCmLatentMoERunner
monkeypatch.setattr(
latent_moe_runner,
"get_tensor_model_parallel_world_size",
lambda: world["size"],
)
def _build(*, tp_world: int = 8, **kwargs) -> ROCmLatentMoERunner:
world["size"] = tp_world
return ROCmLatentMoERunner(**kwargs)
return _build
def test_shards_under_the_kimi_k3_serving_config(build_runner) -> None:
runner = build_runner()
"""``--tensor-parallel-size 8``, MoE experts tensor-parallel."""
runner = build_runner(tp_world=8, tp_size=8, ep_size=1)
assert runner._tail_shardable
assert runner._up_proj_shard_size == HIDDEN_SIZE // 8
def test_shards_under_the_kimi_k3_serving_config_with_expert_parallel(
build_runner,
) -> None:
"""``--tensor-parallel-size 8 --enable-expert-parallel``, what is served.
EP reports ``moe_config.tp_size == 1`` because each rank owns whole experts,
but the hidden dim is still split across all eight TP ranks, so the tail
must shard exactly as it does without EP.
"""
runner = build_runner(tp_world=8, tp_size=1, ep_size=8)
assert runner._tail_shardable
assert runner._up_proj_shard_size == HIDDEN_SIZE // 8
def test_shards_by_the_process_group_not_the_dp_flattened_tp_size(
build_runner,
) -> None:
"""Without EP, ``moe_config.tp_size`` is flattened across DP: TP=2 x DP=2
reports 4 for a TP process group of 2.
Only those 2 ranks share a hidden dim and take part in the final
all-reduce, so sharding by 4 would leave half of it with no routed
contribution on either rank.
"""
runner = build_runner(tp_world=2, tp_size=4, ep_size=1)
assert runner._tail_shardable
assert runner._up_proj_shard_size == HIDDEN_SIZE // 2
@pytest.mark.parametrize(
"override",
[
pytest.param({"tp_size": 1}, id="no-tp"),
pytest.param({"tp_world": 1, "tp_size": 1}, id="single-rank"),
pytest.param({"tp_world": 1, "tp_size": 4}, id="dp-without-tp"),
pytest.param({"hidden": HIDDEN_SIZE + 2}, id="hidden-not-divisible-by-tp"),
pytest.param({"has_up_proj": False}, id="no-up-proj"),
pytest.param({"has_shared_experts": False}, id="no-shared-partial"),
@@ -290,7 +365,12 @@ def test_shards_under_the_kimi_k3_serving_config(build_runner) -> None:
def test_falls_back_when_the_config_breaks_the_shard(
build_runner, override: dict
) -> None:
"""Each of these makes the sharded tail wrong, not merely slower."""
"""Each of these makes the sharded tail wrong, not merely slower.
``dp-without-tp`` is DP=4 with no TP: the hidden dim is not split at all,
so there is nothing to shard and no all-reduce to stitch shards back
together, however large ``moe_config.tp_size`` claims to be.
"""
runner = build_runner(**override)
assert not runner._tail_shardable
@@ -16,7 +16,9 @@ from vllm import LLM, SamplingParams
from vllm.assets.audio import AudioAsset
from vllm.config import CUDAGraphMode
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.engine.protocol import StreamingInput
from vllm.platforms import current_platform
from vllm.renderers.inputs.preprocess import parse_model_prompt
from vllm.utils.math_utils import cdiv
from vllm.v1.engine.async_llm import AsyncLLM
from vllm.v1.kv_cache_interface import SlidingWindowSpec
@@ -222,6 +224,14 @@ async def test_voxtral_realtime_generator(audio_assets, tokenizer, async_engine)
sampling_params = SamplingParams(temperature=0.0, max_tokens=1)
audio_config = tokenizer.instruct_tokenizer.audio_encoder.audio_config
async def input_stream(audio_buffer):
async for prompt in audio_buffer.get_input_stream():
parsed_prompt = parse_model_prompt(async_engine.model_config, prompt)
(engine_input,) = await async_engine.renderer.render_cmpl_async(
[parsed_prompt]
)
yield StreamingInput(prompt=engine_input)
output_tokens_list = []
for i, audio_asset in enumerate(audio_assets):
output_tokens = []
@@ -241,7 +251,7 @@ async def test_voxtral_realtime_generator(audio_assets, tokenizer, async_engine)
request_id = f"session-{i}"
async for resp in async_engine.generate(
prompt=buffer.get_input_stream(),
prompt=input_stream(buffer),
sampling_params=sampling_params,
request_id=request_id,
):
@@ -34,6 +34,7 @@ def vqa_ppl_test(
mm_processor_kwargs = {}
processor = AutoProcessor.from_pretrained(model_info.name)
image_token_id = processor.image_token_id
images = []
prompts = []
@@ -99,8 +100,10 @@ def vqa_ppl_test(
for token_data in token_datas[1:]:
assert token_data is not None
assert len(token_data) == 1
token_log_prob = list(token_data.values())[0].logprob
token_log_probs.append(token_log_prob)
token_id, token_log_prob = next(iter(token_data.items()))
if token_id == image_token_id:
continue
token_log_probs.append(token_log_prob.logprob)
neg_log_likelihood = -torch.tensor(
token_log_probs, dtype=torch.float32, device="cpu"
@@ -132,11 +135,12 @@ def vqa_ppl_test(
inputs = inputs.to("cuda")
input_ids = inputs["input_ids"]
outputs = hf_model.model(**inputs, labels=input_ids)
labels = input_ids.masked_fill(input_ids == image_token_id, -100)
outputs = hf_model.model(**inputs, labels=labels)
neg_log_likelihood = outputs.loss
neg_log_likelihood = neg_log_likelihood.to(torch.float32).cpu()
num_loss_tokens = input_ids.shape[1] - 1
num_loss_tokens = int((labels[:, 1:] != -100).sum())
nll_sum += neg_log_likelihood * num_loss_tokens
n_tokens += num_loss_tokens
@@ -8,26 +8,51 @@ from tests.models.utils import GenerateModelInfo
from .ppl_utils import vqa_ppl_test
MODELS = [
GenerateModelInfo("Qwen/Qwen2-VL-2B-Instruct", hf_ppl=41081356.0),
GenerateModelInfo("Qwen/Qwen2.5-VL-3B-Instruct", hf_ppl=18330016.0),
(
GenerateModelInfo("Qwen/Qwen2-VL-2B-Instruct", hf_ppl=9.810060501098633),
28,
{},
),
(
GenerateModelInfo("Qwen/Qwen2.5-VL-3B-Instruct", hf_ppl=22.17719078063965),
28,
{},
),
(
GenerateModelInfo("Qwen/Qwen3-VL-4B-Instruct", hf_ppl=11.916606903076172),
32,
{},
),
(
GenerateModelInfo("Qwen/Qwen3.5-4B", hf_ppl=6.478141784667969),
32,
{"enable_prefix_caching": False},
),
]
mm_processor_kwargs = {
"min_pixels": 28 * 28,
"max_pixels": 1280 * 28 * 28,
}
def _pixel_factor(factor: int):
"""`factor = patch_size * merge_size` for Qwen models"""
return {"min_pixels": factor**2, "max_pixels": 1280 * factor**2}
@pytest.mark.parametrize("model_info", MODELS)
@pytest.mark.parametrize("model_info,factor,extra_kwargs", MODELS)
@pytest.mark.parametrize("mm_device_do_normalize", [True, False])
def test_ppl(
hf_runner, vllm_runner, model_info: GenerateModelInfo, mm_device_do_normalize: bool
hf_runner,
vllm_runner,
model_info: GenerateModelInfo,
factor: int,
extra_kwargs: dict,
mm_device_do_normalize: bool,
):
vqa_ppl_test(
hf_runner,
vllm_runner,
model_info,
vllm_extra_kwargs={"mm_device_do_normalize": mm_device_do_normalize},
mm_processor_kwargs=mm_processor_kwargs,
vllm_extra_kwargs={
"mm_device_do_normalize": mm_device_do_normalize,
**extra_kwargs,
},
mm_processor_kwargs=_pixel_factor(factor),
)
@@ -202,19 +202,19 @@ def test_get_prompt_updates_respects_nested_max_soft_tokens(model_id: str):
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
@pytest.mark.parametrize("kwargs_on_init", [False, True])
@pytest.mark.parametrize(
"image_kwargs", [{"rescale_factor": 1 / 127.5}, {"max_soft_tokens": 560}]
)
@pytest.mark.parametrize("video_uuid", [None, "same-video"])
def test_video_cache_is_independent_of_image_kwargs(
model_id: str,
kwargs_on_init: bool,
image_kwargs: dict[str, object],
video_uuid: str | None,
):
"""Image overrides must not change video frames or depend on cache warmth."""
kwargs = {"images_kwargs": image_kwargs}
"""Request image overrides must not change video frames or make results
depend on cache warmth.
"""
request_kwargs = {"images_kwargs": image_kwargs}
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1, "video": 1},
@@ -239,18 +239,11 @@ def test_video_cache_is_independent_of_image_kwargs(
cache=cache,
)
baseline = process({}, cache=None)
if kwargs_on_init:
ctx = build_model_context(
model_id,
mm_processor_kwargs=kwargs,
limit_mm_per_prompt={"image": 1, "video": 1},
mm_processor_cache_gb=1,
)
cache = MultiModalProcessorOnlyCache(ctx.model_config)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
request_kwargs = {} if kwargs_on_init else kwargs
process(request_kwargs, cache=cache)
# Populate the video cache without image overrides, then change only
# images_kwargs on the next request. Since images_kwargs are excluded from
# the video hash, and must not affect the processed video, the cached result
# must match a fresh recomputation.
baseline = process({}, cache=cache)
cached = process(request_kwargs, cache=cache)
cache.clear_cache()
fresh = process(request_kwargs, cache=cache)
@@ -100,6 +100,11 @@ def _expected_placeholder_tokens_per_image(
[
({}, (448, 448), 256),
({"size": {"longest_edge": 1008}}, (1540, 1540), 1296),
(
{"images_kwargs": {"size": {"longest_edge": 1008}}},
(1540, 1540),
1296,
),
({"size": {"longest_edge": 1288}}, (1536, 1187), 1656),
({"size": {"longest_edge": 1008}}, (29, 29), 4),
({"size": {"longest_edge": 1000}}, (1540, 1700), 1188),
@@ -148,6 +153,40 @@ def test_processor_size_override(
assert prompt_update_tokens == hf_placeholder_tokens
def test_scoped_request_size_overrides_configured_flat_size():
ctx = build_model_context(
_MODEL_ID,
mm_processor_kwargs={"size": {"longest_edge": 1008}},
limit_mm_per_prompt={"image": 1},
model_config_kwargs=_MODEL_CONFIG_KWARGS,
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor = AutoProcessor.from_pretrained(
_MODEL_ID,
fix_mistral_regex=True,
)
hf_processor_mm_kwargs: dict[str, object] = {
"images_kwargs": {"size": {"longest_edge": 1288}}
}
image = Image.new("RGB", (1536, 1187), color=(127, 127, 127))
merged_mm_kwargs = processor.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
pixel_values, hf_placeholder_tokens = _process_images_with_hf(
hf_processor, [image], merged_mm_kwargs
)
prompt_update_tokens = _placeholder_tokens_from_prompt_updates(
processor, [image], pixel_values, hf_processor_mm_kwargs
)
expected_from_pixel_values = _expected_placeholder_tokens_per_image(
hf_processor, pixel_values[0]
)
assert expected_from_pixel_values == 1656
assert hf_placeholder_tokens.count(hf_processor.image_token_id) == 1656
assert prompt_update_tokens == hf_placeholder_tokens
def test_lightonocr_keeps_vision_config_image_size():
ctx = build_model_context(
_LIGHTON_MODEL_ID,
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from copy import deepcopy
from types import SimpleNamespace
from typing import Any
import numpy as np
@@ -8,16 +10,238 @@ import pytest
import torch
from PIL import Image
from vllm.config.multimodal import MultiModalConfig
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.model_executor.models.qwen2_vl import Qwen2VLProcessingInfo
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
from vllm.multimodal.inputs import batched_tensors_equal
from vllm.multimodal.processing.context import InputProcessingContext
from vllm.platforms import current_platform
from ....conftest import ImageTestAssets
from ...utils import build_model_context
@pytest.mark.parametrize(
("mm_kwargs", "expected"),
[
(
{"size": {"shortest_edge": 64, "longest_edge": 1024}},
{
"size": {"shortest_edge": 64, "longest_edge": 1024},
"min_pixels": 64,
"max_pixels": 1024,
},
),
(
{"min_pixels": 64, "max_pixels": 1024},
{
"min_pixels": 64,
"max_pixels": 1024,
"size": {"shortest_edge": 64, "longest_edge": 1024},
},
),
(
{
"size": {"shortest_edge": 64, "longest_edge": 1024},
"min_pixels": 128,
"max_pixels": 2048,
},
{
"size": {"shortest_edge": 64, "longest_edge": 1024},
"min_pixels": 128,
"max_pixels": 2048,
},
),
(
{
"min_pixels": None,
"size": {"shortest_edge": 0},
"images_kwargs": {"max_pixels": 4096},
"videos_kwargs": {"size": {"longest_edge": 8192}},
},
{
"min_pixels": None,
"size": {"shortest_edge": 0},
"images_kwargs": {
"max_pixels": 4096,
"size": {"longest_edge": 4096},
},
"videos_kwargs": {
"size": {"longest_edge": 8192},
"max_pixels": 8192,
},
},
),
(
{"min_pixels": 0},
{"min_pixels": 0, "size": {"shortest_edge": 0}},
),
(
{"size": None, "min_pixels": 64},
{"size": {"shortest_edge": 64}, "min_pixels": 64},
),
],
)
def test_complete_mm_processor_size_aliases(
mm_kwargs: dict[str, object],
expected: dict[str, object],
) -> None:
original = deepcopy(mm_kwargs)
actual = Qwen2VLProcessingInfo._complete_mm_processor_size_aliases(mm_kwargs)
assert actual == expected
assert mm_kwargs == original
def test_merge_and_resolve_mm_processor_kwargs_preserves_request_alias_precedence(
monkeypatch: pytest.MonkeyPatch,
) -> None:
mm_config = MultiModalConfig(
mm_processor_kwargs={
"images_kwargs": {"min_pixels": 64},
},
mm_device_do_normalize=False,
)
model_config = SimpleNamespace(get_multimodal_config=lambda: mm_config)
ctx = InputProcessingContext(model_config, tokenizer=None) # type: ignore[arg-type]
info = Qwen2VLProcessingInfo(ctx)
monkeypatch.setattr(
info,
"get_supported_mm_processor_kwargs",
lambda: {"images_kwargs": {"size", "min_pixels"}},
)
assert info._merge_and_resolve_mm_processor_kwargs(
{"size": {"shortest_edge": 128}}
) == {
"images_kwargs": {
"size": {"shortest_edge": 128},
"min_pixels": 128,
},
}
@pytest.mark.parametrize(
("mm_kwargs", "default_size", "expected"),
[
(
{},
{"shortest_edge": 32, "longest_edge": 1024},
{"shortest_edge": 32, "longest_edge": 1024},
),
(
{"size": {"shortest_edge": 64}},
{"shortest_edge": 32, "longest_edge": 1024},
{"shortest_edge": 64, "longest_edge": 1024},
),
(
{
"size": {"shortest_edge": 64, "longest_edge": 2048},
"min_pixels": 128,
"max_pixels": 4096,
},
{"shortest_edge": 32, "longest_edge": 1024},
{"shortest_edge": 128, "longest_edge": 4096},
),
(
{
"size": {"shortest_edge": 64},
"max_pixels": 4096,
},
None,
{"shortest_edge": 64, "longest_edge": 4096},
),
(
{
"size": {"shortest_edge": 64, "longest_edge": 2048},
"min_pixels": None,
"max_pixels": 0,
},
{"shortest_edge": 32, "longest_edge": 1024},
{"shortest_edge": 64, "longest_edge": 0},
),
],
)
def test_get_vision_size(
mm_kwargs: dict[str, object],
default_size: dict[str, object] | None,
expected: dict[str, object],
) -> None:
original_mm_kwargs = {
key: dict(value) if isinstance(value, dict) else value
for key, value in mm_kwargs.items()
}
original_default_size = dict(default_size) if default_size is not None else None
actual = Qwen2VLProcessingInfo._get_vision_size(
mm_kwargs,
default_size=default_size,
)
assert actual == expected
assert mm_kwargs == original_mm_kwargs
assert default_size == original_default_size
@pytest.mark.parametrize(
("modality", "scope"),
[
("image", "images_kwargs"),
("video", "videos_kwargs"),
],
)
def test_get_vision_info_reads_merged_modality_scope(
monkeypatch: pytest.MonkeyPatch,
modality: str,
scope: str,
) -> None:
vision_config = SimpleNamespace(
patch_size=14,
spatial_merge_size=2,
temporal_patch_size=2,
)
ctx = SimpleNamespace(
get_hf_config=lambda *_: SimpleNamespace(vision_config=vision_config)
)
info = Qwen2VLProcessingInfo(ctx)
merged = {
"images_kwargs": {"size": {"shortest_edge": 111}},
"videos_kwargs": {"size": {"shortest_edge": 222}},
}
monkeypatch.setattr(
info,
"_merge_and_resolve_mm_processor_kwargs",
lambda _: merged,
)
seen: list[dict[str, object]] = []
def get_size(
mm_kwargs: dict[str, object],
default_size: dict[str, object] | None = None,
) -> dict[str, object]:
seen.append(mm_kwargs)
return dict(default_size or {})
monkeypatch.setattr(info, "_get_vision_size", get_size)
image_processor = SimpleNamespace(size={"shortest_edge": 32, "longest_edge": 1024})
info._get_vision_info(
image_width=28,
image_height=28,
num_frames=2,
do_resize=False,
image_processor=image_processor,
mm_kwargs={},
modality=modality, # type: ignore[arg-type]
)
assert seen == [merged[scope]]
def test_jina_vl_processing_order() -> None:
"""Jina's document-first prompt keeps cached features and hashes aligned."""
ctx = build_model_context(
@@ -1,19 +1,21 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Regression tests for Qwen3-VL processor.
Covers the fix for num_frames-based timestamp calculation
(issue vllm-project/vllm#35909).
"""
"""Regression tests for Qwen3-VL multimodal processing."""
from copy import deepcopy
from types import SimpleNamespace
from typing import Any
import numpy as np
import pytest
import torch
from transformers.models.qwen3_vl import Qwen3VLVideoProcessor
from vllm.config import ModelConfig
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.model_executor.models.qwen3_vl import Qwen3VLProcessingInfo
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.processing.context import BaseProcessingInfo
from ...registry import HF_EXAMPLE_MODELS
from ...utils import build_model_context
@@ -21,6 +23,32 @@ from ...utils import build_model_context
MODEL_ID = "Qwen/Qwen3-VL-4B-Instruct"
def test_qwen3_video_routing_schema_adds_vllm_compatibility_kwargs(
monkeypatch: pytest.MonkeyPatch,
) -> None:
native_video_kwargs = set(Qwen3VLVideoProcessor.valid_kwargs.__annotations__)
assert "size" in native_video_kwargs
assert "min_pixels" not in native_video_kwargs
assert "max_pixels" not in native_video_kwargs
info = Qwen3VLProcessingInfo(SimpleNamespace())
parent_supported = {
"text_kwargs": {"padding"},
"images_kwargs": {"size", "min_pixels", "max_pixels"},
"videos_kwargs": set(native_video_kwargs),
}
monkeypatch.setattr(
BaseProcessingInfo,
"get_supported_mm_processor_kwargs",
lambda self: {scope: set(kwargs) for scope, kwargs in parent_supported.items()},
)
supported = info.supported_mm_processor_kwargs
assert {"min_pixels", "max_pixels"} <= supported["videos_kwargs"]
assert parent_supported["videos_kwargs"] == native_video_kwargs
def _build_video_mm_data(
num_frames: int,
width: int = 128,
@@ -45,6 +73,122 @@ def _build_video_mm_data(
return {"video": [(video, metadata)]}
@pytest.mark.parametrize("model_id", [MODEL_ID])
def test_processor_partial_scoped_image_size(model_id: str) -> None:
"""Partial image ``size`` must be completed and applied before the HF call.
Keep video compatibility and per-video kwargs in an image-only call as well:
Qwen3 must sanitize the video scope before HF validates it. Regression for
#56363.
"""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1, "video": 0},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
image = np.zeros((128, 128, 3), dtype=np.uint8)
min_pixels = 256 * 32 * 32
default_size = dict(processor.info.get_image_processor().size)
complete_size = default_size | {"shortest_edge": min_pixels}
def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor:
mm_items = processor.info.parse_mm_data({"image": [image]})
return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
"image_grid_thw"
]
stock_grid = process({})
complete_grid = process({"images_kwargs": {"size": complete_size}})
partial_grid = process(
{
"images_kwargs": {
"size": {"shortest_edge": min_pixels},
},
"videos_kwargs": {
"min_pixels": 4096,
"max_pixels": 469_762_048,
"fps": [2.0],
"num_frames": [8],
},
}
)
assert torch.equal(partial_grid, complete_grid)
assert not torch.equal(partial_grid, stock_grid)
@pytest.mark.parametrize("model_id", [MODEL_ID])
def test_processor_partial_scoped_video_size(model_id: str) -> None:
"""Partial scoped video ``size`` must be completed and applied before HF."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 0, "video": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
min_pixels = 512 * 32 * 32
default_size = dict(processor.info.get_video_processor().size)
complete_size = default_size | {"shortest_edge": min_pixels}
def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor:
mm_items = processor.info.parse_mm_data(_build_video_mm_data(num_frames=8))
return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
"video_grid_thw"
]
stock_grid = process({})
complete_grid = process({"videos_kwargs": {"size": complete_size}})
partial_grid = process(
{
"videos_kwargs": {
"size": {"shortest_edge": min_pixels},
},
}
)
assert torch.equal(partial_grid, complete_grid)
assert not torch.equal(partial_grid, stock_grid)
@pytest.mark.parametrize("model_id", [MODEL_ID])
def test_processor_flat_video_pixel_aliases_match_scoped_size(model_id: str) -> None:
"""Flat min/max pixel aliases keep their historical Qwen3 video effect."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 0, "video": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
pixel_budget = 512 * 32 * 32
def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor:
mm_items = processor.info.parse_mm_data(_build_video_mm_data(num_frames=8))
return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
"video_grid_thw"
]
stock_grid = process({})
scoped_grid = process(
{
"videos_kwargs": {
"size": {
"shortest_edge": pixel_budget,
"longest_edge": pixel_budget,
},
},
}
)
flat_grid = process(
{
"min_pixels": pixel_budget,
"max_pixels": pixel_budget,
}
)
assert torch.equal(flat_grid, scoped_grid)
assert not torch.equal(flat_grid, stock_grid)
@pytest.mark.parametrize("model_id", [MODEL_ID])
@pytest.mark.parametrize(
"num_frames",
@@ -189,20 +333,21 @@ def test_processor_kwargs_videos_kwargs_does_not_leak_into_image_budget(
@pytest.mark.parametrize("model_id", [MODEL_ID])
@pytest.mark.parametrize(
"hf_mm_kwargs",
[{"num_frames": [8, 16]}, {"fps": [2.0, 4.0]}],
("hf_mm_kwargs", "scalar_kwargs"),
[
({"num_frames": [8, 16]}, [{"num_frames": 8}, {"num_frames": 16}]),
({"fps": [4.0, 12.0]}, [{"fps": 4.0}, {"fps": 12.0}]),
],
)
def test_processor_multi_video_list_kwargs(
model_id: str,
hf_mm_kwargs: dict[str, Any],
scalar_kwargs: list[dict[str, Any]],
) -> None:
"""Regression test: a multi-video request with list-valued per-video
``mm_processor_kwargs`` (one ``fps``/``num_frames`` per video) must not
crash.
"""List-valued video kwargs must be scalarized per item without mutation.
Before the fix, ``_apply_hf_processor_main`` copied the whole kwargs to every
video without slicing, so ``_get_video_second_idx`` received the list
where a scalar was expected and raised ``TypeError``.
Each video in the multi-video request must match processing the same input
independently with its corresponding scalar ``fps``/``num_frames`` value.
"""
ctx = build_model_context(
model_id,
@@ -216,17 +361,36 @@ def test_processor_multi_video_list_kwargs(
)
mm_data = {
"video": [
_build_video_mm_data(num_frames=16)["video"][0],
_build_video_mm_data(num_frames=32)["video"][0],
_build_video_mm_data(num_frames=32)["video"][0],
]
}
hf_mm_kwargs_before = deepcopy(hf_mm_kwargs)
processed = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=hf_mm_kwargs,
)
def process_single(hf_kwargs: dict[str, Any]) -> torch.Tensor:
single_prompt = "<|vision_start|><|video_pad|><|vision_end|>"
single_mm_data = _build_video_mm_data(num_frames=32)
single = processor(
single_prompt,
mm_items=processor.info.parse_mm_data(single_mm_data),
hf_processor_mm_kwargs=hf_kwargs,
)
return single["mm_kwargs"].get_data()["video_grid_thw"][0]
expected_grids = [process_single(kwargs) for kwargs in scalar_kwargs]
multi_grids = processed["mm_kwargs"].get_data()["video_grid_thw"]
assert not torch.equal(expected_grids[0], expected_grids[1])
assert torch.equal(multi_grids[0], expected_grids[0])
assert torch.equal(multi_grids[1], expected_grids[1])
assert hf_mm_kwargs == hf_mm_kwargs_before
video_phs = processed["mm_placeholders"].get("video", [])
assert len(video_phs) == 2, (
f"Expected exactly 2 video placeholders, got {len(video_phs)}"
@@ -336,7 +500,12 @@ def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> Non
capped_ctx = build_model_context(
model_id,
mm_processor_kwargs={"size": size, "cap_pixels_per_frame": True},
mm_processor_kwargs={
"videos_kwargs": {
"size": size,
"cap_pixels_per_frame": True,
},
},
limit_mm_per_prompt={"image": 0, "video": 1},
)
capped = MULTIMODAL_REGISTRY.create_processor(capped_ctx.model_config)
@@ -350,7 +519,7 @@ def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> Non
uncapped_ctx = build_model_context(
model_id,
mm_processor_kwargs={"size": size},
mm_processor_kwargs={"videos_kwargs": {"size": size}},
limit_mm_per_prompt={"image": 0, "video": 1},
)
uncapped = MULTIMODAL_REGISTRY.create_processor(uncapped_ctx.model_config)
+30
View File
@@ -4,11 +4,13 @@
import pytest
import torch
from vllm.models.qwen4_exp.nvidia.ops.cute_dsl.hc_down_silu import hc_down_silu
from vllm.models.qwen4_exp.nvidia.ops.hc import (
grouped_gemma_rmsnorm,
hc_combine,
hc_combine_norm,
hc_gate_mix,
hc_silu,
)
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
@@ -22,6 +24,13 @@ HC = 4
HIDDEN_SIZE = 2560
HYPER_HIDDEN_SIZE = HC * HIDDEN_SIZE
EPS = 1e-6
LORA_RANK = 320
DOWN_N = LORA_RANK + HC + 12 # merged down+inject weight, 16-row padded
requires_sm90 = pytest.mark.skipif(
not current_platform.has_device_capability(90),
reason="fused HC down+SiLU requires SM90+",
)
def test_grouped_gemma_rmsnorm() -> None:
@@ -125,3 +134,24 @@ def test_hc_combine_norm_unit_injection(num_tokens: int) -> None:
expected_norm = grouped_gemma_rmsnorm(expected, weight, EPS, HC)
assert torch.equal(actual, expected)
torch.testing.assert_close(actual_norm, expected_norm)
@requires_sm90
@pytest.mark.parametrize("num_tokens", [1, 3, 5, 17, 48])
def test_hc_down_silu_fused(num_tokens: int) -> None:
# Compare computed columns with the unfused ll_bf16 + hc_silu reference.
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import ll_bf16_gemm
torch.manual_seed(0)
x = torch.randn(num_tokens, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(DOWN_N, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
lora, injection = hc_down_silu(x, weight, LORA_RANK, HC)
down = ll_bf16_gemm(x, weight).to(torch.bfloat16)
torch.testing.assert_close(
lora, hc_silu(down[:, :LORA_RANK], HC), rtol=0.01, atol=0.01
)
torch.testing.assert_close(
injection, down[:, LORA_RANK : LORA_RANK + HC], rtol=0.01, atol=0.01
)
+125
View File
@@ -20,6 +20,9 @@ import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module
import vllm.model_executor.parameter as parameter_module
import vllm.models.qwen4_exp.common.ngram_embedding as ngram_embedding_module
import vllm.models.qwen4_exp.nvidia.ngram_embedding as nvidia_ngram_embedding_module
from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata
from vllm.config import SpeculativeConfig, VllmConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.config.quantization import QuantizationConfigArgs
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.model_executor.layers.quantization.modelopt import (
@@ -50,8 +53,125 @@ from vllm.models.qwen4_exp.nvidia.ple_layer import Qwen4ExpPLELayer
from vllm.utils.torch_utils import weak_ref_tensor
from vllm.v1.attention.backends.short_conv_attn import (
PleShortConvAttentionMetadata,
PleShortConvAttentionMetadataBuilder,
)
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
from vllm.v1.kv_cache_interface import MambaSpec
@pytest.mark.parametrize("device", ["cpu", "cuda"])
@pytest.mark.parametrize("full_graph", [False, True])
@pytest.mark.parametrize(
"query_lens,drafts,spec_rows,non_spec_rows,token_order",
[
pytest.param([4, 2, 0], [3, 1, -1], [0, 1], [], None, id="spec-padding"),
pytest.param(
[3, 1, 2, 5, 0],
[2, -1, 1, -1, -1],
[0, 2],
[1, 3],
[0, 1, 2, 4, 5, 3, 6, 7, 8, 9, 10],
id="mixed-padding",
),
pytest.param([1, 5], [-1, -1], [], [0, 1], None, id="non-spec"),
],
)
def test_ple_metadata_preserves_request_and_token_order(
device, full_graph, query_lens, drafts, spec_rows, non_spec_rows, token_order
):
"""Preserve state routing, acceptance and prefill offsets without GPU sync."""
if device == "cuda" and not torch.cuda.is_available():
pytest.skip("CUDA required")
device = torch.device(device)
config = VllmConfig()
config.speculative_config = SpeculativeConfig(
method="ngram", num_speculative_tokens=3
)
config.compilation_config.cudagraph_mode = (
CUDAGraphMode.FULL if full_graph else CUDAGraphMode.NONE
)
config.compilation_config.max_cudagraph_capture_size = 32
builder = PleShortConvAttentionMetadataBuilder(
MambaSpec(block_size=16, shapes=((16, 64),), dtypes=(torch.float32,)),
["ple"],
config,
device,
)
common = create_common_attn_metadata(
BatchSpec(seq_lens=[q + 16 for q in query_lens], query_lens=query_lens),
16,
device,
arange_block_indices=True,
)
common.is_prefilling = torch.tensor(
[q > 1 and d < 0 for q, d in zip(query_lens, drafts)]
)
# Token padding is independent of the sum of per-request query lengths.
common.num_actual_tokens += 3
accepted = torch.arange(1, len(query_lens) + 1, dtype=torch.int32, device=device)
drafts_cpu = torch.tensor(drafts, dtype=torch.int32)
sync_mode = torch.cuda.get_sync_debug_mode() if device.type == "cuda" else None
if sync_mode is not None:
torch.cuda.set_sync_debug_mode("error")
try:
metadata = builder.build(
0,
common,
num_accepted_tokens=accepted,
num_decode_draft_tokens_cpu=drafts_cpu,
)
finally:
if sync_mode is not None:
torch.cuda.set_sync_debug_mode(sync_mode)
def check(actual, expected):
torch.testing.assert_close(
actual, torch.tensor(expected, dtype=actual.dtype, device=device)
)
assert metadata.num_spec_decodes == len(spec_rows)
assert metadata.num_decodes == sum(query_lens[i] == 1 for i in non_spec_rows)
assert metadata.num_prefills == sum(query_lens[i] > 1 for i in non_spec_rows)
if spec_rows:
torch.testing.assert_close(
metadata.spec_state_indices_tensor[: len(spec_rows)],
common.block_table_tensor[spec_rows, 0],
)
torch.testing.assert_close(
metadata.num_accepted_tokens[: len(spec_rows)], accepted[spec_rows]
)
check(
metadata.spec_query_start_loc[: len(spec_rows) + 1],
[0, *accumulate(query_lens[i] for i in spec_rows)],
)
if full_graph and not non_spec_rows:
check(metadata.spec_state_indices_tensor[len(spec_rows) :], [NULL_BLOCK_ID])
check(metadata.num_accepted_tokens[len(spec_rows) :], [1])
check(
metadata.spec_query_start_loc[len(spec_rows) + 1 :], [sum(query_lens)]
)
assert (
metadata.spec_state_indices_tensor.data_ptr()
== builder.spec_state_indices_tensor.data_ptr()
)
if non_spec_rows:
torch.testing.assert_close(
metadata.state_indices_tensor, common.block_table_tensor[non_spec_rows, 0]
)
prefill_lens = [query_lens[i] for i in non_spec_rows if query_lens[i] > 1]
check(metadata.query_start_loc_p, [0, *accumulate(prefill_lens)])
check(metadata.has_initial_states_p, [True] * len(prefill_lens))
if token_order is not None:
check(
torch.cat((metadata.spec_token_indx, metadata.non_spec_token_indx)),
token_order,
)
else:
assert metadata.spec_token_indx is None
assert metadata.non_spec_token_indx is None
assert metadata.nums_dict is None
assert metadata.batch_ptr is None
assert metadata.token_chunk_offset_ptr is None
def _mock_etp_group(
@@ -1892,6 +2012,11 @@ def _make_conv_metadata(
else None
),
non_spec_query_start_loc=(non_spec_query_start_loc if has_non_spec else None),
query_start_loc_p=(
non_spec_query_start_loc[case.num_decodes :] - case.num_decodes
if num_prefills
else None
),
has_initial_states_p=(
torch.arange(num_prefills, device=device) % 2 == 0 if num_prefills else None
),
+22 -1
View File
@@ -17,6 +17,7 @@ from vllm.models.qwen4_exp.nvidia.ops import qsa as qsa_ops
from vllm.models.qwen4_exp.nvidia.ops import qsa_indexer as qsa_indexer_ops
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
from vllm.v1.worker.utils import clear_layer_kv_caches
requires_qsa_kernels = pytest.mark.skipif(
not current_platform.is_cuda() or not HAS_TRITON,
@@ -365,7 +366,9 @@ def test_qsa_circular_buffer_survives_one_speculative_step(chunk_start: int) ->
assert set(slots.tolist()).isdisjoint((committed % capacity).tolist())
def _qsa_key_cache(block_size: int, compress_ratio: int) -> qsa_cache.QSAKeyStateCache:
def _qsa_key_cache(
block_size: int, compress_ratio: int, **kwargs
) -> qsa_cache.QSAKeyStateCache:
return qsa_cache.QSAKeyStateCache(
head_size=64,
dtype=torch.bfloat16,
@@ -375,6 +378,7 @@ def _qsa_key_cache(block_size: int, compress_ratio: int) -> qsa_cache.QSAKeyStat
compilation_config=SimpleNamespace(static_forward_context={})
),
compress_ratio=compress_ratio,
**kwargs,
)
@@ -402,6 +406,23 @@ def test_qsa_state_caches_adapt_the_unified_logical_layout() -> None:
assert compressed_cache.kv_cache.data_ptr() == compressed_view.data_ptr()
def test_clearing_qsa_key_cache_releases_its_storage() -> None:
"""Profiling teardown clears `kv_cache`; no derived view may keep it alive."""
raw_cache = _qsa_key_cache(
block_size=32, compress_ratio=4, cache_rope_positions=True
)
kv = torch.zeros(2, 1, 8, raw_cache.head_size, dtype=torch.bfloat16)
references_before_bind = torch._C._storage_Use_Count(kv.untyped_storage()._cdata)
raw_cache.bind_kv_cache(kv)
assert raw_cache.key_cache.shape[-1] == 64
assert raw_cache.rope_position_cache.dtype == torch.int64
clear_layer_kv_caches([raw_cache])
references = torch._C._storage_Use_Count(kv.untyped_storage()._cdata)
assert references == references_before_bind
@pytest.mark.parametrize(
("compress_ratio", "num_spec", "expected"),
[(4, 0, 4), (4, 1, 8), (4, 3, 8), (4, 4, 8), (4, 5, 12), (2, 3, 6)],