Merge remote-tracking branch 'upstream/main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -23,6 +23,7 @@ from vllm.distributed import get_tp_group
|
||||
from vllm.model_executor.layers.fused_moe.runner import moe_runner
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.models.kimi_k3.amd import latent_moe_runner
|
||||
from vllm.models.kimi_k3.amd.latent_moe_runner import ROCmLatentMoERunner
|
||||
from vllm.models.kimi_k3.amd.linear import KimiRoutedOutputTransform
|
||||
from vllm.platforms import current_platform
|
||||
@@ -56,21 +57,26 @@ def _build_transform(device: torch.device) -> KimiRoutedOutputTransform:
|
||||
|
||||
|
||||
def _tail_runner(
|
||||
transform: KimiRoutedOutputTransform, tp_size: int
|
||||
transform: KimiRoutedOutputTransform, tp_world: int, use_ep: bool = False
|
||||
) -> ROCmLatentMoERunner:
|
||||
"""A runner carrying only what the tail reads, so no engine is needed.
|
||||
|
||||
``_maybe_reduce_final_output`` is left as the real base-class method, so the
|
||||
all-reduce that stitches the shards is the real collective.
|
||||
|
||||
``tp_world`` is the size of the TP process group, which is what the hidden
|
||||
dim is split by. Under expert parallelism the MoE config reports
|
||||
``tp_size=1``/``ep_size=tp_world`` for that same group, so ``use_ep`` pins
|
||||
that the final reduce still fires.
|
||||
"""
|
||||
runner = object.__new__(ROCmLatentMoERunner)
|
||||
attrs = {
|
||||
"routed_output_transform": transform,
|
||||
"_up_proj_shard_size": HIDDEN_SIZE // tp_size,
|
||||
"_up_proj_shard_size": HIDDEN_SIZE // tp_world,
|
||||
"_logged_sharded_tail": False,
|
||||
"moe_config": SimpleNamespace(
|
||||
tp_size=tp_size,
|
||||
ep_size=1,
|
||||
tp_size=1 if use_ep else tp_world,
|
||||
ep_size=tp_world if use_ep else 1,
|
||||
is_sequence_parallel=False,
|
||||
skip_final_all_reduce=False,
|
||||
),
|
||||
@@ -94,9 +100,11 @@ def _rank_partials(
|
||||
return routed.mul_(0.01), shared
|
||||
|
||||
|
||||
def _check_matches_replicated(device: torch.device, tp_size: int, rank: int) -> None:
|
||||
def _check_matches_replicated(
|
||||
device: torch.device, tp_world: int, rank: int, use_ep: bool = False
|
||||
) -> None:
|
||||
transform = _build_transform(device)
|
||||
runner = _tail_runner(transform, tp_size)
|
||||
runner = _tail_runner(transform, tp_world, use_ep)
|
||||
group = get_tp_group().device_group
|
||||
|
||||
for iteration, num_tokens in enumerate((1, 5, 8, 16, 5)):
|
||||
@@ -119,13 +127,22 @@ def _check_matches_replicated(device: torch.device, tp_size: int, rank: int) ->
|
||||
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
|
||||
|
||||
|
||||
def _check_matches_replicated_under_ep(
|
||||
device: torch.device, tp_world: int, rank: int
|
||||
) -> None:
|
||||
"""The served config: the routed experts are expert-parallel, so the MoE
|
||||
config reports no tensor parallelism at all for a group that still splits
|
||||
the hidden dim ``tp_world`` ways."""
|
||||
_check_matches_replicated(device, tp_world, rank, use_ep=True)
|
||||
|
||||
|
||||
def _check_writes_only_its_own_shard(
|
||||
device: torch.device, tp_size: int, rank: int
|
||||
device: torch.device, tp_world: int, rank: int
|
||||
) -> None:
|
||||
"""Two ranks that swapped both weight rows and write offsets still sum to
|
||||
the right total, so inspect each slice before the final collective."""
|
||||
transform = _build_transform(device)
|
||||
runner = _tail_runner(transform, tp_size)
|
||||
runner = _tail_runner(transform, tp_world)
|
||||
group = get_tp_group().device_group
|
||||
|
||||
torch.manual_seed(rank + 1)
|
||||
@@ -149,7 +166,7 @@ def _check_writes_only_its_own_shard(
|
||||
|
||||
assert captured["output_is_reduced"] is False
|
||||
|
||||
shard = HIDDEN_SIZE // tp_size
|
||||
shard = HIDDEN_SIZE // tp_world
|
||||
start, end = rank * shard, (rank + 1) * shard
|
||||
local = captured["states"]
|
||||
projected = F.linear(latent, transform.up_proj.weight[start:end])
|
||||
@@ -163,6 +180,7 @@ def _check_writes_only_its_own_shard(
|
||||
|
||||
_CHECKS = {
|
||||
"matches_replicated": _check_matches_replicated,
|
||||
"matches_replicated_ep": _check_matches_replicated_under_ep,
|
||||
"own_shard_only": _check_writes_only_its_own_shard,
|
||||
}
|
||||
|
||||
@@ -196,6 +214,11 @@ def test_sharded_tail_tp8_matches_replicated_projection() -> None:
|
||||
_run_ranks("matches_replicated", 8)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=8)
|
||||
def test_sharded_tail_tp8_ep8_matches_replicated_projection() -> None:
|
||||
_run_ranks("matches_replicated_ep", 8)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=4)
|
||||
def test_sharded_tail_tp4_writes_only_its_own_shard() -> None:
|
||||
_run_ranks("own_shard_only", 4)
|
||||
@@ -241,12 +264,19 @@ def test_forward_shards_only_when_the_tail_is_valid(
|
||||
|
||||
@pytest.fixture
|
||||
def build_runner(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Construct through the real subclass ``__init__``, stubbing only the base."""
|
||||
"""Construct through the real subclass ``__init__``, stubbing only the base.
|
||||
|
||||
``tp_world`` is the size of the TP process group and is varied independently
|
||||
of ``moe_config.tp_size``, because under expert parallelism those two are
|
||||
different numbers for the same eight ranks.
|
||||
"""
|
||||
world = {"size": 8}
|
||||
|
||||
def _base_init(
|
||||
self,
|
||||
*,
|
||||
tp_size: int = 8,
|
||||
ep_size: int = 1,
|
||||
hidden: int = HIDDEN_SIZE,
|
||||
is_sequence_parallel: bool = False,
|
||||
has_up_proj: bool = True,
|
||||
@@ -254,7 +284,9 @@ def build_runner(monkeypatch: pytest.MonkeyPatch):
|
||||
routed_scaling_factor: float = 1.0,
|
||||
) -> None:
|
||||
self.moe_config = SimpleNamespace(
|
||||
tp_size=tp_size, is_sequence_parallel=is_sequence_parallel
|
||||
tp_size=tp_size,
|
||||
ep_size=ep_size,
|
||||
is_sequence_parallel=is_sequence_parallel,
|
||||
)
|
||||
self.routed_output_transform = SimpleNamespace(
|
||||
norm=None,
|
||||
@@ -266,20 +298,63 @@ def build_runner(monkeypatch: pytest.MonkeyPatch):
|
||||
self._shared_experts = object() if has_shared_experts else None
|
||||
|
||||
monkeypatch.setattr(moe_runner.MoERunner, "__init__", _base_init)
|
||||
return ROCmLatentMoERunner
|
||||
monkeypatch.setattr(
|
||||
latent_moe_runner,
|
||||
"get_tensor_model_parallel_world_size",
|
||||
lambda: world["size"],
|
||||
)
|
||||
|
||||
def _build(*, tp_world: int = 8, **kwargs) -> ROCmLatentMoERunner:
|
||||
world["size"] = tp_world
|
||||
return ROCmLatentMoERunner(**kwargs)
|
||||
|
||||
return _build
|
||||
|
||||
|
||||
def test_shards_under_the_kimi_k3_serving_config(build_runner) -> None:
|
||||
runner = build_runner()
|
||||
"""``--tensor-parallel-size 8``, MoE experts tensor-parallel."""
|
||||
runner = build_runner(tp_world=8, tp_size=8, ep_size=1)
|
||||
|
||||
assert runner._tail_shardable
|
||||
assert runner._up_proj_shard_size == HIDDEN_SIZE // 8
|
||||
|
||||
|
||||
def test_shards_under_the_kimi_k3_serving_config_with_expert_parallel(
|
||||
build_runner,
|
||||
) -> None:
|
||||
"""``--tensor-parallel-size 8 --enable-expert-parallel``, what is served.
|
||||
|
||||
EP reports ``moe_config.tp_size == 1`` because each rank owns whole experts,
|
||||
but the hidden dim is still split across all eight TP ranks, so the tail
|
||||
must shard exactly as it does without EP.
|
||||
"""
|
||||
runner = build_runner(tp_world=8, tp_size=1, ep_size=8)
|
||||
|
||||
assert runner._tail_shardable
|
||||
assert runner._up_proj_shard_size == HIDDEN_SIZE // 8
|
||||
|
||||
|
||||
def test_shards_by_the_process_group_not_the_dp_flattened_tp_size(
|
||||
build_runner,
|
||||
) -> None:
|
||||
"""Without EP, ``moe_config.tp_size`` is flattened across DP: TP=2 x DP=2
|
||||
reports 4 for a TP process group of 2.
|
||||
|
||||
Only those 2 ranks share a hidden dim and take part in the final
|
||||
all-reduce, so sharding by 4 would leave half of it with no routed
|
||||
contribution on either rank.
|
||||
"""
|
||||
runner = build_runner(tp_world=2, tp_size=4, ep_size=1)
|
||||
|
||||
assert runner._tail_shardable
|
||||
assert runner._up_proj_shard_size == HIDDEN_SIZE // 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"override",
|
||||
[
|
||||
pytest.param({"tp_size": 1}, id="no-tp"),
|
||||
pytest.param({"tp_world": 1, "tp_size": 1}, id="single-rank"),
|
||||
pytest.param({"tp_world": 1, "tp_size": 4}, id="dp-without-tp"),
|
||||
pytest.param({"hidden": HIDDEN_SIZE + 2}, id="hidden-not-divisible-by-tp"),
|
||||
pytest.param({"has_up_proj": False}, id="no-up-proj"),
|
||||
pytest.param({"has_shared_experts": False}, id="no-shared-partial"),
|
||||
@@ -290,7 +365,12 @@ def test_shards_under_the_kimi_k3_serving_config(build_runner) -> None:
|
||||
def test_falls_back_when_the_config_breaks_the_shard(
|
||||
build_runner, override: dict
|
||||
) -> None:
|
||||
"""Each of these makes the sharded tail wrong, not merely slower."""
|
||||
"""Each of these makes the sharded tail wrong, not merely slower.
|
||||
|
||||
``dp-without-tp`` is DP=4 with no TP: the hidden dim is not split at all,
|
||||
so there is nothing to shard and no all-reduce to stitch shards back
|
||||
together, however large ``moe_config.tp_size`` claims to be.
|
||||
"""
|
||||
runner = build_runner(**override)
|
||||
|
||||
assert not runner._tail_shardable
|
||||
|
||||
@@ -16,7 +16,9 @@ from vllm import LLM, SamplingParams
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.engine.protocol import StreamingInput
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.renderers.inputs.preprocess import parse_model_prompt
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
from vllm.v1.kv_cache_interface import SlidingWindowSpec
|
||||
@@ -222,6 +224,14 @@ async def test_voxtral_realtime_generator(audio_assets, tokenizer, async_engine)
|
||||
sampling_params = SamplingParams(temperature=0.0, max_tokens=1)
|
||||
audio_config = tokenizer.instruct_tokenizer.audio_encoder.audio_config
|
||||
|
||||
async def input_stream(audio_buffer):
|
||||
async for prompt in audio_buffer.get_input_stream():
|
||||
parsed_prompt = parse_model_prompt(async_engine.model_config, prompt)
|
||||
(engine_input,) = await async_engine.renderer.render_cmpl_async(
|
||||
[parsed_prompt]
|
||||
)
|
||||
yield StreamingInput(prompt=engine_input)
|
||||
|
||||
output_tokens_list = []
|
||||
for i, audio_asset in enumerate(audio_assets):
|
||||
output_tokens = []
|
||||
@@ -241,7 +251,7 @@ async def test_voxtral_realtime_generator(audio_assets, tokenizer, async_engine)
|
||||
request_id = f"session-{i}"
|
||||
|
||||
async for resp in async_engine.generate(
|
||||
prompt=buffer.get_input_stream(),
|
||||
prompt=input_stream(buffer),
|
||||
sampling_params=sampling_params,
|
||||
request_id=request_id,
|
||||
):
|
||||
|
||||
@@ -34,6 +34,7 @@ def vqa_ppl_test(
|
||||
mm_processor_kwargs = {}
|
||||
|
||||
processor = AutoProcessor.from_pretrained(model_info.name)
|
||||
image_token_id = processor.image_token_id
|
||||
|
||||
images = []
|
||||
prompts = []
|
||||
@@ -99,8 +100,10 @@ def vqa_ppl_test(
|
||||
for token_data in token_datas[1:]:
|
||||
assert token_data is not None
|
||||
assert len(token_data) == 1
|
||||
token_log_prob = list(token_data.values())[0].logprob
|
||||
token_log_probs.append(token_log_prob)
|
||||
token_id, token_log_prob = next(iter(token_data.items()))
|
||||
if token_id == image_token_id:
|
||||
continue
|
||||
token_log_probs.append(token_log_prob.logprob)
|
||||
|
||||
neg_log_likelihood = -torch.tensor(
|
||||
token_log_probs, dtype=torch.float32, device="cpu"
|
||||
@@ -132,11 +135,12 @@ def vqa_ppl_test(
|
||||
inputs = inputs.to("cuda")
|
||||
input_ids = inputs["input_ids"]
|
||||
|
||||
outputs = hf_model.model(**inputs, labels=input_ids)
|
||||
labels = input_ids.masked_fill(input_ids == image_token_id, -100)
|
||||
outputs = hf_model.model(**inputs, labels=labels)
|
||||
neg_log_likelihood = outputs.loss
|
||||
neg_log_likelihood = neg_log_likelihood.to(torch.float32).cpu()
|
||||
|
||||
num_loss_tokens = input_ids.shape[1] - 1
|
||||
num_loss_tokens = int((labels[:, 1:] != -100).sum())
|
||||
nll_sum += neg_log_likelihood * num_loss_tokens
|
||||
n_tokens += num_loss_tokens
|
||||
|
||||
|
||||
@@ -8,26 +8,51 @@ from tests.models.utils import GenerateModelInfo
|
||||
from .ppl_utils import vqa_ppl_test
|
||||
|
||||
MODELS = [
|
||||
GenerateModelInfo("Qwen/Qwen2-VL-2B-Instruct", hf_ppl=41081356.0),
|
||||
GenerateModelInfo("Qwen/Qwen2.5-VL-3B-Instruct", hf_ppl=18330016.0),
|
||||
(
|
||||
GenerateModelInfo("Qwen/Qwen2-VL-2B-Instruct", hf_ppl=9.810060501098633),
|
||||
28,
|
||||
{},
|
||||
),
|
||||
(
|
||||
GenerateModelInfo("Qwen/Qwen2.5-VL-3B-Instruct", hf_ppl=22.17719078063965),
|
||||
28,
|
||||
{},
|
||||
),
|
||||
(
|
||||
GenerateModelInfo("Qwen/Qwen3-VL-4B-Instruct", hf_ppl=11.916606903076172),
|
||||
32,
|
||||
{},
|
||||
),
|
||||
(
|
||||
GenerateModelInfo("Qwen/Qwen3.5-4B", hf_ppl=6.478141784667969),
|
||||
32,
|
||||
{"enable_prefix_caching": False},
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
mm_processor_kwargs = {
|
||||
"min_pixels": 28 * 28,
|
||||
"max_pixels": 1280 * 28 * 28,
|
||||
}
|
||||
def _pixel_factor(factor: int):
|
||||
"""`factor = patch_size * merge_size` for Qwen models"""
|
||||
return {"min_pixels": factor**2, "max_pixels": 1280 * factor**2}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_info", MODELS)
|
||||
@pytest.mark.parametrize("model_info,factor,extra_kwargs", MODELS)
|
||||
@pytest.mark.parametrize("mm_device_do_normalize", [True, False])
|
||||
def test_ppl(
|
||||
hf_runner, vllm_runner, model_info: GenerateModelInfo, mm_device_do_normalize: bool
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model_info: GenerateModelInfo,
|
||||
factor: int,
|
||||
extra_kwargs: dict,
|
||||
mm_device_do_normalize: bool,
|
||||
):
|
||||
vqa_ppl_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model_info,
|
||||
vllm_extra_kwargs={"mm_device_do_normalize": mm_device_do_normalize},
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
vllm_extra_kwargs={
|
||||
"mm_device_do_normalize": mm_device_do_normalize,
|
||||
**extra_kwargs,
|
||||
},
|
||||
mm_processor_kwargs=_pixel_factor(factor),
|
||||
)
|
||||
|
||||
@@ -202,19 +202,19 @@ def test_get_prompt_updates_respects_nested_max_soft_tokens(model_id: str):
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
|
||||
@pytest.mark.parametrize("kwargs_on_init", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"image_kwargs", [{"rescale_factor": 1 / 127.5}, {"max_soft_tokens": 560}]
|
||||
)
|
||||
@pytest.mark.parametrize("video_uuid", [None, "same-video"])
|
||||
def test_video_cache_is_independent_of_image_kwargs(
|
||||
model_id: str,
|
||||
kwargs_on_init: bool,
|
||||
image_kwargs: dict[str, object],
|
||||
video_uuid: str | None,
|
||||
):
|
||||
"""Image overrides must not change video frames or depend on cache warmth."""
|
||||
kwargs = {"images_kwargs": image_kwargs}
|
||||
"""Request image overrides must not change video frames or make results
|
||||
depend on cache warmth.
|
||||
"""
|
||||
request_kwargs = {"images_kwargs": image_kwargs}
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1, "video": 1},
|
||||
@@ -239,18 +239,11 @@ def test_video_cache_is_independent_of_image_kwargs(
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
baseline = process({}, cache=None)
|
||||
if kwargs_on_init:
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=kwargs,
|
||||
limit_mm_per_prompt={"image": 1, "video": 1},
|
||||
mm_processor_cache_gb=1,
|
||||
)
|
||||
cache = MultiModalProcessorOnlyCache(ctx.model_config)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
request_kwargs = {} if kwargs_on_init else kwargs
|
||||
process(request_kwargs, cache=cache)
|
||||
# Populate the video cache without image overrides, then change only
|
||||
# images_kwargs on the next request. Since images_kwargs are excluded from
|
||||
# the video hash, and must not affect the processed video, the cached result
|
||||
# must match a fresh recomputation.
|
||||
baseline = process({}, cache=cache)
|
||||
cached = process(request_kwargs, cache=cache)
|
||||
cache.clear_cache()
|
||||
fresh = process(request_kwargs, cache=cache)
|
||||
|
||||
@@ -100,6 +100,11 @@ def _expected_placeholder_tokens_per_image(
|
||||
[
|
||||
({}, (448, 448), 256),
|
||||
({"size": {"longest_edge": 1008}}, (1540, 1540), 1296),
|
||||
(
|
||||
{"images_kwargs": {"size": {"longest_edge": 1008}}},
|
||||
(1540, 1540),
|
||||
1296,
|
||||
),
|
||||
({"size": {"longest_edge": 1288}}, (1536, 1187), 1656),
|
||||
({"size": {"longest_edge": 1008}}, (29, 29), 4),
|
||||
({"size": {"longest_edge": 1000}}, (1540, 1700), 1188),
|
||||
@@ -148,6 +153,40 @@ def test_processor_size_override(
|
||||
assert prompt_update_tokens == hf_placeholder_tokens
|
||||
|
||||
|
||||
def test_scoped_request_size_overrides_configured_flat_size():
|
||||
ctx = build_model_context(
|
||||
_MODEL_ID,
|
||||
mm_processor_kwargs={"size": {"longest_edge": 1008}},
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
model_config_kwargs=_MODEL_CONFIG_KWARGS,
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
hf_processor = AutoProcessor.from_pretrained(
|
||||
_MODEL_ID,
|
||||
fix_mistral_regex=True,
|
||||
)
|
||||
|
||||
hf_processor_mm_kwargs: dict[str, object] = {
|
||||
"images_kwargs": {"size": {"longest_edge": 1288}}
|
||||
}
|
||||
image = Image.new("RGB", (1536, 1187), color=(127, 127, 127))
|
||||
|
||||
merged_mm_kwargs = processor.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
|
||||
pixel_values, hf_placeholder_tokens = _process_images_with_hf(
|
||||
hf_processor, [image], merged_mm_kwargs
|
||||
)
|
||||
prompt_update_tokens = _placeholder_tokens_from_prompt_updates(
|
||||
processor, [image], pixel_values, hf_processor_mm_kwargs
|
||||
)
|
||||
|
||||
expected_from_pixel_values = _expected_placeholder_tokens_per_image(
|
||||
hf_processor, pixel_values[0]
|
||||
)
|
||||
assert expected_from_pixel_values == 1656
|
||||
assert hf_placeholder_tokens.count(hf_processor.image_token_id) == 1656
|
||||
assert prompt_update_tokens == hf_placeholder_tokens
|
||||
|
||||
|
||||
def test_lightonocr_keeps_vision_config_image_size():
|
||||
ctx = build_model_context(
|
||||
_LIGHTON_MODEL_ID,
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from copy import deepcopy
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
@@ -8,16 +10,238 @@ import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
|
||||
from vllm.model_executor.models.qwen2_vl import Qwen2VLProcessingInfo
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
|
||||
from vllm.multimodal.inputs import batched_tensors_equal
|
||||
from vllm.multimodal.processing.context import InputProcessingContext
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mm_kwargs", "expected"),
|
||||
[
|
||||
(
|
||||
{"size": {"shortest_edge": 64, "longest_edge": 1024}},
|
||||
{
|
||||
"size": {"shortest_edge": 64, "longest_edge": 1024},
|
||||
"min_pixels": 64,
|
||||
"max_pixels": 1024,
|
||||
},
|
||||
),
|
||||
(
|
||||
{"min_pixels": 64, "max_pixels": 1024},
|
||||
{
|
||||
"min_pixels": 64,
|
||||
"max_pixels": 1024,
|
||||
"size": {"shortest_edge": 64, "longest_edge": 1024},
|
||||
},
|
||||
),
|
||||
(
|
||||
{
|
||||
"size": {"shortest_edge": 64, "longest_edge": 1024},
|
||||
"min_pixels": 128,
|
||||
"max_pixels": 2048,
|
||||
},
|
||||
{
|
||||
"size": {"shortest_edge": 64, "longest_edge": 1024},
|
||||
"min_pixels": 128,
|
||||
"max_pixels": 2048,
|
||||
},
|
||||
),
|
||||
(
|
||||
{
|
||||
"min_pixels": None,
|
||||
"size": {"shortest_edge": 0},
|
||||
"images_kwargs": {"max_pixels": 4096},
|
||||
"videos_kwargs": {"size": {"longest_edge": 8192}},
|
||||
},
|
||||
{
|
||||
"min_pixels": None,
|
||||
"size": {"shortest_edge": 0},
|
||||
"images_kwargs": {
|
||||
"max_pixels": 4096,
|
||||
"size": {"longest_edge": 4096},
|
||||
},
|
||||
"videos_kwargs": {
|
||||
"size": {"longest_edge": 8192},
|
||||
"max_pixels": 8192,
|
||||
},
|
||||
},
|
||||
),
|
||||
(
|
||||
{"min_pixels": 0},
|
||||
{"min_pixels": 0, "size": {"shortest_edge": 0}},
|
||||
),
|
||||
(
|
||||
{"size": None, "min_pixels": 64},
|
||||
{"size": {"shortest_edge": 64}, "min_pixels": 64},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_complete_mm_processor_size_aliases(
|
||||
mm_kwargs: dict[str, object],
|
||||
expected: dict[str, object],
|
||||
) -> None:
|
||||
original = deepcopy(mm_kwargs)
|
||||
|
||||
actual = Qwen2VLProcessingInfo._complete_mm_processor_size_aliases(mm_kwargs)
|
||||
|
||||
assert actual == expected
|
||||
assert mm_kwargs == original
|
||||
|
||||
|
||||
def test_merge_and_resolve_mm_processor_kwargs_preserves_request_alias_precedence(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
mm_config = MultiModalConfig(
|
||||
mm_processor_kwargs={
|
||||
"images_kwargs": {"min_pixels": 64},
|
||||
},
|
||||
mm_device_do_normalize=False,
|
||||
)
|
||||
model_config = SimpleNamespace(get_multimodal_config=lambda: mm_config)
|
||||
ctx = InputProcessingContext(model_config, tokenizer=None) # type: ignore[arg-type]
|
||||
info = Qwen2VLProcessingInfo(ctx)
|
||||
monkeypatch.setattr(
|
||||
info,
|
||||
"get_supported_mm_processor_kwargs",
|
||||
lambda: {"images_kwargs": {"size", "min_pixels"}},
|
||||
)
|
||||
|
||||
assert info._merge_and_resolve_mm_processor_kwargs(
|
||||
{"size": {"shortest_edge": 128}}
|
||||
) == {
|
||||
"images_kwargs": {
|
||||
"size": {"shortest_edge": 128},
|
||||
"min_pixels": 128,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mm_kwargs", "default_size", "expected"),
|
||||
[
|
||||
(
|
||||
{},
|
||||
{"shortest_edge": 32, "longest_edge": 1024},
|
||||
{"shortest_edge": 32, "longest_edge": 1024},
|
||||
),
|
||||
(
|
||||
{"size": {"shortest_edge": 64}},
|
||||
{"shortest_edge": 32, "longest_edge": 1024},
|
||||
{"shortest_edge": 64, "longest_edge": 1024},
|
||||
),
|
||||
(
|
||||
{
|
||||
"size": {"shortest_edge": 64, "longest_edge": 2048},
|
||||
"min_pixels": 128,
|
||||
"max_pixels": 4096,
|
||||
},
|
||||
{"shortest_edge": 32, "longest_edge": 1024},
|
||||
{"shortest_edge": 128, "longest_edge": 4096},
|
||||
),
|
||||
(
|
||||
{
|
||||
"size": {"shortest_edge": 64},
|
||||
"max_pixels": 4096,
|
||||
},
|
||||
None,
|
||||
{"shortest_edge": 64, "longest_edge": 4096},
|
||||
),
|
||||
(
|
||||
{
|
||||
"size": {"shortest_edge": 64, "longest_edge": 2048},
|
||||
"min_pixels": None,
|
||||
"max_pixels": 0,
|
||||
},
|
||||
{"shortest_edge": 32, "longest_edge": 1024},
|
||||
{"shortest_edge": 64, "longest_edge": 0},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_get_vision_size(
|
||||
mm_kwargs: dict[str, object],
|
||||
default_size: dict[str, object] | None,
|
||||
expected: dict[str, object],
|
||||
) -> None:
|
||||
original_mm_kwargs = {
|
||||
key: dict(value) if isinstance(value, dict) else value
|
||||
for key, value in mm_kwargs.items()
|
||||
}
|
||||
original_default_size = dict(default_size) if default_size is not None else None
|
||||
|
||||
actual = Qwen2VLProcessingInfo._get_vision_size(
|
||||
mm_kwargs,
|
||||
default_size=default_size,
|
||||
)
|
||||
|
||||
assert actual == expected
|
||||
assert mm_kwargs == original_mm_kwargs
|
||||
assert default_size == original_default_size
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("modality", "scope"),
|
||||
[
|
||||
("image", "images_kwargs"),
|
||||
("video", "videos_kwargs"),
|
||||
],
|
||||
)
|
||||
def test_get_vision_info_reads_merged_modality_scope(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
modality: str,
|
||||
scope: str,
|
||||
) -> None:
|
||||
vision_config = SimpleNamespace(
|
||||
patch_size=14,
|
||||
spatial_merge_size=2,
|
||||
temporal_patch_size=2,
|
||||
)
|
||||
ctx = SimpleNamespace(
|
||||
get_hf_config=lambda *_: SimpleNamespace(vision_config=vision_config)
|
||||
)
|
||||
info = Qwen2VLProcessingInfo(ctx)
|
||||
merged = {
|
||||
"images_kwargs": {"size": {"shortest_edge": 111}},
|
||||
"videos_kwargs": {"size": {"shortest_edge": 222}},
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
info,
|
||||
"_merge_and_resolve_mm_processor_kwargs",
|
||||
lambda _: merged,
|
||||
)
|
||||
|
||||
seen: list[dict[str, object]] = []
|
||||
|
||||
def get_size(
|
||||
mm_kwargs: dict[str, object],
|
||||
default_size: dict[str, object] | None = None,
|
||||
) -> dict[str, object]:
|
||||
seen.append(mm_kwargs)
|
||||
return dict(default_size or {})
|
||||
|
||||
monkeypatch.setattr(info, "_get_vision_size", get_size)
|
||||
image_processor = SimpleNamespace(size={"shortest_edge": 32, "longest_edge": 1024})
|
||||
|
||||
info._get_vision_info(
|
||||
image_width=28,
|
||||
image_height=28,
|
||||
num_frames=2,
|
||||
do_resize=False,
|
||||
image_processor=image_processor,
|
||||
mm_kwargs={},
|
||||
modality=modality, # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
assert seen == [merged[scope]]
|
||||
|
||||
|
||||
def test_jina_vl_processing_order() -> None:
|
||||
"""Jina's document-first prompt keeps cached features and hashes aligned."""
|
||||
ctx = build_model_context(
|
||||
|
||||
@@ -1,19 +1,21 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Regression tests for Qwen3-VL processor.
|
||||
|
||||
Covers the fix for num_frames-based timestamp calculation
|
||||
(issue vllm-project/vllm#35909).
|
||||
"""
|
||||
"""Regression tests for Qwen3-VL multimodal processing."""
|
||||
|
||||
from copy import deepcopy
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from transformers.models.qwen3_vl import Qwen3VLVideoProcessor
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.config.multimodal import MultiModalDummyOptions
|
||||
from vllm.model_executor.models.qwen3_vl import Qwen3VLProcessingInfo
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.processing.context import BaseProcessingInfo
|
||||
|
||||
from ...registry import HF_EXAMPLE_MODELS
|
||||
from ...utils import build_model_context
|
||||
@@ -21,6 +23,32 @@ from ...utils import build_model_context
|
||||
MODEL_ID = "Qwen/Qwen3-VL-4B-Instruct"
|
||||
|
||||
|
||||
def test_qwen3_video_routing_schema_adds_vllm_compatibility_kwargs(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
native_video_kwargs = set(Qwen3VLVideoProcessor.valid_kwargs.__annotations__)
|
||||
assert "size" in native_video_kwargs
|
||||
assert "min_pixels" not in native_video_kwargs
|
||||
assert "max_pixels" not in native_video_kwargs
|
||||
|
||||
info = Qwen3VLProcessingInfo(SimpleNamespace())
|
||||
parent_supported = {
|
||||
"text_kwargs": {"padding"},
|
||||
"images_kwargs": {"size", "min_pixels", "max_pixels"},
|
||||
"videos_kwargs": set(native_video_kwargs),
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
BaseProcessingInfo,
|
||||
"get_supported_mm_processor_kwargs",
|
||||
lambda self: {scope: set(kwargs) for scope, kwargs in parent_supported.items()},
|
||||
)
|
||||
|
||||
supported = info.supported_mm_processor_kwargs
|
||||
|
||||
assert {"min_pixels", "max_pixels"} <= supported["videos_kwargs"]
|
||||
assert parent_supported["videos_kwargs"] == native_video_kwargs
|
||||
|
||||
|
||||
def _build_video_mm_data(
|
||||
num_frames: int,
|
||||
width: int = 128,
|
||||
@@ -45,6 +73,122 @@ def _build_video_mm_data(
|
||||
return {"video": [(video, metadata)]}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
def test_processor_partial_scoped_image_size(model_id: str) -> None:
|
||||
"""Partial image ``size`` must be completed and applied before the HF call.
|
||||
|
||||
Keep video compatibility and per-video kwargs in an image-only call as well:
|
||||
Qwen3 must sanitize the video scope before HF validates it. Regression for
|
||||
#56363.
|
||||
"""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1, "video": 0},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
image = np.zeros((128, 128, 3), dtype=np.uint8)
|
||||
min_pixels = 256 * 32 * 32
|
||||
default_size = dict(processor.info.get_image_processor().size)
|
||||
complete_size = default_size | {"shortest_edge": min_pixels}
|
||||
|
||||
def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor:
|
||||
mm_items = processor.info.parse_mm_data({"image": [image]})
|
||||
return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
|
||||
"image_grid_thw"
|
||||
]
|
||||
|
||||
stock_grid = process({})
|
||||
complete_grid = process({"images_kwargs": {"size": complete_size}})
|
||||
partial_grid = process(
|
||||
{
|
||||
"images_kwargs": {
|
||||
"size": {"shortest_edge": min_pixels},
|
||||
},
|
||||
"videos_kwargs": {
|
||||
"min_pixels": 4096,
|
||||
"max_pixels": 469_762_048,
|
||||
"fps": [2.0],
|
||||
"num_frames": [8],
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
assert torch.equal(partial_grid, complete_grid)
|
||||
assert not torch.equal(partial_grid, stock_grid)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
def test_processor_partial_scoped_video_size(model_id: str) -> None:
|
||||
"""Partial scoped video ``size`` must be completed and applied before HF."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 0, "video": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
min_pixels = 512 * 32 * 32
|
||||
default_size = dict(processor.info.get_video_processor().size)
|
||||
complete_size = default_size | {"shortest_edge": min_pixels}
|
||||
|
||||
def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor:
|
||||
mm_items = processor.info.parse_mm_data(_build_video_mm_data(num_frames=8))
|
||||
return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
|
||||
"video_grid_thw"
|
||||
]
|
||||
|
||||
stock_grid = process({})
|
||||
complete_grid = process({"videos_kwargs": {"size": complete_size}})
|
||||
partial_grid = process(
|
||||
{
|
||||
"videos_kwargs": {
|
||||
"size": {"shortest_edge": min_pixels},
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
assert torch.equal(partial_grid, complete_grid)
|
||||
assert not torch.equal(partial_grid, stock_grid)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
def test_processor_flat_video_pixel_aliases_match_scoped_size(model_id: str) -> None:
|
||||
"""Flat min/max pixel aliases keep their historical Qwen3 video effect."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 0, "video": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
pixel_budget = 512 * 32 * 32
|
||||
|
||||
def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor:
|
||||
mm_items = processor.info.parse_mm_data(_build_video_mm_data(num_frames=8))
|
||||
return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
|
||||
"video_grid_thw"
|
||||
]
|
||||
|
||||
stock_grid = process({})
|
||||
scoped_grid = process(
|
||||
{
|
||||
"videos_kwargs": {
|
||||
"size": {
|
||||
"shortest_edge": pixel_budget,
|
||||
"longest_edge": pixel_budget,
|
||||
},
|
||||
},
|
||||
}
|
||||
)
|
||||
flat_grid = process(
|
||||
{
|
||||
"min_pixels": pixel_budget,
|
||||
"max_pixels": pixel_budget,
|
||||
}
|
||||
)
|
||||
|
||||
assert torch.equal(flat_grid, scoped_grid)
|
||||
assert not torch.equal(flat_grid, stock_grid)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
@pytest.mark.parametrize(
|
||||
"num_frames",
|
||||
@@ -189,20 +333,21 @@ def test_processor_kwargs_videos_kwargs_does_not_leak_into_image_budget(
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
@pytest.mark.parametrize(
|
||||
"hf_mm_kwargs",
|
||||
[{"num_frames": [8, 16]}, {"fps": [2.0, 4.0]}],
|
||||
("hf_mm_kwargs", "scalar_kwargs"),
|
||||
[
|
||||
({"num_frames": [8, 16]}, [{"num_frames": 8}, {"num_frames": 16}]),
|
||||
({"fps": [4.0, 12.0]}, [{"fps": 4.0}, {"fps": 12.0}]),
|
||||
],
|
||||
)
|
||||
def test_processor_multi_video_list_kwargs(
|
||||
model_id: str,
|
||||
hf_mm_kwargs: dict[str, Any],
|
||||
scalar_kwargs: list[dict[str, Any]],
|
||||
) -> None:
|
||||
"""Regression test: a multi-video request with list-valued per-video
|
||||
``mm_processor_kwargs`` (one ``fps``/``num_frames`` per video) must not
|
||||
crash.
|
||||
"""List-valued video kwargs must be scalarized per item without mutation.
|
||||
|
||||
Before the fix, ``_apply_hf_processor_main`` copied the whole kwargs to every
|
||||
video without slicing, so ``_get_video_second_idx`` received the list
|
||||
where a scalar was expected and raised ``TypeError``.
|
||||
Each video in the multi-video request must match processing the same input
|
||||
independently with its corresponding scalar ``fps``/``num_frames`` value.
|
||||
"""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
@@ -216,17 +361,36 @@ def test_processor_multi_video_list_kwargs(
|
||||
)
|
||||
mm_data = {
|
||||
"video": [
|
||||
_build_video_mm_data(num_frames=16)["video"][0],
|
||||
_build_video_mm_data(num_frames=32)["video"][0],
|
||||
_build_video_mm_data(num_frames=32)["video"][0],
|
||||
]
|
||||
}
|
||||
|
||||
hf_mm_kwargs_before = deepcopy(hf_mm_kwargs)
|
||||
processed = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs=hf_mm_kwargs,
|
||||
)
|
||||
|
||||
def process_single(hf_kwargs: dict[str, Any]) -> torch.Tensor:
|
||||
single_prompt = "<|vision_start|><|video_pad|><|vision_end|>"
|
||||
single_mm_data = _build_video_mm_data(num_frames=32)
|
||||
single = processor(
|
||||
single_prompt,
|
||||
mm_items=processor.info.parse_mm_data(single_mm_data),
|
||||
hf_processor_mm_kwargs=hf_kwargs,
|
||||
)
|
||||
return single["mm_kwargs"].get_data()["video_grid_thw"][0]
|
||||
|
||||
expected_grids = [process_single(kwargs) for kwargs in scalar_kwargs]
|
||||
multi_grids = processed["mm_kwargs"].get_data()["video_grid_thw"]
|
||||
|
||||
assert not torch.equal(expected_grids[0], expected_grids[1])
|
||||
assert torch.equal(multi_grids[0], expected_grids[0])
|
||||
assert torch.equal(multi_grids[1], expected_grids[1])
|
||||
assert hf_mm_kwargs == hf_mm_kwargs_before
|
||||
|
||||
video_phs = processed["mm_placeholders"].get("video", [])
|
||||
assert len(video_phs) == 2, (
|
||||
f"Expected exactly 2 video placeholders, got {len(video_phs)}"
|
||||
@@ -336,7 +500,12 @@ def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> Non
|
||||
|
||||
capped_ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs={"size": size, "cap_pixels_per_frame": True},
|
||||
mm_processor_kwargs={
|
||||
"videos_kwargs": {
|
||||
"size": size,
|
||||
"cap_pixels_per_frame": True,
|
||||
},
|
||||
},
|
||||
limit_mm_per_prompt={"image": 0, "video": 1},
|
||||
)
|
||||
capped = MULTIMODAL_REGISTRY.create_processor(capped_ctx.model_config)
|
||||
@@ -350,7 +519,7 @@ def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> Non
|
||||
|
||||
uncapped_ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs={"size": size},
|
||||
mm_processor_kwargs={"videos_kwargs": {"size": size}},
|
||||
limit_mm_per_prompt={"image": 0, "video": 1},
|
||||
)
|
||||
uncapped = MULTIMODAL_REGISTRY.create_processor(uncapped_ctx.model_config)
|
||||
|
||||
@@ -4,11 +4,13 @@
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.qwen4_exp.nvidia.ops.cute_dsl.hc_down_silu import hc_down_silu
|
||||
from vllm.models.qwen4_exp.nvidia.ops.hc import (
|
||||
grouped_gemma_rmsnorm,
|
||||
hc_combine,
|
||||
hc_combine_norm,
|
||||
hc_gate_mix,
|
||||
hc_silu,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
@@ -22,6 +24,13 @@ HC = 4
|
||||
HIDDEN_SIZE = 2560
|
||||
HYPER_HIDDEN_SIZE = HC * HIDDEN_SIZE
|
||||
EPS = 1e-6
|
||||
LORA_RANK = 320
|
||||
DOWN_N = LORA_RANK + HC + 12 # merged down+inject weight, 16-row padded
|
||||
|
||||
requires_sm90 = pytest.mark.skipif(
|
||||
not current_platform.has_device_capability(90),
|
||||
reason="fused HC down+SiLU requires SM90+",
|
||||
)
|
||||
|
||||
|
||||
def test_grouped_gemma_rmsnorm() -> None:
|
||||
@@ -125,3 +134,24 @@ def test_hc_combine_norm_unit_injection(num_tokens: int) -> None:
|
||||
expected_norm = grouped_gemma_rmsnorm(expected, weight, EPS, HC)
|
||||
assert torch.equal(actual, expected)
|
||||
torch.testing.assert_close(actual_norm, expected_norm)
|
||||
|
||||
|
||||
@requires_sm90
|
||||
@pytest.mark.parametrize("num_tokens", [1, 3, 5, 17, 48])
|
||||
def test_hc_down_silu_fused(num_tokens: int) -> None:
|
||||
# Compare computed columns with the unfused ll_bf16 + hc_silu reference.
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import ll_bf16_gemm
|
||||
|
||||
torch.manual_seed(0)
|
||||
x = torch.randn(num_tokens, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn(DOWN_N, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
lora, injection = hc_down_silu(x, weight, LORA_RANK, HC)
|
||||
|
||||
down = ll_bf16_gemm(x, weight).to(torch.bfloat16)
|
||||
torch.testing.assert_close(
|
||||
lora, hc_silu(down[:, :LORA_RANK], HC), rtol=0.01, atol=0.01
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
injection, down[:, LORA_RANK : LORA_RANK + HC], rtol=0.01, atol=0.01
|
||||
)
|
||||
|
||||
@@ -20,6 +20,9 @@ import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module
|
||||
import vllm.model_executor.parameter as parameter_module
|
||||
import vllm.models.qwen4_exp.common.ngram_embedding as ngram_embedding_module
|
||||
import vllm.models.qwen4_exp.nvidia.ngram_embedding as nvidia_ngram_embedding_module
|
||||
from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata
|
||||
from vllm.config import SpeculativeConfig, VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.config.quantization import QuantizationConfigArgs
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
@@ -50,8 +53,125 @@ from vllm.models.qwen4_exp.nvidia.ple_layer import Qwen4ExpPLELayer
|
||||
from vllm.utils.torch_utils import weak_ref_tensor
|
||||
from vllm.v1.attention.backends.short_conv_attn import (
|
||||
PleShortConvAttentionMetadata,
|
||||
PleShortConvAttentionMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device", ["cpu", "cuda"])
|
||||
@pytest.mark.parametrize("full_graph", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"query_lens,drafts,spec_rows,non_spec_rows,token_order",
|
||||
[
|
||||
pytest.param([4, 2, 0], [3, 1, -1], [0, 1], [], None, id="spec-padding"),
|
||||
pytest.param(
|
||||
[3, 1, 2, 5, 0],
|
||||
[2, -1, 1, -1, -1],
|
||||
[0, 2],
|
||||
[1, 3],
|
||||
[0, 1, 2, 4, 5, 3, 6, 7, 8, 9, 10],
|
||||
id="mixed-padding",
|
||||
),
|
||||
pytest.param([1, 5], [-1, -1], [], [0, 1], None, id="non-spec"),
|
||||
],
|
||||
)
|
||||
def test_ple_metadata_preserves_request_and_token_order(
|
||||
device, full_graph, query_lens, drafts, spec_rows, non_spec_rows, token_order
|
||||
):
|
||||
"""Preserve state routing, acceptance and prefill offsets without GPU sync."""
|
||||
if device == "cuda" and not torch.cuda.is_available():
|
||||
pytest.skip("CUDA required")
|
||||
device = torch.device(device)
|
||||
config = VllmConfig()
|
||||
config.speculative_config = SpeculativeConfig(
|
||||
method="ngram", num_speculative_tokens=3
|
||||
)
|
||||
config.compilation_config.cudagraph_mode = (
|
||||
CUDAGraphMode.FULL if full_graph else CUDAGraphMode.NONE
|
||||
)
|
||||
config.compilation_config.max_cudagraph_capture_size = 32
|
||||
builder = PleShortConvAttentionMetadataBuilder(
|
||||
MambaSpec(block_size=16, shapes=((16, 64),), dtypes=(torch.float32,)),
|
||||
["ple"],
|
||||
config,
|
||||
device,
|
||||
)
|
||||
common = create_common_attn_metadata(
|
||||
BatchSpec(seq_lens=[q + 16 for q in query_lens], query_lens=query_lens),
|
||||
16,
|
||||
device,
|
||||
arange_block_indices=True,
|
||||
)
|
||||
common.is_prefilling = torch.tensor(
|
||||
[q > 1 and d < 0 for q, d in zip(query_lens, drafts)]
|
||||
)
|
||||
# Token padding is independent of the sum of per-request query lengths.
|
||||
common.num_actual_tokens += 3
|
||||
accepted = torch.arange(1, len(query_lens) + 1, dtype=torch.int32, device=device)
|
||||
drafts_cpu = torch.tensor(drafts, dtype=torch.int32)
|
||||
sync_mode = torch.cuda.get_sync_debug_mode() if device.type == "cuda" else None
|
||||
if sync_mode is not None:
|
||||
torch.cuda.set_sync_debug_mode("error")
|
||||
try:
|
||||
metadata = builder.build(
|
||||
0,
|
||||
common,
|
||||
num_accepted_tokens=accepted,
|
||||
num_decode_draft_tokens_cpu=drafts_cpu,
|
||||
)
|
||||
finally:
|
||||
if sync_mode is not None:
|
||||
torch.cuda.set_sync_debug_mode(sync_mode)
|
||||
|
||||
def check(actual, expected):
|
||||
torch.testing.assert_close(
|
||||
actual, torch.tensor(expected, dtype=actual.dtype, device=device)
|
||||
)
|
||||
|
||||
assert metadata.num_spec_decodes == len(spec_rows)
|
||||
assert metadata.num_decodes == sum(query_lens[i] == 1 for i in non_spec_rows)
|
||||
assert metadata.num_prefills == sum(query_lens[i] > 1 for i in non_spec_rows)
|
||||
if spec_rows:
|
||||
torch.testing.assert_close(
|
||||
metadata.spec_state_indices_tensor[: len(spec_rows)],
|
||||
common.block_table_tensor[spec_rows, 0],
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
metadata.num_accepted_tokens[: len(spec_rows)], accepted[spec_rows]
|
||||
)
|
||||
check(
|
||||
metadata.spec_query_start_loc[: len(spec_rows) + 1],
|
||||
[0, *accumulate(query_lens[i] for i in spec_rows)],
|
||||
)
|
||||
if full_graph and not non_spec_rows:
|
||||
check(metadata.spec_state_indices_tensor[len(spec_rows) :], [NULL_BLOCK_ID])
|
||||
check(metadata.num_accepted_tokens[len(spec_rows) :], [1])
|
||||
check(
|
||||
metadata.spec_query_start_loc[len(spec_rows) + 1 :], [sum(query_lens)]
|
||||
)
|
||||
assert (
|
||||
metadata.spec_state_indices_tensor.data_ptr()
|
||||
== builder.spec_state_indices_tensor.data_ptr()
|
||||
)
|
||||
if non_spec_rows:
|
||||
torch.testing.assert_close(
|
||||
metadata.state_indices_tensor, common.block_table_tensor[non_spec_rows, 0]
|
||||
)
|
||||
prefill_lens = [query_lens[i] for i in non_spec_rows if query_lens[i] > 1]
|
||||
check(metadata.query_start_loc_p, [0, *accumulate(prefill_lens)])
|
||||
check(metadata.has_initial_states_p, [True] * len(prefill_lens))
|
||||
if token_order is not None:
|
||||
check(
|
||||
torch.cat((metadata.spec_token_indx, metadata.non_spec_token_indx)),
|
||||
token_order,
|
||||
)
|
||||
else:
|
||||
assert metadata.spec_token_indx is None
|
||||
assert metadata.non_spec_token_indx is None
|
||||
assert metadata.nums_dict is None
|
||||
assert metadata.batch_ptr is None
|
||||
assert metadata.token_chunk_offset_ptr is None
|
||||
|
||||
|
||||
def _mock_etp_group(
|
||||
@@ -1892,6 +2012,11 @@ def _make_conv_metadata(
|
||||
else None
|
||||
),
|
||||
non_spec_query_start_loc=(non_spec_query_start_loc if has_non_spec else None),
|
||||
query_start_loc_p=(
|
||||
non_spec_query_start_loc[case.num_decodes :] - case.num_decodes
|
||||
if num_prefills
|
||||
else None
|
||||
),
|
||||
has_initial_states_p=(
|
||||
torch.arange(num_prefills, device=device) % 2 == 0 if num_prefills else None
|
||||
),
|
||||
|
||||
@@ -17,6 +17,7 @@ from vllm.models.qwen4_exp.nvidia.ops import qsa as qsa_ops
|
||||
from vllm.models.qwen4_exp.nvidia.ops import qsa_indexer as qsa_indexer_ops
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
from vllm.v1.worker.utils import clear_layer_kv_caches
|
||||
|
||||
requires_qsa_kernels = pytest.mark.skipif(
|
||||
not current_platform.is_cuda() or not HAS_TRITON,
|
||||
@@ -365,7 +366,9 @@ def test_qsa_circular_buffer_survives_one_speculative_step(chunk_start: int) ->
|
||||
assert set(slots.tolist()).isdisjoint((committed % capacity).tolist())
|
||||
|
||||
|
||||
def _qsa_key_cache(block_size: int, compress_ratio: int) -> qsa_cache.QSAKeyStateCache:
|
||||
def _qsa_key_cache(
|
||||
block_size: int, compress_ratio: int, **kwargs
|
||||
) -> qsa_cache.QSAKeyStateCache:
|
||||
return qsa_cache.QSAKeyStateCache(
|
||||
head_size=64,
|
||||
dtype=torch.bfloat16,
|
||||
@@ -375,6 +378,7 @@ def _qsa_key_cache(block_size: int, compress_ratio: int) -> qsa_cache.QSAKeyStat
|
||||
compilation_config=SimpleNamespace(static_forward_context={})
|
||||
),
|
||||
compress_ratio=compress_ratio,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
@@ -402,6 +406,23 @@ def test_qsa_state_caches_adapt_the_unified_logical_layout() -> None:
|
||||
assert compressed_cache.kv_cache.data_ptr() == compressed_view.data_ptr()
|
||||
|
||||
|
||||
def test_clearing_qsa_key_cache_releases_its_storage() -> None:
|
||||
"""Profiling teardown clears `kv_cache`; no derived view may keep it alive."""
|
||||
raw_cache = _qsa_key_cache(
|
||||
block_size=32, compress_ratio=4, cache_rope_positions=True
|
||||
)
|
||||
kv = torch.zeros(2, 1, 8, raw_cache.head_size, dtype=torch.bfloat16)
|
||||
references_before_bind = torch._C._storage_Use_Count(kv.untyped_storage()._cdata)
|
||||
raw_cache.bind_kv_cache(kv)
|
||||
assert raw_cache.key_cache.shape[-1] == 64
|
||||
assert raw_cache.rope_position_cache.dtype == torch.int64
|
||||
|
||||
clear_layer_kv_caches([raw_cache])
|
||||
|
||||
references = torch._C._storage_Use_Count(kv.untyped_storage()._cdata)
|
||||
assert references == references_before_bind
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("compress_ratio", "num_spec", "expected"),
|
||||
[(4, 0, 4), (4, 1, 8), (4, 3, 8), (4, 4, 8), (4, 5, 12), (2, 3, 6)],
|
||||
|
||||
Reference in New Issue
Block a user