[Mini-vLLM] SM86-only cut: models, backends, Rust, tests

- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries
- Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels
- Remove inc/quark quant loaders; keep compressed-tensors and Marlin
- Drop the Rust frontend, pooling entrypoints, and idle test suites
- Rebuild kernels for sm_80;sm_86 and strip the extensions
- Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-18 19:19:56 +07:00
co-authored by Hinata
parent c5ab42bc0f
commit e61dcb9143
2728 changed files with 159 additions and 950401 deletions
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -1,337 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
import torch
from safetensors.torch import save_file
from transformers import DeepseekV2Config, DeepseekV2ForCausalLM
from vllm.config import (
AttentionConfig,
DeviceConfig,
KernelConfig,
LoadConfig,
ModelConfig,
VllmConfig,
)
from vllm.model_executor.layers.fused_moe.expert_substitution import (
ConstantExpertSubstitution,
)
from vllm.model_executor.model_loader import get_model_loader
from vllm.model_executor.models.expert_substitution import as_expert_substitution_model
from vllm.model_executor.models.utils import AutoWeightsLoader, PPMissingLayer
from vllm.platforms import current_platform
from vllm.v1.attention.backends.registry import AttentionBackendEnum
MODEL_CONFIG = SimpleNamespace(
hf_config=SimpleNamespace(approximate_experts={"0": [1, 2], "1": [1, 2]})
)
def _compression_config(value_names: dict[int, dict[int, str]]) -> dict:
"""Explicitly named constants given as ``{layer: {expert: tensor name}}``."""
targets = {
f"model.layers.{layer}.mlp.experts": {
"num_logical_experts": 3,
"weight_layout": "compact_retained_experts",
"replacements": {
str(expert): {"format": "constant-v1", "tensors": {"value": name}}
for expert, name in names.items()
},
}
for layer, names in value_names.items()
}
router_semantics = {
"preserve_logical_expert_ids": True,
"preserve_router_weights": True,
"renormalize_after_substitution": False,
}
return {
"transform_config": {
"expert_substitution": {
"version": 1,
"router_semantics": router_semantics,
"targets": targets,
}
}
}
# Expert 1 of both layers shares one explicitly named constant.
EXPLICIT_MODEL_CONFIG = SimpleNamespace(
hf_config=SimpleNamespace(
compression_config=_compression_config(
{
layer: {1: "constants.shared", 2: f"constants.layer_{layer}"}
for layer in (0, 1)
}
)
)
)
def _approx_value(layer: int, expert: int) -> str:
return f"model.layers.{layer}.mlp.experts.{expert}.approx_value"
def _cpu_model(local_layers=(0, 1), tracked=True, model_config=MODEL_CONFIG):
"""Two decoder layers; the others are placeholders as on another PP stage."""
class Model(torch.nn.Module):
def __init__(self, prefix=""):
super().__init__()
self.weight = torch.nn.Parameter(torch.zeros(2), requires_grad=False)
self.model = torch.nn.Module()
self.model.layers = torch.nn.ModuleList()
for index in range(2):
if index not in local_layers:
self.model.layers.append(PPMissingLayer())
continue
layer = torch.nn.Module()
layer.mlp = torch.nn.Module()
layer.mlp.experts = ConstantExpertSubstitution(
index, 3, (1, 2), 2, torch.float32
)
self.model.layers.append(layer)
def load_weights(self, weights):
loaded = AutoWeightsLoader(self).load_weights(weights)
return loaded if tracked else None
return as_expert_substitution_model(Model, model_config)()
@pytest.mark.parametrize("local_layers", [(0, 1), (0,), (1,), ()])
def test_loads_local_approx_values_and_passes_through_others(local_layers):
"""Other PP stages and MTP drafts own none or some of the configured layers."""
model = _cpu_model(local_layers)
loaded = model.load_weights(
[
("weight", torch.tensor([1.0, 2.0])),
*(
(_approx_value(layer, expert), torch.full((2,), 10.0 * layer + expert))
for layer in range(2)
for expert in (1, 2)
),
]
)
model.process_weights_after_loading()
assert loaded == {"weight"} | {
f"model.layers.{layer}.mlp.experts.values" for layer in local_layers
}
for layer in local_layers:
torch.testing.assert_close(
model.model.layers[layer].mlp.experts.values,
torch.tensor([[1.0, 1.0], [2.0, 2.0]]) + 10.0 * layer,
)
@pytest.mark.parametrize("local_layers", [(0, 1), (1,), ()])
def test_loads_explicitly_named_values(local_layers):
"""Named values are consumed even when their layer is built elsewhere."""
model = _cpu_model(local_layers, model_config=EXPLICIT_MODEL_CONFIG)
loaded = model.load_weights(
[
("weight", torch.tensor([1.0, 2.0])),
("constants.shared", torch.full((2,), 1.0)),
("constants.layer_0", torch.full((2,), 2.0)),
("constants.layer_1", torch.full((2,), 3.0)),
]
)
model.process_weights_after_loading()
assert loaded == {"weight"} | {
f"model.layers.{layer}.mlp.experts.values" for layer in local_layers
}
for layer in local_layers:
torch.testing.assert_close(
model.model.layers[layer].mlp.experts.values,
torch.tensor([[1.0, 1.0], [2.0 + layer, 2.0 + layer]]),
)
def test_direct_reload_updates_values_incrementally():
model = _cpu_model(local_layers=(0,), tracked=False)
values = model.model.layers[0].mlp.experts.values
model.load_weights([(_approx_value(0, 2), torch.ones(2))])
with pytest.raises(ValueError, match="missing approx_value .* \\[1\\]"):
model.process_weights_after_loading()
assert model.load_weights([(_approx_value(0, 1), torch.zeros(2))]) is None
model.process_weights_after_loading()
model.load_weights([(_approx_value(0, 2), torch.full((2,), 3.0))])
torch.testing.assert_close(values, torch.tensor([[0.0, 0.0], [3.0, 3.0]]))
with pytest.raises(ValueError, match="not listed"):
model.load_weights([(_approx_value(0, 0), torch.ones(2))])
def test_layerwise_reload_is_rejected_and_keeps_weights():
from vllm.model_executor.model_loader.reload import (
initialize_layerwise_reload,
record_metadata_for_reloading,
)
model = _cpu_model(local_layers=(0,))
record_metadata_for_reloading(model)
loaded = [
("weight", torch.tensor([3.0, 4.0])),
(_approx_value(0, 1), torch.zeros(2)),
(_approx_value(0, 2), torch.ones(2)),
]
model.load_weights(loaded)
model.process_weights_after_loading()
substitution = model.model.layers[0].mlp.experts
values = substitution.values
initialize_layerwise_reload(model)
with pytest.raises(NotImplementedError, match="layerwise weight reload"):
model.load_weights([(name, weight + 1) for name, weight in loaded])
assert substitution.values is values
torch.testing.assert_close(values, torch.tensor([[0.0, 0.0], [1.0, 1.0]]))
torch.testing.assert_close(model.weight, torch.tensor([3.0, 4.0]))
assert substitution.expert_substitution_routes.tolist() == [0, -1, -2]
@pytest.mark.parametrize("missing_value", [False, True])
def test_tensorizer_finalizes_local_substitution_values(monkeypatch, missing_value):
"""Raw Tensorizer loading must reject an unloaded local constant row."""
from vllm.model_executor.model_loader import tensorizer_loader
model = _cpu_model(local_layers=(0,))
weights = [(_approx_value(0, 1), torch.zeros(2))]
if not missing_value:
weights.append((_approx_value(0, 2), torch.ones(2)))
finalize = Mock(wraps=model.process_weights_after_loading)
monkeypatch.setattr(model, "process_weights_after_loading", finalize)
monkeypatch.setattr(tensorizer_loader, "initialize_model", lambda **kwargs: model)
loader = object.__new__(tensorizer_loader.TensorizerLoader)
monkeypatch.setattr(loader, "_get_weights_iterator", lambda: iter(weights))
config = SimpleNamespace(
model_config=SimpleNamespace(dtype=torch.float32),
device_config=SimpleNamespace(device="cpu"),
)
if missing_value:
with pytest.raises(ValueError, match=r"missing approx_value .* \[2\]"):
loader._load_model_serialized_cpu(config)
else:
assert loader._load_model_serialized_cpu(config) is model
torch.testing.assert_close(
model.model.layers[0].mlp.experts.values,
torch.tensor([[0.0, 0.0], [1.0, 1.0]]),
)
finalize.assert_called_once_with()
def _write_tiny_deepseek_mone_checkpoint(
model_dir: Path, explicit_names: bool
) -> tuple[dict[str, torch.Tensor], str]:
config = DeepseekV2Config(
architectures=["DeepseekV2ForCausalLM"],
vocab_size=32,
hidden_size=16,
intermediate_size=32,
moe_intermediate_size=8,
num_hidden_layers=1,
num_attention_heads=2,
num_key_value_heads=2,
max_position_embeddings=32,
first_k_dense_replace=0,
n_routed_experts=3,
n_shared_experts=1,
num_experts_per_tok=1,
n_group=1,
topk_group=1,
norm_topk_prob=False,
q_lora_rank=None,
kv_lora_rank=4,
qk_nope_head_dim=128,
qk_rope_head_dim=64,
v_head_dim=128,
tie_word_embeddings=False,
dtype="float16",
)
experts = "model.layers.0.mlp.experts"
if explicit_names:
value_name = "model.layers.0.mlp.expert_replacements.1.value"
config.compression_config = _compression_config({0: {1: value_name}})
else:
value_name = f"{experts}.1.approx_value"
config.approximate_experts = {"0": [1]}
hf_weights = DeepseekV2ForCausalLM(config).to(torch.float16).state_dict()
gate_up = hf_weights.pop(f"{experts}.gate_up_proj")
down = hf_weights.pop(f"{experts}.down_proj")
weights = {name: value.contiguous() for name, value in hf_weights.items()}
for expert in (0, 2):
gate, up = gate_up[expert].chunk(2, dim=0)
weights[f"{experts}.{expert}.gate_proj.weight"] = gate.contiguous()
weights[f"{experts}.{expert}.up_proj.weight"] = up.contiguous()
weights[f"{experts}.{expert}.down_proj.weight"] = down[expert].contiguous()
weights[value_name] = torch.arange(16, dtype=torch.float16)
config.save_pretrained(model_dir)
save_file(weights, model_dir / "model.safetensors")
return weights, value_name
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like device"
)
@pytest.mark.usefixtures("dist_init", "workspace_init")
@pytest.mark.parametrize(
"explicit_names", [False, True], ids=["approximate_experts", "expert_substitution"]
)
def test_loads_mone_checkpoint_into_compact_experts(
tmp_path: Path, explicit_names: bool
):
weights, value_name = _write_tiny_deepseek_mone_checkpoint(tmp_path, explicit_names)
model_config = ModelConfig(
model=str(tmp_path),
tokenizer=str(tmp_path),
skip_tokenizer_init=True,
dtype="float16",
max_model_len=32,
enforce_eager=True,
)
load_config = LoadConfig(load_format="safetensors", use_tqdm_on_load=False)
vllm_config = VllmConfig(
model_config=model_config,
device_config=DeviceConfig(device="cuda"),
load_config=load_config,
attention_config=AttentionConfig(backend=AttentionBackendEnum.TRITON_MLA),
kernel_config=KernelConfig(moe_backend="triton"),
)
model = get_model_loader(load_config).load_model(vllm_config, model_config)
experts = "model.layers.0.mlp.experts"
routed_experts = model.model.layers[0].mlp.experts.routed_experts
# Logical expert 2 is stored in physical row 1.
torch.testing.assert_close(
routed_experts.w13_weight[1].cpu(),
torch.cat(
[
weights[f"{experts}.2.gate_proj.weight"],
weights[f"{experts}.2.up_proj.weight"],
]
),
)
torch.testing.assert_close(
routed_experts.w2_weight[1].cpu(), weights[f"{experts}.2.down_proj.weight"]
)
substitution = routed_experts.expert_substitution
torch.testing.assert_close(
substitution.values.cpu(), weights[value_name].unsqueeze(0)
)
updated = weights[value_name] + 1
assert model.load_weights([(value_name, updated)]) == {
f"{experts}.routed_experts.expert_substitution.values"
}
torch.testing.assert_close(substitution.values.cpu(), updated.unsqueeze(0))
@@ -1,152 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from vllm.model_executor.models.gpt_oss import (
GptOssForCausalLM,
GptOssModel,
GptOssRoutedExperts,
)
class _TestExperts(GptOssRoutedExperts):
def __init__(
self,
param_name: str,
shape: tuple[int, ...],
*,
tp_rank: int = 0,
quant_method_name: str = "TestQuantMethod",
weight_dtype: str = "",
dtype: torch.dtype = torch.float32,
) -> None:
torch.nn.Module.__init__(self)
self.moe_config = SimpleNamespace(
moe_parallel_config=SimpleNamespace(tp_size=2, tp_rank=tp_rank)
)
self.quant_method = type(
quant_method_name, (), {"weight_dtype": weight_dtype}
)()
param = torch.nn.Parameter(torch.zeros(shape, dtype=dtype), requires_grad=False)
param.weight_loader = self.weight_loader
self.register_parameter(param_name, param)
@staticmethod
def _map_global_expert_id_to_local_expert_id(expert_id: int) -> int:
return {3: 0, 7: 1}.get(expert_id, -1)
def _load(
experts: _TestExperts,
param_name: str,
loaded_weight: torch.Tensor,
shard_id: str,
expert_id: int = 3,
) -> bool:
param = getattr(experts, param_name)
return param.weight_loader(
param,
loaded_weight,
weight_name=f"model.layers.0.mlp.experts.routed_experts.{param_name}",
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
)
@pytest.mark.parametrize("base_layer_prefix", ["", "base_layer."])
def test_streamed_expert_resolves_wrapped_parameter(base_layer_prefix: str) -> None:
experts = _TestExperts("w13_weight", (2, 4, 3))
checkpoint_name = "model.layers.2.mlp.experts.7.gate_up_proj"
mapped_name = GptOssForCausalLM.hf_to_vllm_mapper.apply_list([checkpoint_name])[0]
fused_name = (
f"model.layers.2.mlp.experts.{base_layer_prefix}routed_experts.w13_weight"
)
params_dict = {fused_name: experts.w13_weight}
loaded_params: set[str] = set()
loaded_weight = torch.arange(24, dtype=torch.float32).reshape(3, 8)
assert GptOssModel._get_streamed_expert_info(mapped_name, params_dict) == (
7,
fused_name,
"gpt_oss_w13",
)
assert GptOssModel._try_load_streamed_expert(
mapped_name, loaded_weight, params_dict, loaded_params
)
assert loaded_params == {fused_name}
assert torch.equal(experts.w13_weight[1], loaded_weight[:, :4].t())
assert torch.count_nonzero(experts.w13_weight[0]) == 0
def test_streamed_expert_skips_non_local_expert() -> None:
experts = _TestExperts("w2_bias", (2, 3))
assert not _load(experts, "w2_bias", torch.ones(3), "gpt_oss_w2", expert_id=6)
assert torch.count_nonzero(experts.w2_bias) == 0
@pytest.mark.parametrize("param_name", ["w13_weight", "w2_weight"])
def test_streamed_unquantized_expert_tp_slice(param_name: str) -> None:
if param_name == "w13_weight":
experts = _TestExperts(param_name, (2, 4, 3), tp_rank=1)
loaded_weight = torch.arange(24, dtype=torch.float32).reshape(3, 8)
shard_id = "gpt_oss_w13"
expected = loaded_weight[:, 4:].t()
else:
experts = _TestExperts(param_name, (2, 3, 2), tp_rank=1)
loaded_weight = torch.arange(12, dtype=torch.float32).reshape(4, 3)
shard_id = "gpt_oss_w2"
expected = loaded_weight[2:].t()
assert _load(experts, param_name, loaded_weight, shard_id)
assert torch.equal(getattr(experts, param_name)[0], expected)
@pytest.mark.parametrize("tp_rank", [0, 1])
def test_streamed_down_bias_owned_by_tp_rank_zero(tp_rank: int) -> None:
experts = _TestExperts("w2_bias", (2, 3), tp_rank=tp_rank)
loaded_weight = torch.arange(1, 4, dtype=torch.float32)
assert _load(experts, "w2_bias", loaded_weight, "gpt_oss_w2")
expected = loaded_weight if tp_rank == 0 else torch.zeros_like(loaded_weight)
assert torch.equal(experts.w2_bias[0], expected)
def test_streamed_gate_up_bias_tp_slice() -> None:
experts = _TestExperts("w13_bias", (2, 4), tp_rank=1)
loaded_weight = torch.arange(8, dtype=torch.float32)
assert _load(experts, "w13_bias", loaded_weight, "gpt_oss_w13")
assert torch.equal(experts.w13_bias[0], loaded_weight[4:])
@pytest.mark.parametrize("case", ["mxfp4", "nvfp4"])
def test_streamed_quantized_expert_layout(case: str) -> None:
if case == "mxfp4":
experts = _TestExperts(
"w13_weight",
(2, 4, 4),
tp_rank=1,
weight_dtype="gpt_oss_mxfp4",
dtype=torch.uint8,
)
loaded_weight = torch.arange(32, dtype=torch.uint8).reshape(8, 2, 2)
expected = loaded_weight.reshape(8, 4)[4:]
else:
experts = _TestExperts(
"w13_weight",
(2, 4, 2),
tp_rank=1,
quant_method_name="Nvfp4OnlineMoEMethod",
dtype=torch.uint8,
)
loaded_weight = torch.arange(16, dtype=torch.uint8).reshape(8, 2)
expected = loaded_weight[4:]
assert _load(experts, "w13_weight", loaded_weight, "gpt_oss_w13")
assert torch.equal(experts.w13_weight[0], expected)
File diff suppressed because it is too large Load Diff
-176
View File
@@ -1,176 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
from types import SimpleNamespace
import pytest
import torch
from vllm.config import DeviceConfig, VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.rotary_embedding.bailing_mrope import (
BailingMRotaryEmbedding,
)
from vllm.model_executor.models.bailing_moe_v3 import _build_mla_rotary_embedding
from vllm.platforms import current_platform
def test_bailing_mrope_selects_alternating_spatial_then_temporal():
with set_current_vllm_config(VllmConfig(device_config=DeviceConfig("cpu"))):
rotary_emb = BailingMRotaryEmbedding(
head_size=12,
rotary_dim=12,
max_position_embeddings=8,
base=10_000,
is_neox_style=False,
dtype=torch.float32,
mrope_section=[2, 2, 2],
)
positions = torch.tensor([[1, 2], [3, 4], [5, 6]])
frequencies = torch.arange(6).expand(8, -1)
position_values = torch.arange(8).unsqueeze(1) * 10
cos = position_values + frequencies
sin = cos + 100
cos_sin_cache = torch.cat((cos, sin), dim=-1)
actual_cos, actual_sin = rotary_emb.select_cos_sin(
positions,
cos_sin_cache,
)
expected_cos = torch.tensor(
[
[30, 51, 32, 53, 14, 15],
[40, 61, 42, 63, 24, 25],
]
)
torch.testing.assert_close(actual_cos, expected_cos)
torch.testing.assert_close(actual_sin, expected_cos + 100)
def test_bailing_factory_preserves_mla_width_and_config():
config = SimpleNamespace(
max_position_embeddings=128,
rope_parameters={
"rope_type": "default",
"rope_theta": 6_000_000,
"partial_rotary_factor": 0.5,
"mrope_section": [8, 12, 12],
},
rope_scaling={"partial_rotary_factor": 0.5},
)
original = copy.deepcopy(vars(config))
with set_current_vllm_config(VllmConfig(device_config=DeviceConfig("cpu"))):
rope = _build_mla_rotary_embedding(config, head_size=64)
assert rope is _build_mla_rotary_embedding(config, head_size=64)
generic = get_rope(
head_size=64,
max_position=128,
is_neox_style=False,
rope_parameters={
"rope_type": "default",
"rope_theta": 6_000_000,
"mrope_section": [8, 12, 12],
},
)
assert isinstance(rope, BailingMRotaryEmbedding)
assert rope is not generic
assert rope.rotary_dim == 64
assert vars(config) == original
def test_bailing_factory_rejects_unsupported_scaling():
config = SimpleNamespace(
rope_parameters={
"rope_type": "yarn",
"factor": 2.0,
"mrope_section": [8, 12, 12],
}
)
with pytest.raises(ValueError, match="only supports rope_type='default'"):
_build_mla_rotary_embedding(config, head_size=64)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="Skipping CUDA/ROCm-only test.",
)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("position_kind", ["text", "multimodal"])
@torch.inference_mode()
def test_bailing_mrope_cuda_matches_native_for_strided_mla_views(
dtype: torch.dtype,
position_kind: str,
):
device = torch.device("cuda")
num_tokens = 17
num_query_heads = 8
with set_current_vllm_config(VllmConfig()):
rotary_emb = BailingMRotaryEmbedding(
head_size=64,
rotary_dim=64,
max_position_embeddings=64,
base=6_000_000,
is_neox_style=False,
dtype=dtype,
mrope_section=[8, 12, 12],
).to(device)
torch.manual_seed(0)
multimodal_positions = torch.randint(
0,
64,
(3, 2 * num_tokens),
dtype=torch.int32,
device=device,
)[:, ::2]
if position_kind == "multimodal":
positions = multimodal_positions
reference_positions = positions
else:
positions = torch.randint(
0,
64,
(2 * num_tokens,),
dtype=torch.int64,
device=device,
)[::2]
reference_positions = positions.expand(3, -1)
query_buffer = torch.randn(
num_tokens,
num_query_heads,
192,
dtype=dtype,
device=device,
)
key_buffer = torch.randn(
num_tokens,
576,
dtype=dtype,
device=device,
)
query = query_buffer[..., 128:]
key = key_buffer[:, 512:].unsqueeze(1)
assert not query.is_contiguous()
assert not key.is_contiguous()
query_prefix = query_buffer[..., :128].clone()
key_prefix = key_buffer[:, :512].clone()
expected_query, expected_key = rotary_emb.forward_native(
reference_positions,
query.clone(),
key.clone(),
)
actual_query, actual_key = rotary_emb.forward_cuda(positions, query, key)
assert expected_key is not None
assert actual_key is not None
torch.testing.assert_close(actual_query, expected_query, atol=1e-2, rtol=1.6e-2)
torch.testing.assert_close(actual_key, expected_key, atol=1e-2, rtol=1.6e-2)
assert torch.equal(query_buffer[..., :128], query_prefix)
assert torch.equal(key_buffer[:, :512], key_prefix)
if position_kind == "multimodal":
assert actual_query.data_ptr() == query.data_ptr()
assert actual_key.data_ptr() == key.data_ptr()
@@ -1,134 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import sys
from types import SimpleNamespace
import pytest
import torch
from vllm.model_executor.kernels.linear.scaled_mm.deep_gemm import (
DeepGemmFp8BlockScaledMMKernel,
)
from vllm.model_executor.warmup import deep_gemm_warmup
def _block_fp8_layer(n: int, k: int, scale_name: str = "weight_scale"):
layer = SimpleNamespace(
weight=torch.empty((n, k), dtype=torch.float8_e4m3fn),
weight_block_size=[128, 128],
deep_gemm_warmup_provider=object.__new__(DeepGemmFp8BlockScaledMMKernel),
)
setattr(layer, scale_name, torch.empty((n // 128, k // 128), dtype=torch.float32))
return layer
def _run_warmup(monkeypatch, layers) -> list[dict]:
calls: list[dict] = []
monkeypatch.setattr(
deep_gemm_warmup,
"get_mk_alignment_for_contiguous_layout",
lambda: [128, 128],
)
monkeypatch.setattr(
deep_gemm_warmup,
"_deepgemm_fp8_gemm_nt_warmup",
lambda **kwargs: calls.append(kwargs),
)
model = SimpleNamespace(modules=lambda: iter(layers))
deep_gemm_warmup.deepgemm_fp8_gemm_nt_warmup(model, max_tokens=16)
return calls
@pytest.mark.parametrize("scale_name", ["weight_scale", "weight_scale_inv"])
def test_registered_deep_gemm_layer_is_warmed_up(monkeypatch, scale_name) -> None:
"""Any layer stamped by the DeepGEMM kernel is warmed, regardless of the
quantization method that owns it or the name of its scale parameter."""
layer = _block_fp8_layer(256, 128, scale_name)
calls = _run_warmup(monkeypatch, [layer])
assert len(calls) == 1
assert calls[0]["w"] is layer.weight
assert calls[0]["ws"] is getattr(layer, scale_name)
def test_n_multiple_of_64_matches_kernel_selection(monkeypatch) -> None:
"""DeepGEMM accepts N % 64 == 0 (e.g. DeepSeek kv_a_proj_with_mqa, N=576),
so warmup must not require N % 128 == 0."""
assert len(_run_warmup(monkeypatch, [_block_fp8_layer(576, 256)])) == 1
def test_unstamped_and_mismatched_block_layers_are_skipped(monkeypatch) -> None:
mismatched = _block_fp8_layer(256, 128)
mismatched.weight_block_size = [1, 32]
layers = [SimpleNamespace(), mismatched, _block_fp8_layer(256, 128)]
calls = _run_warmup(monkeypatch, layers)
assert len(calls) == 1
assert calls[0]["w"] is layers[-1].weight
@pytest.mark.parametrize("is_bmm", [False, True])
def test_kernel_registers_itself_as_warmup_provider(is_bmm) -> None:
"""Bmm layers bypass the kernel at runtime, so they must not be warmed."""
kernel = object.__new__(DeepGemmFp8BlockScaledMMKernel)
kernel.is_deep_gemm_supported = False
layer = torch.nn.Module()
layer.weight = torch.nn.Parameter(
torch.empty((256, 128), dtype=torch.float8_e4m3fn), requires_grad=False
)
layer.weight_scale = torch.nn.Parameter(
torch.empty((2, 1), dtype=torch.float32), requires_grad=False
)
layer.weight_block_size = [128, 128]
layer.is_bmm = is_bmm
kernel.process_weights_after_loading(layer)
provider = getattr(layer, "deep_gemm_warmup_provider", None)
assert provider is (None if is_bmm else kernel)
def test_mega_moe_predicate_matches_only_native_mega_moe_layers(monkeypatch) -> None:
"""Without the MegaMoE model modules loaded, nothing matches and nothing is
imported; with them loaded, only DeepSeek-V4 and Kimi K3 MegaMoE layers match."""
name = "vllm.models.deepseek_v4.nvidia.model"
kimi_name = "vllm.models.kimi_k3.nvidia.model"
monkeypatch.delitem(sys.modules, name, raising=False)
monkeypatch.delitem(sys.modules, kimi_name, raising=False)
other = SimpleNamespace(use_native_mega_moe=True, use_mega_moe=True)
assert not deep_gemm_warmup._mega_moe_may_use_deep_gemm(other)
assert name not in sys.modules and kimi_name not in sys.modules
moe_cls = type("DeepseekV4MoE", (), {})
monkeypatch.setitem(sys.modules, name, SimpleNamespace(DeepseekV4MoE=moe_cls))
native, fused = moe_cls(), moe_cls()
native.use_native_mega_moe, fused.use_native_mega_moe = True, False
assert deep_gemm_warmup._mega_moe_may_use_deep_gemm(native)
assert not deep_gemm_warmup._mega_moe_may_use_deep_gemm(fused)
assert not deep_gemm_warmup._mega_moe_may_use_deep_gemm(other)
kimi_cls = type("KimiMoE", (), {})
monkeypatch.setitem(sys.modules, kimi_name, SimpleNamespace(KimiMoE=kimi_cls))
kimi_mega, kimi_fused = kimi_cls(), kimi_cls()
kimi_mega.use_mega_moe, kimi_fused.use_mega_moe = True, False
assert deep_gemm_warmup._mega_moe_may_use_deep_gemm(kimi_mega)
assert not deep_gemm_warmup._mega_moe_may_use_deep_gemm(kimi_fused)
def test_mega_moe_m_values_reach_mega_gate(monkeypatch) -> None:
"""Each bf16_mega_gate config is warmed with an M above the GateLinear
threshold (1 for 128 experts), so the forward actually reaches the kernel."""
dg = SimpleNamespace(
get_bf16_mega_gate_config=lambda m, *_: {"num_sms": -(-m // 16)},
)
monkeypatch.setattr(deep_gemm_warmup, "_import_deep_gemm", lambda: dg)
monkeypatch.setattr(deep_gemm_warmup, "_mega_moe_uses_mega_gate", lambda _: True)
module = SimpleNamespace(
gate=SimpleNamespace(input_size=4096, output_size=128),
experts=SimpleNamespace(top_k=6, get_block_m=lambda num_tokens: 16),
use_sequence_parallel=False,
)
assert deep_gemm_warmup._get_fp8_fp4_mega_moe_m_values(module, 40) == [16, 32, 40]
@@ -1,160 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import Mock, patch
import pytest
import torch
from vllm.config import LoadConfig, ModelConfig, SpeculativeConfig, VllmConfig
from vllm.model_executor.models.utils import get_draft_quant_config
from vllm.platforms import current_platform
DEVICE_TYPE = current_platform.device_type
DEVICES = (
[f"{DEVICE_TYPE}:{i}" for i in range(min(torch.accelerator.device_count(), 2))]
if not current_platform.is_cpu()
else ["cpu"]
)
def test_get_draft_quant_config_with_draft_model():
mock_draft_model_config = Mock(spec=ModelConfig)
mock_load_config = Mock(spec=LoadConfig)
mock_speculative_config = Mock(spec=SpeculativeConfig)
mock_speculative_config.draft_model_config = mock_draft_model_config
mock_vllm_config = Mock(spec=VllmConfig)
mock_vllm_config.speculative_config = mock_speculative_config
mock_vllm_config.load_config = mock_load_config
mock_quant_config = Mock()
with patch.object(
VllmConfig, "get_quantization_config", return_value=mock_quant_config
):
result = get_draft_quant_config(mock_vllm_config)
# Verify the function calls get_quantization_config with draft model config
VllmConfig.get_quantization_config.assert_called_once_with(
mock_draft_model_config, mock_load_config
)
assert result == mock_quant_config
def test_get_draft_quant_config_without_draft_model():
mock_speculative_config = Mock(spec=SpeculativeConfig)
mock_speculative_config.draft_model_config = None
mock_vllm_config = Mock(spec=VllmConfig)
mock_vllm_config.speculative_config = mock_speculative_config
mock_vllm_config.load_config = Mock(spec=LoadConfig)
result = get_draft_quant_config(mock_vllm_config)
assert result is None
@torch.inference_mode()
@pytest.mark.parametrize("device", DEVICES)
def test_fc_layer_quant_config_usage(default_vllm_config, dist_init, device) -> None:
import torch
from vllm.model_executor.layers.linear import ReplicatedLinear
if current_platform.is_cuda_alike():
torch.accelerator.set_device_index(device)
torch.set_default_device(device)
input_size = 256
output_size = 128
fc_no_quant = ReplicatedLinear(
input_size=input_size,
output_size=output_size,
bias=False,
params_dtype=torch.float16,
quant_config=None,
prefix="fc",
)
assert fc_no_quant.quant_config is None
assert fc_no_quant.input_size == input_size
assert fc_no_quant.output_size == output_size
mock_quant_config = Mock()
mock_quant_config.online_quantization_config = None
fc_with_quant = ReplicatedLinear(
input_size=input_size,
output_size=output_size,
bias=False,
params_dtype=torch.float16,
quant_config=mock_quant_config,
prefix="fc",
)
assert fc_with_quant.quant_config == mock_quant_config
# Check forward pass
x = torch.randn(2, input_size, dtype=torch.float16)
output, _ = fc_no_quant(x)
assert output.shape == (2, output_size)
def test_maybe_remap_kv_scale_name():
from vllm.model_executor.model_loader.weight_utils import maybe_remap_kv_scale_name
params_dict = {
"layers.0.self_attn.kv_scale": Mock(),
"layers.1.self_attn.kv_scale": Mock(),
}
name = "layers.0.self_attn.some_scale"
remapped = maybe_remap_kv_scale_name(name, params_dict)
assert remapped in params_dict or remapped == name or remapped is None
def test_eagle3_lm_head_receives_quant_config():
"""Eagle3LlamaForCausalLM must pass quant_config to ParallelLMHead.
Without quant_config, quantized lm_head weights (e.g. INT8 per-channel)
in Eagle3 drafter checkpoints fail to load because ParallelLMHead doesn't
expect weight_packed tensors.
"""
from vllm.model_executor.models.llama_eagle3 import Eagle3LlamaForCausalLM
mock_quant_config = Mock()
mock_hf_config = Mock()
mock_hf_config.draft_vocab_size = 1000
mock_hf_config.hidden_size = 256
mock_hf_config.vocab_size = 32000
mock_hf_config.logit_scale = 1.0
mock_vllm_config = Mock()
mock_vllm_config.speculative_config.draft_model_config.hf_config = mock_hf_config
mock_vllm_config.model_config.get_num_layers.return_value = 32
mock_vllm_config.speculative_config.parallel_drafting = False
with (
patch("vllm.model_executor.models.llama_eagle3.LlamaModel") as MockModel,
patch("vllm.model_executor.models.llama_eagle3.ParallelLMHead") as MockLMHead,
patch("vllm.model_executor.models.llama_eagle3.LogitsProcessor"),
patch(
"vllm.model_executor.models.llama_eagle3.get_draft_quant_config",
return_value=mock_quant_config,
),
):
MockModel.return_value.use_aux_hidden_state = True
Eagle3LlamaForCausalLM(vllm_config=mock_vllm_config)
MockLMHead.assert_called_once()
call_kwargs = MockLMHead.call_args.kwargs
assert "quant_config" in call_kwargs, (
"ParallelLMHead must receive quant_config for quantized lm_head weights"
)
assert call_kwargs["quant_config"] is mock_quant_config, (
"ParallelLMHead must receive the draft model's quant_config"
)
@@ -1,143 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
import pytest
import torch
from vllm.model_executor.models.ernie45_vl import (
Ernie4_5_VLMoeForConditionalGeneration,
)
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
MultiModalFieldElem,
MultiModalKwargsItem,
PlaceholderRange,
)
pytestmark = pytest.mark.skip_global_cleanup
@pytest.fixture(autouse=True, scope="module")
def _force_cpu_default_device():
original = torch.get_default_device()
torch.set_default_device("cpu")
yield
torch.set_default_device(original)
@dataclass
class DummyConfig:
spatial_conv_size: int = 2
temporal_conv_size: int = 2
def make_model(config: DummyConfig) -> Ernie4_5_VLMoeForConditionalGeneration:
model = object.__new__(Ernie4_5_VLMoeForConditionalGeneration)
model.config = config
return model
def make_mm_feature(
*,
modality: str,
offset: int,
length: int,
grid_thw: tuple[int, int, int],
) -> MultiModalFeatureSpec:
field_name = "image_grid_thw" if modality == "image" else "video_grid_thw"
return MultiModalFeatureSpec(
data=MultiModalKwargsItem(
{
field_name: MultiModalFieldElem(
data=torch.tensor(grid_thw),
field=None, # HACK.
),
}
),
modality=modality,
identifier="DUMMY",
mm_position=PlaceholderRange(offset=offset, length=length),
)
def test_get_mrope_input_positions_text_only():
model = make_model(DummyConfig())
positions, delta = model.get_mrope_input_positions(
input_tokens=[11, 12, 13, 14, 15],
mm_features=[],
)
expected = torch.tensor(
[
[0, 1, 2, 3, 4],
[0, 1, 2, 3, 4],
[0, 1, 2, 3, 4],
]
)
assert torch.equal(positions, expected)
assert delta == 0
def test_get_mrope_input_positions_single_image():
model = make_model(DummyConfig())
mm_features = [
make_mm_feature(
modality="image",
offset=1,
length=4,
grid_thw=(1, 4, 4),
)
]
positions, delta = model.get_mrope_input_positions(
input_tokens=[10, 20, 21, 22, 23, 30, 31],
mm_features=mm_features,
)
expected = torch.tensor(
[
[0, 1, 1, 1, 1, 3, 4],
[0, 1, 1, 2, 2, 3, 4],
[0, 1, 2, 1, 2, 3, 4],
]
)
assert torch.equal(positions, expected)
assert delta == -2
def test_get_mrope_input_positions_interleaved_image_and_video():
model = make_model(DummyConfig())
mm_features = [
make_mm_feature(
modality="image",
offset=1,
length=4,
grid_thw=(1, 4, 4),
),
make_mm_feature(
modality="video",
offset=7,
length=2,
grid_thw=(2, 4, 2),
),
]
positions, delta = model.get_mrope_input_positions(
input_tokens=[10, 20, 21, 22, 23, 30, 31, 40, 41, 50, 51],
mm_features=mm_features,
)
expected = torch.tensor(
[
[0, 1, 1, 1, 1, 3, 4, 5, 5, 7, 8],
[0, 1, 1, 2, 2, 3, 4, 5, 6, 7, 8],
[0, 1, 2, 1, 2, 3, 4, 5, 5, 7, 8],
]
)
assert torch.equal(positions, expected)
assert delta == -2
@@ -1,682 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import sys
from collections import defaultdict
from contextlib import contextmanager, nullcontext
from inspect import signature
from pathlib import Path
from types import ModuleType, SimpleNamespace
from typing import Any
from unittest.mock import Mock, call, patch
import pytest
import torch
import vllm.model_executor.warmup.kernel_warmup as warmup
import vllm.utils.flashinfer as fi_utils
from vllm.model_executor.warmup.kernel_warmup import (
_flashinfer_autotune_token_counts,
_run_flashinfer_autotune_dummy_runs,
flashinfer_autotune,
)
pytestmark = pytest.mark.cpu_test
class _FakeMoERunner:
"""Minimal MoERunner stand-in for token-count discovery tests."""
moe_config: Any
def _make_moe(*, max_deferred_tokens: int = 128, enabled: bool = True):
"""Create a fake MoE layer with a deferred-finalize token limit."""
moe = _FakeMoERunner()
moe.moe_config = SimpleNamespace(
use_deferred_moe_finalize=enabled,
defer_moe_finalize_max_num_tokens=max_deferred_tokens,
)
return moe
def _make_runner(modules, *, max_tokens: int = 8192):
"""Create a runner carrying only the state used by FlashInfer warmup."""
return SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=max_tokens),
vllm_config=SimpleNamespace(
attention_config=SimpleNamespace(hisparse_config=None),
parallel_config=SimpleNamespace(
enable_elastic_ep=False,
nnodes=1,
pipeline_parallel_size=1,
data_parallel_rank=0,
),
),
get_model=Mock(
return_value=SimpleNamespace(modules=Mock(return_value=modules))
),
_dummy_run=Mock(),
)
def test_flashinfer_autotune_token_counts_include_deferred_moe_limits():
runner = _make_runner(
[
object(),
_make_moe(max_deferred_tokens=128),
_make_moe(max_deferred_tokens=128),
_make_moe(max_deferred_tokens=64, enabled=False),
_make_moe(max_deferred_tokens=-1),
]
)
with patch("vllm.model_executor.layers.fused_moe.MoERunner", _FakeMoERunner):
token_counts = _flashinfer_autotune_token_counts(runner)
assert token_counts == (8192, 128)
def test_flashinfer_autotune_token_counts_are_bounded_and_deduplicated():
runner = _make_runner(
[_make_moe(max_deferred_tokens=4096)],
max_tokens=32,
)
with patch("vllm.model_executor.layers.fused_moe.MoERunner", _FakeMoERunner):
token_counts = _flashinfer_autotune_token_counts(runner)
assert token_counts == (32,)
@pytest.mark.parametrize("skip_attn", [False, True])
def test_flashinfer_autotune_uses_token_buckets_for_each_dummy_run(skip_attn):
from vllm.v1.worker.gpu.model_runner import GPUModelRunner as V2Runner
from vllm.v1.worker.gpu_model_runner import GPUModelRunner as V1Runner
runner = _make_runner([])
dummy_run_signature = signature((V2Runner if skip_attn else V1Runner)._dummy_run)
runner._dummy_run.side_effect = lambda **kwargs: dummy_run_signature.bind(
None, **kwargs
)
max_buckets = (1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 8192)
deferred_buckets = (1, 2, 4, 8, 16, 32, 64, 128)
with (
patch(
"vllm.model_executor.warmup.kernel_warmup."
"_flashinfer_autotune_token_counts",
return_value=(8192, 128),
),
patch(
"vllm.utils.flashinfer.flashinfer_get_hybrid_num_tokens_buckets",
side_effect=(max_buckets, deferred_buckets),
) as get_buckets,
patch("vllm.utils.flashinfer.autotune") as autotune,
):
_run_flashinfer_autotune_dummy_runs(runner, skip_attn=skip_attn)
assert get_buckets.call_args_list == [call(8192), call(128)]
assert autotune.call_args_list == [
call(tuning_buckets=max_buckets, round_up=True),
call(tuning_buckets=deferred_buckets, round_up=True),
]
assert runner._dummy_run.call_args_list == [
call(
num_tokens=8192,
skip_eplb=True,
is_profile=True,
randomize_inputs=True,
**({"skip_attn": True} if skip_attn else {}),
),
call(
num_tokens=128,
skip_eplb=True,
is_profile=True,
randomize_inputs=True,
**({"skip_attn": True} if skip_attn else {}),
),
]
@pytest.mark.parametrize(
"has_v2,elastic_ep,node_count",
[(False, False, 1), (True, False, 1), (True, True, 1), (True, False, 2)],
ids=["v1", "v2", "elastic-v1", "multi-node-v1"],
)
@pytest.mark.parametrize(
"hisparse,world_size,rank,skip_ops,failure",
[
(False, 1, 0, [], None),
(False, 2, 0, [], None),
(False, 2, 1, [], None),
(True, 2, 0, ["fp4_gemm", "bmm_fp8"], None),
(False, 2, 1, ["fp4_gemm"], "dummy"),
(False, 2, 1, [], "replayssm"),
(True, 2, 0, [], "kimi"),
],
)
def test_flashinfer_autotune_lifecycle(
monkeypatch,
tmp_path,
has_v2,
elastic_ep,
node_count,
hisparse,
world_size,
rank,
skip_ops,
failure,
):
use_v2 = has_v2 and not elastic_ep and node_count == 1 and world_size == 1
runner = _make_runner([])
runner.vllm_config.parallel_config.enable_elastic_ep = elastic_ep
runner.vllm_config.attention_config.hisparse_config = object() if hisparse else None
model = runner.get_model()
cache_path = tmp_path / "autotune.json"
rank_cache_path = cache_path.with_name(f"autotune_dp0_rank{rank}.json")
rank_cache_path.write_text(json.dumps({"_metadata": {}, "op": 1}))
events = Mock()
events.tuner.load_configs.return_value = True
events.tuner._dirty = True
world = SimpleNamespace(
rank_in_group=rank,
world_size=world_size,
cpu_group=object(),
broadcast_object=events.broadcast,
barrier=events.barrier,
)
@contextmanager
def autotune_context(**kwargs):
assert torch.is_inference_mode_enabled()
events.enter()
try:
yield
finally:
events.exit()
events.autotune.side_effect = autotune_context
if failure is not None:
getattr(events, failure).side_effect = RuntimeError("warmup failed")
monkeypatch.setitem(
sys.modules,
"flashinfer.autotuner",
SimpleNamespace(
AutoTuner=SimpleNamespace(get=lambda: events.tuner),
set_autotune_process_group=events.set_group,
),
)
monkeypatch.setitem(
sys.modules,
"flashinfer",
SimpleNamespace(autotune_v2=events.autotune, autotune_v2_reload=events.reload),
)
monkeypatch.setattr(
"vllm.distributed.parallel_state.get_world_group", lambda: world
)
monkeypatch.setattr(
"vllm.distributed.parallel_state.get_node_count", lambda: node_count
)
monkeypatch.setattr(
"vllm.distributed.parallel_state.get_pp_group",
lambda: SimpleNamespace(world_size=1),
)
monkeypatch.setattr(fi_utils, "has_flashinfer_autotune_v2", lambda: has_v2)
monkeypatch.setattr(fi_utils, "autotune", events.autotune)
monkeypatch.setattr(warmup.envs, "VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS", skip_ops)
resolve_file = Mock(return_value=cache_path)
resolve_root = Mock(return_value=str(tmp_path))
monkeypatch.setattr(warmup, "resolve_flashinfer_autotune_file", resolve_file)
monkeypatch.setattr(warmup, "resolve_flashinfer_autotune_v2_root", resolve_root)
def gather(out, value, *, group):
out[:] = [value] * world_size
monkeypatch.setattr(torch.distributed, "all_gather_object", gather)
monkeypatch.setattr(warmup, "_run_flashinfer_autotune_dummy_runs", events.dummy)
monkeypatch.setattr(warmup, "replayssm_autotune_warmup", events.replayssm)
monkeypatch.setattr(warmup, "_autotune_kimi_k3_kda_qkvg", events.kimi)
monkeypatch.setattr(
warmup, "autotune_hisparse_flashinfer_attention", events.hisparse
)
if failure is None:
warmup.flashinfer_autotune(runner)
else:
with pytest.raises(RuntimeError, match="warmup failed"):
warmup.flashinfer_autotune(runner)
kwargs = {"skip_ops": set(skip_ops)} if skip_ops else {}
expected = []
if use_v2:
resolve_file.assert_not_called()
resolve_root.assert_called_once_with()
expected.append(call.autotune(mode="tune", cache_root=str(tmp_path), **kwargs))
else:
resolve_root.assert_not_called()
resolve_file.assert_called_once_with(runner)
expected.extend(
[
call.tuner.load_configs(str(rank_cache_path)),
call.autotune(tune_mode=True, **kwargs),
]
)
expected.extend(
[call.set_group(world.cpu_group if world_size > 1 else None), call.enter()]
)
if hisparse:
expected.append(call.hisparse(runner))
for name, arg, kwargs in [
("dummy", runner, {"skip_attn": hisparse}),
("replayssm", runner, {}),
("kimi", model, {}),
]:
expected.append(getattr(call, name)(arg, **kwargs))
if failure == name:
break
expected.append(call.exit())
expected.append(call.set_group(None))
if failure is None:
if world_size > 1:
expected.append(call.barrier())
if use_v2 and world_size > 1:
expected.extend([call.reload(), call.barrier()])
elif not use_v2:
expected.append(call.tuner.save_configs(str(rank_cache_path)))
assert events.mock_calls == expected
@pytest.mark.parametrize(
"node_count,elastic_ep,pp_size",
[(1, False, 1), (2, False, 1), (1, True, 1), (1, False, 2)],
)
@pytest.mark.parametrize("world_size,rank", [(1, 0), (2, 0), (2, 1)])
@pytest.mark.parametrize("model_runner_v2,max_reqs", [(False, 2), (True, 1), (True, 2)])
def test_sparse_autotune_respects_cache_selection(
monkeypatch,
tmp_path,
node_count,
elastic_ep,
pp_size,
world_size,
rank,
model_runner_v2,
max_reqs,
):
import vllm.model_executor.warmup.flashinfer_sparse_mla_warmup as sparse
runner = _make_runner([])
runner.max_num_reqs = max_reqs
runner.vllm_config.use_v2_model_runner = model_runner_v2
runner.vllm_config.kernel_config = SimpleNamespace(enable_flashinfer_autotune=True)
runner.vllm_config.parallel_config.enable_elastic_ep = elastic_ep
runner.vllm_config.parallel_config.pipeline_parallel_size = pp_size
worker = SimpleNamespace(
model_runner=runner,
vllm_config=runner.vllm_config,
execute_model=Mock(),
sample_tokens=Mock(),
)
world = SimpleNamespace(
rank_in_group=rank,
world_size=world_size,
broadcast_object=Mock(return_value=b"cached"),
barrier=Mock(),
)
managed = Mock(return_value=nullcontext())
legacy = Mock(return_value=nullcontext())
reload = Mock()
tuner = Mock()
monkeypatch.setitem(
sys.modules,
"flashinfer",
SimpleNamespace(autotune_v2=managed, autotune_v2_reload=reload),
)
monkeypatch.setitem(
sys.modules,
"flashinfer.autotuner",
SimpleNamespace(AutoTuner=SimpleNamespace(get=lambda: tuner)),
)
monkeypatch.setattr(
"vllm.distributed.parallel_state.get_world_group", lambda: world
)
monkeypatch.setattr(
"vllm.distributed.parallel_state.get_node_count", lambda: node_count
)
monkeypatch.setattr(fi_utils, "has_flashinfer_autotune_v2", lambda: True)
monkeypatch.setattr(sparse, "has_flashinfer", lambda: True)
monkeypatch.setattr(
sparse.current_platform, "is_device_capability_family", lambda _: True
)
monkeypatch.setattr(
sparse, "_flashinfer_sparse_mla_decode_label", lambda *_: "DSv3.2"
)
monkeypatch.setattr(sparse, "resolve_flashinfer_autotune_v2_root", lambda: tmp_path)
cache_path = tmp_path / "autotune.json"
cache_path.write_bytes(b"cached")
monkeypatch.setattr(
sparse, "resolve_flashinfer_autotune_file", lambda _: cache_path
)
monkeypatch.setattr(sparse, "flashinfer_autotune", legacy)
def mixed_warmup(*args, mixed_step_context=None, **kwargs):
context = (
mixed_step_context if mixed_step_context is not None else nullcontext()
)
with context:
assert torch.is_inference_mode_enabled()
return True
mixed = Mock(side_effect=mixed_warmup)
monkeypatch.setattr(sparse, "run_mixed_prefill_decode_warmup", mixed)
assert sparse._run_flashinfer_sparse_mla_decode_autotune(worker, 16, frozenset())
if node_count == 1 and not elastic_ep and pp_size == 1 and world_size == 1:
managed.assert_called_once_with(mode="tune", cache_root=tmp_path)
legacy.assert_not_called()
world.broadcast_object.assert_not_called()
tuner.load_configs.assert_not_called()
reload.assert_not_called()
world.barrier.assert_not_called()
else:
managed.assert_not_called()
reload.assert_not_called()
assert legacy.call_count == (1 if rank == 0 else 0)
world.broadcast_object.assert_called_once_with(
b"cached" if rank == 0 else None, src=0
)
tuner.load_configs.assert_called_once_with(str(cache_path))
assert mixed.call_count == (1 if model_runner_v2 and max_reqs >= 2 else 0)
assert runner._dummy_run.call_count == (0 if mixed.called else 1)
def test_flashinfer_autotune_buckets_cover_drafter_tokens():
"""The drafter's M can exceed the pass size; buckets must include it."""
runner = _make_runner([])
runner.max_num_reqs = 256
runner.max_num_tokens = 8192
runner.speculator = SimpleNamespace(num_query_per_req=6)
with (
patch(
"vllm.model_executor.warmup.kernel_warmup."
"_flashinfer_autotune_token_counts",
return_value=(8192, 128),
),
patch(
"vllm.utils.flashinfer.flashinfer_get_hybrid_num_tokens_buckets"
) as get_buckets,
patch("vllm.utils.flashinfer.autotune"),
):
_run_flashinfer_autotune_dummy_runs(runner)
assert get_buckets.call_args_list == [call(8192), call(128 * 6)]
assert [c.kwargs["num_tokens"] for c in runner._dummy_run.call_args_list] == [
8192,
128,
]
class _AutotuneGroup:
def __init__(self, run, ranks):
self.run = run
self.ranks = tuple(ranks)
self.world_size = len(self.ranks)
self.rank_in_group = self.ranks.index(run.rank)
self.cpu_group = self
def record(self, operation):
self.run.collectives[self.ranks][self.run.rank].append(operation)
def all_gather_object(self, out, obj):
# Ranks run sequentially: snapshot every rank's file before any saves.
self.record(("all_gather",))
if isinstance(obj, bool):
out[:] = [rank not in self.run.rejected_load_ranks for rank in self.ranks]
assert out[self.rank_in_group] == obj
return
out[:] = self.run.gathered.setdefault(
self.ranks,
[
warmup._autotune_cache_fingerprint(self.run.cache_path(rank))
for rank in self.ranks
],
)
def barrier(self):
self.record(("barrier",))
class _AutotuneTuner:
def __init__(self, run):
self.run = run
self.cache = {}
self.loaded = None
self._dirty = False
def load_configs(self, path):
if self.run.rank in self.run.rejected_load_ranks:
return False
self.loaded = json.loads(Path(path).read_text())
self.cache.update(self.loaded)
return True
def clear_cache(self):
self.cache.clear()
self._dirty = False
def save_configs(self, path):
self.run.saves.append((self.run.rank, Path(path), dict(self.cache)))
Path(path).write_text(json.dumps(self.cache))
def profile(self, operation):
if operation in self.cache:
return
group = self.run.tuning_group
self.run.profile_groups[self.run.rank].append(
None if group is None else group.ranks
)
for tactic in range(2):
if group is not None:
group.record(("all_reduce", operation, tactic))
self.cache[operation] = self.run.rank // self.run.tp
self._dirty = True
class _AutotuneRun:
def __init__(self, pp, tp, cache_dir):
self.pp, self.tp = pp, tp
self.cache_dir = cache_dir
self.rank = 0
self.tuning_group = None
self.collectives: dict[tuple[int, ...], dict[int, list[tuple[Any, ...]]]] = (
defaultdict(lambda: defaultdict(list))
)
self.gathered = {}
self.rejected_load_ranks: set[int] = set()
self.tuners = {}
self.saves = []
self.profile_groups = defaultdict(list)
def cache_path(self, rank):
return self.cache_dir / f"autotune_configs_dp0_rank{rank}.json"
def world(self):
return _AutotuneGroup(self, range(self.pp * self.tp))
def tensor_group(self):
start = self.rank // self.tp * self.tp
return _AutotuneGroup(self, range(start, start + self.tp))
def pipeline_group(self):
return SimpleNamespace(world_size=self.pp)
def set_group(self, group):
self.tuning_group = group
def dummy_runs(self, runner, *, skip_attn=False):
self.tuners[self.rank].profile("shared_gemm")
if self.pp > 1 and self.rank // self.tp == 0:
self.tuners[self.rank].profile("pp0_extra_gemm")
def execute(self):
for rank in range(self.pp * self.tp):
self.rank = rank
self.tuners[rank] = _AutotuneTuner(self)
runner = _make_runner([])
runner.vllm_config.parallel_config.pipeline_parallel_size = self.pp
flashinfer_autotune(runner)
return self
def assert_collectives_match(self):
for ranks, traces in self.collectives.items():
assert set(traces) == set(ranks)
expected = traces[ranks[0]]
assert all(trace == expected for trace in traces.values()), dict(traces)
@pytest.fixture
def autotune_run(monkeypatch, tmp_path):
import vllm.utils.flashinfer as fi_utils
from vllm.distributed import parallel_state
def make_run(*, pp=2, tp=4):
run = _AutotuneRun(pp, tp, tmp_path)
monkeypatch.setattr(fi_utils, "has_flashinfer_autotune_v2", lambda: pp > 1)
monkeypatch.setattr(parallel_state, "get_node_count", lambda: 1)
autotuner = ModuleType("flashinfer.autotuner")
monkeypatch.setattr(
autotuner,
"AutoTuner",
SimpleNamespace(get=lambda: run.tuners[run.rank]),
raising=False,
)
monkeypatch.setattr(
autotuner, "set_autotune_process_group", run.set_group, raising=False
)
monkeypatch.setitem(sys.modules, "flashinfer.autotuner", autotuner)
monkeypatch.setattr(parallel_state, "get_world_group", run.world)
monkeypatch.setattr(parallel_state, "get_tp_group", run.tensor_group)
monkeypatch.setattr(parallel_state, "get_pp_group", run.pipeline_group)
monkeypatch.setattr(fi_utils, "autotune", lambda **kwargs: nullcontext())
monkeypatch.setattr(
"torch.distributed.all_gather_object",
lambda out, obj, group: group.all_gather_object(out, obj),
)
monkeypatch.setattr(
warmup,
"resolve_flashinfer_autotune_file",
lambda runner: tmp_path / "autotune_configs.json",
)
monkeypatch.setattr(
warmup, "_flashinfer_autotune_skip_ops", lambda runner: None
)
monkeypatch.setattr(
warmup, "_run_flashinfer_autotune_dummy_runs", run.dummy_runs
)
monkeypatch.setattr(warmup, "replayssm_autotune_warmup", lambda runner: None)
monkeypatch.setattr(warmup, "_autotune_kimi_k3_kda_qkvg", lambda model: None)
return run
return make_run
@pytest.mark.parametrize("tp", [1, 4])
def test_heterogeneous_pp_stages_have_compatible_collectives(autotune_run, tp):
run = autotune_run(tp=tp).execute()
run.assert_collectives_match()
for rank, groups in run.profile_groups.items():
expected = tuple(range(rank // tp * tp, (rank // tp + 1) * tp))
assert groups and all(
group == (expected if tp > 1 else None) for group in groups
)
def test_pp_stage_cache_roundtrip_isolated_and_asymmetric_hits_safe(
autotune_run, tmp_path
):
legacy = tmp_path / "autotune_configs.json"
legacy.write_text('{"legacy_world_cache": 99}')
cold = autotune_run().execute()
cold.assert_collectives_match()
assert [rank for rank, _, _ in cold.saves] == list(range(8))
assert [path for _, path, _ in cold.saves] == [cold.cache_path(r) for r in range(8)]
assert json.loads(legacy.read_text()) == {"legacy_world_cache": 99}
stage_caches = [{"shared_gemm": 0, "pp0_extra_gemm": 0}, {"shared_gemm": 1}]
assert all(cache == stage_caches[r // 4] for r, _, cache in cold.saves)
warm = autotune_run().execute()
warm.assert_collectives_match()
assert not warm.profile_groups and not warm.saves
for rank, tuner in warm.tuners.items():
assert tuner.loaded == stage_caches[rank // 4]
cold.cache_path(5).unlink()
mixed = autotune_run().execute()
mixed.assert_collectives_match()
assert set(mixed.profile_groups) == {4, 5, 6, 7}
assert all(mixed.tuners[rank].loaded is not None for rank in range(4))
assert all(mixed.tuners[rank].loaded is None for rank in range(4, 8))
def test_pp1_tunes_world_group_and_saves_per_rank(autotune_run):
run = autotune_run(pp=1, tp=4).execute()
run.assert_collectives_match()
assert [(rank, path) for rank, path, _ in run.saves] == [
(rank, run.cache_path(rank)) for rank in range(4)
]
assert all(groups == [(0, 1, 2, 3)] for groups in run.profile_groups.values())
def test_mismatched_rank_caches_are_ignored_by_every_rank(autotune_run):
"""One rank loading while others tune deadlocks the per-tactic reduce."""
cold = autotune_run(pp=1, tp=4).execute()
stale = cold.cache_path(2)
stale.write_text(json.dumps({**json.loads(stale.read_text()), "old_gemm": 0}))
rerun = autotune_run(pp=1, tp=4).execute()
rerun.assert_collectives_match()
assert all(tuner.loaded is None for tuner in rerun.tuners.values())
assert set(rerun.profile_groups) == {0, 1, 2, 3}
@pytest.mark.parametrize("pp, tp", [(1, 1), (1, 4), (2, 4)])
def test_rejected_cache_retunes_every_rank_in_its_tuning_group(autotune_run, pp, tp):
"""Matching files can still be incompatible with one rank's runtime."""
autotune_run(pp=pp, tp=tp).execute()
rerun = autotune_run(pp=pp, tp=tp)
rerun.rejected_load_ranks = {tp - 1}
rerun.execute()
rerun.assert_collectives_match()
assert set(rerun.profile_groups) == set(range(tp))
assert {rank for rank, _, _ in rerun.saves} == set(range(tp))
def test_autotune_cache_key_normalizes_kv_init_fields():
"""mamba_block_size/kv_cache_layout only resolve at engine KV-init time;
they must not feed the autotune cache key, or nothing can compute the key
before startup (the daemon tunes before any KV init). block_size must
stay in the key: the daemon replays the engine's block-size resolution
before tuning, and different block sizes may select different ops."""
from vllm.config.cache import CacheConfig
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
_normalize_cache_config_for_hash,
)
fresh = CacheConfig()
# Simulate the engine's KV-init write-backs (hybrid model values).
mutated = CacheConfig()
mutated.mamba_block_size = 512
mutated.kv_cache_layout = "LBHNC"
# Negative control: the write-backs are hashed factors.
assert mutated.compute_hash() != fresh.compute_hash()
normalized = _normalize_cache_config_for_hash(mutated)
assert normalized.compute_hash() == fresh.compute_hash()
# Factory state is returned untouched.
assert _normalize_cache_config_for_hash(fresh) is fresh
# block_size stays significant (the daemon keys on the resolved value).
resolved = CacheConfig(block_size=64)
assert (
_normalize_cache_config_for_hash(resolved).compute_hash()
!= fresh.compute_hash()
)
@@ -1,60 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.model_executor.layers.activation import (
GeluAndMul,
SiluAndMul,
get_act_and_mul_fn,
get_act_fn,
)
from vllm.model_executor.models.gemma3 import Gemma3MLP
from vllm.model_executor.models.gemma4 import Gemma4MLP
@pytest.mark.parametrize(
("activation_name", "expected_type"),
[
("gelu_pytorch_tanh", GeluAndMul),
("silu", SiluAndMul),
("swish", SiluAndMul),
],
)
def test_get_act_and_mul_fn_supports_gemma_hidden_act_aliases(
activation_name: str,
expected_type: type[torch.nn.Module],
default_vllm_config,
) -> None:
assert isinstance(get_act_and_mul_fn(activation_name), expected_type)
def test_get_act_fn_supports_swish_alias() -> None:
assert isinstance(get_act_fn("swish"), torch.nn.SiLU)
@pytest.mark.parametrize("mlp_cls", [Gemma3MLP, Gemma4MLP])
@pytest.mark.parametrize(
("activation_name", "expected_type"),
[
("gelu_pytorch_tanh", GeluAndMul),
("silu", SiluAndMul),
("swish", SiluAndMul),
],
)
def test_gemma_mlp_supports_hidden_act_variants(
mlp_cls: type[torch.nn.Module],
activation_name: str,
expected_type: type[torch.nn.Module],
default_vllm_config,
dist_init,
) -> None:
mlp = mlp_cls(
hidden_size=16,
intermediate_size=32,
hidden_activation=activation_name,
)
assert isinstance(mlp.act_fn, expected_type)
assert mlp(torch.randn(3, 16)).shape == (3, 16)
@@ -1,636 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from torch import nn
import vllm.model_executor.models.k2_horizon as k2_horizon
from tests.quantization.utils import is_quant_method_supported
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.distributed import (
cleanup_dist_env_and_memory,
init_distributed_environment,
initialize_model_parallel,
)
from vllm.model_executor.layers.linear import (
LinearBase,
UnquantizedLinearMethod,
)
from vllm.model_executor.layers.quantization.fp8 import Fp8Config, Fp8LinearMethod
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.models.k2_horizon import (
K2HorizonDecoderLayer,
K2HorizonModel,
K2HorizonRMSNorm,
_rope_weight_perm,
apply_partial_rope,
is_rope_weights_folding_supported,
)
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_open_port
from vllm.utils.torch_utils import set_default_torch_dtype, set_random_seed
DEVICE = current_platform.device_type
DTYPE = torch.bfloat16
SHAPES = [
(128, 64),
(128, 128),
(192, 128),
(96, 32),
]
def get_gptj_rope(
positions: torch.Tensor,
q: torch.Tensor,
k: torch.Tensor,
num_heads: int,
num_kv_heads: int,
head_dim: int,
rope_head_dim: int,
max_position: int,
base: float,
dtype: torch.dtype,
) -> tuple[torch.Tensor, torch.Tensor]:
rotary_emb = get_rope(
rope_head_dim,
max_position=max_position,
is_neox_style=True,
rope_parameters={"rope_theta": base},
dtype=dtype,
)
return apply_partial_rope(
rotary_emb,
positions,
q,
k,
num_heads,
num_kv_heads,
head_dim,
rope_head_dim,
fold_rope_weights=False,
)
def _apply_head_perm(
x: torch.Tensor, num: int, head_dim: int, idx: torch.Tensor
) -> torch.Tensor:
return (
x.reshape(*x.shape[:-1], num, head_dim)[..., idx].reshape(*x.shape).contiguous()
)
def get_neox_rope(
positions: torch.Tensor,
q: torch.Tensor,
k: torch.Tensor,
num_heads: int,
num_kv_heads: int,
head_dim: int,
rope_head_dim: int,
max_position: int,
base: float,
dtype: torch.dtype,
) -> tuple[torch.Tensor, torch.Tensor]:
rotary_emb = get_rope(
head_dim,
max_position=max_position,
is_neox_style=True,
rope_parameters={"rope_theta": base, "rope_dim": rope_head_dim},
dtype=dtype,
)
return apply_partial_rope(
rotary_emb,
positions,
q,
k,
num_heads,
num_kv_heads,
head_dim,
rope_head_dim,
fold_rope_weights=True,
)
def fold_qk_proj_weight(
weight: torch.Tensor, head_dim: int, idx: torch.Tensor
) -> torch.Tensor:
hidden = weight.shape[-1]
return weight.view(-1, head_dim, hidden)[:, idx, :].reshape(-1, hidden).contiguous()
def fold_qk_channels(
vec: torch.Tensor, head_dim: int, idx: torch.Tensor
) -> torch.Tensor:
return vec.view(-1, head_dim)[:, idx].reshape(-1).contiguous()
@pytest.mark.parametrize("head_dim,rope_head_dim", SHAPES)
def test_rope_fold_matches_permute_then_rope(
head_dim, rope_head_dim, default_vllm_config
):
dtype = DTYPE
set_random_seed(0)
num_tokens = 17
num_heads = 8
num_kv_heads = 2
hidden_size = 512
max_position = 4096
base = 10000.0
torch.set_default_dtype(dtype)
positions = torch.randint(0, max_position, (num_tokens,), device=DEVICE)
scale = hidden_size**-0.5
x = torch.randn(num_tokens, hidden_size, dtype=dtype, device=DEVICE)
w_q = (
torch.randn(num_heads * head_dim, hidden_size, dtype=dtype, device=DEVICE)
* scale
)
w_k = (
torch.randn(num_kv_heads * head_dim, hidden_size, dtype=dtype, device=DEVICE)
* scale
)
idx = _rope_weight_perm(head_dim, rope_head_dim).to(DEVICE)
q = torch.nn.functional.linear(x, w_q)
k = torch.nn.functional.linear(x, w_k)
q_gptj, k_gptj = get_gptj_rope(
positions,
q,
k,
num_heads,
num_kv_heads,
head_dim,
rope_head_dim,
max_position,
base,
dtype,
)
q_folded = torch.nn.functional.linear(x, fold_qk_proj_weight(w_q, head_dim, idx))
k_folded = torch.nn.functional.linear(x, fold_qk_proj_weight(w_k, head_dim, idx))
q_neox, k_neox = get_neox_rope(
positions,
q_folded,
k_folded,
num_heads,
num_kv_heads,
head_dim,
rope_head_dim,
max_position,
base,
dtype,
)
q_gptj_perm = _apply_head_perm(q_gptj, num_heads, head_dim, idx)
k_gptj_perm = _apply_head_perm(k_gptj, num_kv_heads, head_dim, idx)
atol, rtol = 1e-2, 1e-2
# Two complementary checks. First, an element-wise check that the folded
# NeoX path equals the GPT-J path up to the channel permutation P
# (neox_rope(Pq, Pk) == P @ gptj_rope(q, k)), hence permuting q_gptj/k_gptj.
# Second, the scores check below: P cancels in q @ k.T.
torch.testing.assert_close(q_neox, q_gptj_perm, atol=atol, rtol=rtol)
torch.testing.assert_close(k_neox, k_gptj_perm, atol=atol, rtol=rtol)
def scores(qt, kt):
qh = qt.reshape(num_tokens, num_heads, head_dim)
kh = kt.reshape(num_tokens, num_kv_heads, head_dim)
g = num_heads // num_kv_heads
qh = qh.reshape(num_tokens, num_kv_heads, g, head_dim).float()
kh = kh.float()
return torch.einsum("tkgd,skd->tksg", qh, kh)
torch.testing.assert_close(
scores(q_neox, k_neox), scores(q_gptj, k_gptj), atol=atol, rtol=rtol
)
@pytest.mark.parametrize("head_dim,rope_head_dim", SHAPES)
@pytest.mark.parametrize("num_heads", [8, 2])
def test_qk_bias_fold_matches_permute(
head_dim, rope_head_dim, num_heads, default_vllm_config
):
dtype = DTYPE
set_random_seed(0)
num_tokens = 17
hidden_size = 512
torch.set_default_dtype(dtype)
scale = hidden_size**-0.5
x = torch.randn(num_tokens, hidden_size, dtype=dtype, device=DEVICE)
w = (
torch.randn(num_heads * head_dim, hidden_size, dtype=dtype, device=DEVICE)
* scale
)
b = torch.randn(num_heads * head_dim, dtype=dtype, device=DEVICE)
idx = _rope_weight_perm(head_dim, rope_head_dim).to(DEVICE)
ref = _apply_head_perm(
torch.nn.functional.linear(x, w, b), num_heads, head_dim, idx
)
folded = torch.nn.functional.linear(
x,
fold_qk_proj_weight(w, head_dim, idx),
fold_qk_channels(b, head_dim, idx),
)
atol, rtol = 1e-2, 1e-2
torch.testing.assert_close(folded, ref, atol=atol, rtol=rtol)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a GPU (CUDA/ROCm)"
)
@pytest.mark.parametrize("head_dim,rope_head_dim", SHAPES)
@pytest.mark.parametrize("num_heads", [8, 2])
def test_qk_norm_scale_fold_matches_permute(
head_dim, rope_head_dim, num_heads, default_vllm_config
):
dtype = DTYPE
set_random_seed(0)
num_tokens = 13
torch.set_default_dtype(dtype)
hidden = num_heads * head_dim
idx = _rope_weight_perm(head_dim, rope_head_dim).to(DEVICE)
x = torch.randn(num_tokens, hidden, dtype=dtype, device=DEVICE)
norm = K2HorizonRMSNorm(hidden_size=hidden, n_groups=num_heads).to(DEVICE)
with torch.no_grad():
norm.weight.copy_(torch.randn(hidden, dtype=dtype, device=DEVICE))
y_ref = _apply_head_perm(norm(x.clone()), num_heads, head_dim, idx)
folded_norm = K2HorizonRMSNorm(hidden_size=hidden, n_groups=num_heads).to(DEVICE)
with torch.no_grad():
folded_norm.weight.copy_(fold_qk_channels(norm.weight, head_dim, idx))
# Permute x to emulate the folded q_proj/k_proj: at runtime the norm sees the
# already-permuted projection output (P @ x), not the raw activations.
y_folded = folded_norm(_apply_head_perm(x.clone(), num_heads, head_dim, idx))
atol, rtol = 1e-2, 1e-2
torch.testing.assert_close(y_folded, y_ref, atol=atol, rtol=rtol)
class _Fp8QKVOnlyConfig(Fp8Config):
def get_quant_method(self, layer, prefix):
if isinstance(layer, LinearBase) and not (
prefix.endswith("qkv_proj") or prefix.endswith("qk_proj")
):
return UnquantizedLinearMethod()
return super().get_quant_method(layer, prefix)
class _DummyAttention(nn.Module):
def __init__(self, *args, **kwargs) -> None:
super().__init__()
def forward(self, q, k, v):
return q
@pytest.fixture
def dist_init():
with set_current_vllm_config(VllmConfig()):
init_distributed_environment(
world_size=1,
rank=0,
distributed_init_method=f"tcp://127.0.0.1:{get_open_port()}",
local_rank=0,
backend="gloo",
)
initialize_model_parallel(1, 1)
yield
cleanup_dist_env_and_memory()
def _make_hf_config(
*,
hidden_size: int,
num_heads: int,
num_kv_heads: int,
head_dim: int,
rope_head_dim: int,
attention_bias: bool,
query_key_norm: bool,
) -> SimpleNamespace:
return SimpleNamespace(
hidden_size=hidden_size,
max_position_embeddings=4096,
dual_chunk_attention_config=None,
mlp_only_layers=[],
num_experts=0, # dense layer: avoids the MoE path
decoder_sparse_step=1,
mova_num_experts=0,
num_attention_heads=num_heads,
num_key_value_heads=num_kv_heads,
rope_parameters={"rope_theta": 10000.0},
rope_head_dim=rope_head_dim,
query_key_norm=query_key_norm,
rms_norm_eps=1e-6,
attention_bias=attention_bias,
head_dim=head_dim,
attention_gate_func=None,
intermediate_size=2 * hidden_size,
hidden_act="silu",
layernorm_num_groups=1,
vocab_size=1000,
num_hidden_layers=1,
tie_word_embeddings=False,
)
def _build_single_layer_model(
monkeypatch, hf_config: SimpleNamespace, *, quant_config=None
) -> K2HorizonModel:
monkeypatch.setattr(k2_horizon, "Attention", _DummyAttention)
vllm_config = VllmConfig()
vllm_config.model_config = SimpleNamespace(
hf_text_config=hf_config, dtype=torch.get_default_dtype()
)
vllm_config.quant_config = quant_config
with set_current_vllm_config(vllm_config):
layer = K2HorizonDecoderLayer(vllm_config=vllm_config, prefix="model.layers.0")
model = K2HorizonModel.__new__(K2HorizonModel)
nn.Module.__init__(model)
model.config = hf_config
model.quant_config = quant_config
model.num_redundant_experts = 0
model.layers = nn.ModuleList([layer])
return model
def _fold_rows(weight: torch.Tensor, head_dim: int, idx: torch.Tensor) -> torch.Tensor:
hidden = weight.shape[-1]
return weight.view(-1, head_dim, hidden)[:, idx, :].reshape(-1, hidden).contiguous()
def _fold_channels(vec: torch.Tensor, head_dim: int, idx: torch.Tensor) -> torch.Tensor:
return vec.view(-1, head_dim)[:, idx].reshape(-1).contiguous()
def _non_qk_weights(prefix, *, hidden_size, intermediate_size, q_size, device=None):
return [
(
f"{prefix}.self_attn.o_proj.weight",
torch.randn(hidden_size, q_size, device=device),
),
(
f"{prefix}.mlp.gate_proj.weight",
torch.randn(intermediate_size, hidden_size, device=device),
),
(
f"{prefix}.mlp.up_proj.weight",
torch.randn(intermediate_size, hidden_size, device=device),
),
(
f"{prefix}.mlp.down_proj.weight",
torch.randn(hidden_size, intermediate_size, device=device),
),
(
f"{prefix}.input_layernorm.weight",
torch.randn(hidden_size, device=device),
),
(
f"{prefix}.post_attention_layernorm.weight",
torch.randn(hidden_size, device=device),
),
]
@pytest.mark.usefixtures("dist_init")
@pytest.mark.parametrize("head_dim,rope_head_dim", SHAPES)
@pytest.mark.parametrize("attention_bias", [False, True])
def test_load_weights_dense_folds_qk(
head_dim,
rope_head_dim,
attention_bias,
monkeypatch,
):
dtype = DTYPE
set_random_seed(0)
num_heads = 8
num_kv_heads = 2
hidden_size = 128
intermediate_size = 2 * hidden_size
q_size = num_heads * head_dim
kv_size = num_kv_heads * head_dim
prefix = "layers.0"
with set_default_torch_dtype(dtype):
hf_config = _make_hf_config(
hidden_size=hidden_size,
num_heads=num_heads,
num_kv_heads=num_kv_heads,
head_dim=head_dim,
rope_head_dim=rope_head_dim,
attention_bias=attention_bias,
query_key_norm=True,
)
model = _build_single_layer_model(monkeypatch, hf_config, quant_config=None)
assert model.layers[0].self_attn.fold_rope_weights is True
raw = {
"q_proj.weight": torch.randn(q_size, hidden_size),
"k_proj.weight": torch.randn(kv_size, hidden_size),
"v_proj.weight": torch.randn(kv_size, hidden_size),
"q_norm.weight": torch.randn(q_size),
"k_norm.weight": torch.randn(kv_size),
}
if attention_bias:
raw.update(
{
"q_proj.bias": torch.randn(q_size),
"k_proj.bias": torch.randn(kv_size),
"v_proj.bias": torch.randn(kv_size),
"o_proj.bias": torch.randn(hidden_size),
}
)
weights: list[tuple[str, torch.Tensor]] = [
(f"{prefix}.self_attn.{name}", tensor) for name, tensor in raw.items()
]
weights += _non_qk_weights(
prefix,
hidden_size=hidden_size,
intermediate_size=intermediate_size,
q_size=q_size,
)
loaded = model.load_weights(iter(weights))
params = dict(model.named_parameters())
assert loaded == set(params)
idx = _rope_weight_perm(head_dim, rope_head_dim)
qkv = params[f"{prefix}.self_attn.qkv_proj.weight"]
q_part = qkv[:q_size]
k_part = qkv[q_size : q_size + kv_size]
v_part = qkv[q_size + kv_size :]
q_norm = params[f"{prefix}.self_attn.q_norm.weight"]
k_norm = params[f"{prefix}.self_attn.k_norm.weight"]
assert torch.equal(q_part, _fold_rows(raw["q_proj.weight"], head_dim, idx))
assert torch.equal(k_part, _fold_rows(raw["k_proj.weight"], head_dim, idx))
assert torch.equal(v_part, raw["v_proj.weight"]) # v never folded
assert torch.equal(q_norm, _fold_channels(raw["q_norm.weight"], head_dim, idx))
assert torch.equal(k_norm, _fold_channels(raw["k_norm.weight"], head_dim, idx))
assert qkv.dtype == dtype
if attention_bias:
qkv_bias = params[f"{prefix}.self_attn.qkv_proj.bias"]
q_bias = qkv_bias[:q_size]
k_bias = qkv_bias[q_size : q_size + kv_size]
v_bias = qkv_bias[q_size + kv_size :]
assert torch.equal(q_bias, _fold_channels(raw["q_proj.bias"], head_dim, idx))
assert torch.equal(k_bias, _fold_channels(raw["k_proj.bias"], head_dim, idx))
assert torch.equal(v_bias, raw["v_proj.bias"]) # v bias never folded
@pytest.mark.skipif(
not is_quant_method_supported("fp8"),
reason="fp8 is not supported.",
)
@pytest.mark.usefixtures("dist_init")
@pytest.mark.parametrize("head_dim,rope_head_dim", SHAPES)
@pytest.mark.parametrize("attention_bias", [False, True])
def test_load_weights_fp8_qk_no_fold(
head_dim,
rope_head_dim,
attention_bias,
monkeypatch,
):
set_random_seed(0)
num_heads = 8
num_kv_heads = 2
hidden_size = 128
intermediate_size = 2 * hidden_size
q_size = num_heads * head_dim
kv_size = num_kv_heads * head_dim
prefix = "layers.0"
def rand_fp8(*shape):
return (torch.randn(*shape, device=DEVICE) * 32.0).to(torch.float8_e4m3fn)
def rand_scale():
return torch.rand(1, device=DEVICE, dtype=torch.float32) + 0.1
with set_default_torch_dtype(torch.bfloat16), torch.device(DEVICE):
hf_config = _make_hf_config(
hidden_size=hidden_size,
num_heads=num_heads,
num_kv_heads=num_kv_heads,
head_dim=head_dim,
rope_head_dim=rope_head_dim,
attention_bias=attention_bias,
query_key_norm=True,
)
quant_config = _Fp8QKVOnlyConfig(
is_checkpoint_fp8_serialized=True, activation_scheme="dynamic"
)
model = _build_single_layer_model(
monkeypatch, hf_config, quant_config=quant_config
)
qkv_proj = model.layers[0].self_attn.qkv_proj
assert isinstance(qkv_proj.quant_method, Fp8LinearMethod)
assert model.layers[0].self_attn.fold_rope_weights is False
raw_w = {
"q_proj.weight": rand_fp8(q_size, hidden_size),
"k_proj.weight": rand_fp8(kv_size, hidden_size),
"v_proj.weight": rand_fp8(kv_size, hidden_size),
}
raw_s = {
"q_proj.weight_scale": rand_scale(),
"k_proj.weight_scale": rand_scale(),
"v_proj.weight_scale": rand_scale(),
}
q_norm = torch.randn(q_size, device=DEVICE)
k_norm = torch.randn(kv_size, device=DEVICE)
weights: list[tuple[str, torch.Tensor]] = [
(f"{prefix}.self_attn.{name}", tensor)
for name, tensor in {**raw_w, **raw_s}.items()
]
weights += [
(f"{prefix}.self_attn.q_norm.weight", q_norm),
(f"{prefix}.self_attn.k_norm.weight", k_norm),
]
biases = {}
if attention_bias:
biases = {
"q_proj.bias": torch.randn(q_size, device=DEVICE),
"k_proj.bias": torch.randn(kv_size, device=DEVICE),
"v_proj.bias": torch.randn(kv_size, device=DEVICE),
"o_proj.bias": torch.randn(hidden_size, device=DEVICE),
}
weights += [
(f"{prefix}.self_attn.{name}", tensor)
for name, tensor in biases.items()
]
weights += _non_qk_weights(
prefix,
hidden_size=hidden_size,
intermediate_size=intermediate_size,
q_size=q_size,
device=DEVICE,
)
loaded = model.load_weights(iter(weights))
params = dict(model.named_parameters())
assert loaded == set(params)
qkv = params[f"{prefix}.self_attn.qkv_proj.weight"]
weight_scale = params[f"{prefix}.self_attn.qkv_proj.weight_scale"]
assert qkv.dtype == torch.float8_e4m3fn
assert torch.equal(qkv[:q_size], raw_w["q_proj.weight"])
assert torch.equal(qkv[q_size : q_size + kv_size], raw_w["k_proj.weight"])
assert torch.equal(qkv[q_size + kv_size :], raw_w["v_proj.weight"])
assert torch.equal(
weight_scale,
torch.cat(
[
raw_s["q_proj.weight_scale"],
raw_s["k_proj.weight_scale"],
raw_s["v_proj.weight_scale"],
]
),
)
assert torch.equal(params[f"{prefix}.self_attn.q_norm.weight"], q_norm)
assert torch.equal(params[f"{prefix}.self_attn.k_norm.weight"], k_norm)
if attention_bias:
qkv_bias = params[f"{prefix}.self_attn.qkv_proj.bias"]
assert torch.equal(qkv_bias[:q_size], biases["q_proj.bias"])
assert torch.equal(qkv_bias[q_size : q_size + kv_size], biases["k_proj.bias"])
assert torch.equal(qkv_bias[q_size + kv_size :], biases["v_proj.bias"])
def test_is_rope_weights_folding_supported(default_vllm_config):
dense = SimpleNamespace(quant_method=UnquantizedLinearMethod())
quantized = SimpleNamespace(quant_method=object())
assert is_rope_weights_folding_supported(dense, None) is True
assert is_rope_weights_folding_supported(quantized, None) is False
assert is_rope_weights_folding_supported(dense, {"chunk_size": 8}) is False
assert is_rope_weights_folding_supported(SimpleNamespace(), None) is False
-145
View File
@@ -1,145 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass, field
import pytest
import torch
from vllm.model_executor.models.keye import KeyeForConditionalGeneration
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
MultiModalFieldElem,
MultiModalKwargsItem,
PlaceholderRange,
)
pytestmark = pytest.mark.skip_global_cleanup
@pytest.fixture(autouse=True, scope="module")
def _force_cpu_default_device():
original = torch.get_default_device()
torch.set_default_device("cpu")
yield
torch.set_default_device(original)
@dataclass
class DummyVisionConfig:
spatial_merge_size: int = 2
@dataclass
class DummyConfig:
vision_config: DummyVisionConfig = field(default_factory=DummyVisionConfig)
def make_model(config: DummyConfig) -> KeyeForConditionalGeneration:
model = object.__new__(KeyeForConditionalGeneration)
model.config = config
return model
def make_mm_feature(
*,
modality: str,
offset: int,
length: int,
grid_thw: tuple[int, int, int] | list[tuple[int, int, int]],
) -> MultiModalFeatureSpec:
field_name = "image_grid_thw" if modality == "image" else "video_grid_thw"
return MultiModalFeatureSpec(
data=MultiModalKwargsItem(
{
field_name: MultiModalFieldElem(
data=torch.tensor(grid_thw),
field=None, # HACK.
),
}
),
modality=modality,
identifier="DUMMY",
mm_position=PlaceholderRange(offset=offset, length=length),
)
def test_get_mrope_input_positions_text_only():
model = make_model(DummyConfig())
positions, delta = model.get_mrope_input_positions(
input_tokens=[11, 12, 13, 14, 15],
mm_features=[],
)
expected = torch.tensor(
[
[0, 1, 2, 3, 4],
[0, 1, 2, 3, 4],
[0, 1, 2, 3, 4],
]
)
assert torch.equal(positions, expected)
assert delta == 0
def test_get_mrope_input_positions_single_image():
model = make_model(DummyConfig())
mm_features = [
make_mm_feature(
modality="image",
offset=1,
length=4,
grid_thw=(1, 4, 4),
)
]
positions, delta = model.get_mrope_input_positions(
input_tokens=[10, 20, 21, 22, 23, 30, 31],
mm_features=mm_features,
)
expected = torch.tensor(
[
[0, 1, 1, 1, 1, 3, 4],
[0, 1, 1, 2, 2, 3, 4],
[0, 1, 2, 1, 2, 3, 4],
]
)
assert torch.equal(positions, expected)
assert delta == -2
def test_get_mrope_input_positions_interleaved_image_and_video():
model = make_model(DummyConfig())
mm_features = [
make_mm_feature(
modality="image",
offset=1,
length=4,
grid_thw=(1, 4, 4),
),
make_mm_feature(
modality="video",
offset=7,
length=4,
grid_thw=[(2, 4, 2)],
),
]
positions, delta = model.get_mrope_input_positions(
input_tokens=[10, 20, 21, 22, 23, 30, 31, 40, 41, 42, 43, 50, 51],
mm_features=mm_features,
)
expected = torch.tensor(
[
[0, 1, 1, 1, 1, 3, 4, 5, 5, 7, 7, 9, 10],
[0, 1, 1, 2, 2, 3, 4, 5, 6, 7, 8, 9, 10],
[0, 1, 2, 1, 2, 3, 4, 5, 5, 7, 7, 9, 10],
]
)
assert torch.equal(positions, expected)
assert delta == -2
@@ -1,145 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass, field
import pytest
import torch
from vllm.model_executor.models.keye_vl1_5 import KeyeVL1_5ForConditionalGeneration
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
MultiModalFieldElem,
MultiModalKwargsItem,
PlaceholderRange,
)
pytestmark = pytest.mark.skip_global_cleanup
@pytest.fixture(autouse=True, scope="module")
def _force_cpu_default_device():
original = torch.get_default_device()
torch.set_default_device("cpu")
yield
torch.set_default_device(original)
@dataclass
class DummyVisionConfig:
spatial_merge_size: int = 2
@dataclass
class DummyConfig:
vision_config: DummyVisionConfig = field(default_factory=DummyVisionConfig)
def make_model(config: DummyConfig) -> KeyeVL1_5ForConditionalGeneration:
model = object.__new__(KeyeVL1_5ForConditionalGeneration)
model.config = config
return model
def make_mm_feature(
*,
modality: str,
offset: int,
length: int,
grid_thw: tuple[int, int, int] | list[tuple[int, int, int]],
is_embed: list[bool] | None = None,
) -> MultiModalFeatureSpec:
field_name = "image_grid_thw" if modality == "image" else "video_grid_thw"
return MultiModalFeatureSpec(
data=MultiModalKwargsItem(
{
field_name: MultiModalFieldElem(
data=torch.tensor(grid_thw),
field=None, # HACK.
),
}
),
modality=modality,
identifier="DUMMY",
mm_position=PlaceholderRange(
offset=offset,
length=length,
is_embed=None if is_embed is None else torch.tensor(is_embed),
),
)
def test_get_mrope_input_positions_text_only():
model = make_model(DummyConfig())
positions, delta = model.get_mrope_input_positions(
input_tokens=[11, 12, 13, 14, 15],
mm_features=[],
)
expected = torch.tensor(
[
[0, 1, 2, 3, 4],
[0, 1, 2, 3, 4],
[0, 1, 2, 3, 4],
]
)
assert torch.equal(positions, expected)
assert delta == 0
def test_get_mrope_input_positions_single_image():
model = make_model(DummyConfig())
mm_features = [
make_mm_feature(
modality="image",
offset=1,
length=4,
grid_thw=(1, 4, 4),
)
]
positions, delta = model.get_mrope_input_positions(
input_tokens=[10, 20, 21, 22, 23, 30, 31],
mm_features=mm_features,
)
expected = torch.tensor(
[
[0, 1, 1, 1, 1, 3, 4],
[0, 1, 1, 2, 2, 3, 4],
[0, 1, 2, 1, 2, 3, 4],
]
)
assert torch.equal(positions, expected)
assert delta == -2
def test_get_mrope_input_positions_video_uses_embed_ranges():
model = make_model(DummyConfig())
mm_features = [
make_mm_feature(
modality="video",
offset=1,
length=8,
grid_thw=[(2, 4, 2)],
is_embed=[False, False, True, True, False, False, True, True],
)
]
positions, delta = model.get_mrope_input_positions(
input_tokens=[10, 101, 102, 20, 21, 103, 104, 30, 31, 40, 41],
mm_features=mm_features,
)
expected = torch.tensor(
[
[0, 1, 2, 3, 3, 5, 6, 7, 7, 9, 10],
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
[0, 1, 2, 3, 3, 5, 6, 7, 7, 9, 10],
]
)
assert torch.equal(positions, expected)
assert delta == 0
-170
View File
@@ -1,170 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.model_executor.models.minicpmv import MiniCPMV2_6
from vllm.model_executor.models.minicpmv4_6 import (
MiniCPMV4_6ForConditionalGeneration,
_expand_flags,
_select_vision_items,
_vision_downsample_mode,
)
pytestmark = pytest.mark.skip_global_cleanup
def make_model() -> MiniCPMV2_6:
return object.__new__(MiniCPMV2_6)
def make_model_4_6() -> MiniCPMV4_6ForConditionalGeneration:
model = object.__new__(MiniCPMV4_6ForConditionalGeneration)
model._process_vision_input = lambda vision_input, use_vit_merger=None: [
vision_input["image_embeds"]
]
return model
def test_video_embeds_reach_the_vision_parser():
embeds = torch.arange(32, dtype=torch.float32).reshape(1, 4, 8)
modalities = make_model()._parse_and_validate_multimodal_inputs(video_embeds=embeds)
video_input = modalities["videos"]
assert video_input is not None
assert video_input["type"] == "image_embeds"
assert torch.equal(video_input["image_embeds"], embeds)
def test_video_embeds_are_embedded_by_4_6():
embeds = torch.arange(32, dtype=torch.float32).reshape(1, 4, 8)
embeddings = make_model_4_6().embed_multimodal(video_embeds=embeds)
assert len(embeddings) == 1
assert torch.equal(embeddings[0], embeds)
def test_image_and_video_embeds_stay_in_their_own_modality():
image_embeds = torch.zeros(1, 4, 8)
video_embeds = torch.ones(1, 4, 8)
modalities = make_model()._parse_and_validate_multimodal_inputs(
image_embeds=image_embeds,
video_embeds=video_embeds,
)
assert torch.equal(modalities["images"]["image_embeds"], image_embeds)
assert torch.equal(modalities["videos"]["image_embeds"], video_embeds)
class TestPerItemMergerFlags:
"""The merger flag is per request but items from several requests can be
batched together, so it has to be applied per item."""
def test_expand_flags_broadcasts_a_single_value(self):
assert _expand_flags(None, 3) == [None, None, None]
assert _expand_flags(torch.tensor([True]), 3) == [True, True, True]
assert _expand_flags(True, 2) == [True, True]
def test_expand_flags_keeps_per_item_values(self):
flags = [torch.tensor([True]), torch.tensor([False])]
assert _expand_flags(flags, 2) == [True, False]
def test_expand_flags_falls_back_on_a_shape_mismatch(self):
# Too many values to map onto the items: keep one decision for the
# batch rather than failing the request.
assert _expand_flags([True, False, True], 2) == [True, True]
def test_vision_downsample_mode(self):
assert _vision_downsample_mode(None) is None
assert _vision_downsample_mode(True) == "16x"
assert _vision_downsample_mode(False) == "4x"
def test_select_vision_items_slices_by_item(self):
# 3 items holding 2, 1 and 2 slices; one channel, one patch, width 4.
pixel_values = [torch.full((1, 1, 4), float(i)) for i in range(5)]
tgt_sizes = torch.arange(10, dtype=torch.long).reshape(5, 2)
image_input = {
"pixel_values": pixel_values,
"tgt_sizes": tgt_sizes,
"num_slices": torch.tensor([2, 1, 2]),
}
selected = _select_vision_items(image_input, [2], [2, 1, 2])
assert len(selected["pixel_values"]) == 2
assert bool(selected["pixel_values"][0][0, 0, 0] == 3.0)
assert bool(selected["pixel_values"][1][0, 0, 0] == 4.0)
assert selected["num_slices"].tolist() == [2]
assert torch.equal(selected["tgt_sizes"], tgt_sizes[3:5])
def test_select_vision_items_preserves_order(self):
pixel_values = [torch.full((1, 1, 4), float(i)) for i in range(3)]
image_input = {
"pixel_values": pixel_values,
"tgt_sizes": torch.zeros(3, 2, dtype=torch.long),
"num_slices": torch.tensor([1, 1, 1]),
}
selected = _select_vision_items(image_input, [2, 0], [1, 1, 1])
assert selected["num_slices"].tolist() == [1, 1]
assert bool(selected["pixel_values"][0][0, 0, 0] == 2.0)
assert bool(selected["pixel_values"][1][0, 0, 0] == 0.0)
def test_process_vision_input_applies_flags_per_item(self):
calls: list[str | None] = []
def fake_get_vision_hidden_states(data, downsample_mode=None):
calls.append(downsample_mode)
tag = 1.0 if downsample_mode == "16x" else 0.0
return [torch.full((1,), tag) for _ in data["pixel_values"]]
model = object.__new__(MiniCPMV4_6ForConditionalGeneration)
model.get_vision_hidden_states = fake_get_vision_hidden_states
image_input = {
"type": "pixel_values",
"pixel_values": [torch.zeros(1, 1, 4) for _ in range(3)],
"tgt_sizes": torch.zeros(3, 2, dtype=torch.long),
"num_slices": torch.tensor([1, 1, 1]),
}
result = model._process_vision_input(
image_input,
use_vit_merger=[
torch.tensor([True]),
torch.tensor([False]),
torch.tensor([True]),
],
)
# Each item keeps its own mode even though they share one call.
assert [float(r[0]) for r in result] == [1.0, 0.0, 1.0]
# Mixed flags are encoded as separate groups, not one collapsed batch.
assert calls == ["16x", "4x"]
def test_process_vision_input_uses_one_group_for_uniform_flags(self):
calls: list[str | None] = []
def fake_get_vision_hidden_states(data, downsample_mode=None):
calls.append(downsample_mode)
return [torch.zeros(1) for _ in data["pixel_values"]]
model = object.__new__(MiniCPMV4_6ForConditionalGeneration)
model.get_vision_hidden_states = fake_get_vision_hidden_states
image_input = {
"type": "pixel_values",
"pixel_values": [torch.zeros(1, 1, 4) for _ in range(2)],
"tgt_sizes": torch.zeros(2, 2, dtype=torch.long),
"num_slices": torch.tensor([1, 1]),
}
model._process_vision_input(image_input, use_vit_merger=torch.tensor([True]))
# The common case still makes a single encoder call.
assert calls == ["16x"]
-310
View File
@@ -1,310 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.model_executor.models.minicpmv4_6 import _stack_vit_merger_qkv
from vllm.model_executor.models.minicpmv4_7 import (
MiniCPMV4_7MultiModalProcessor,
_compute_canvas_single,
build_image_bounds,
canvas_rope_delta,
)
pytestmark = pytest.mark.skip_global_cleanup
def test_canvas_mrope_sizes_survive_prefix_cache_stripping():
"""A cached image must keep `tgt_sizes`, or canvas M-RoPE falls back.
`strip_covered_mm_data` drops the payload of a prefix-cache-covered item and
keeps only CPU-side metadata. Canvas M-RoPE places slices from `tgt_sizes`,
so 4.7 marks it `keep_on_cpu` while the shared 4.6 config stays as it was.
"""
from vllm.model_executor.models.minicpmv4_6 import _minicpmv4_6_field_config
shared = _minicpmv4_6_field_config({})
assert shared["tgt_sizes"].field.keep_on_cpu is False
assert shared["video_tgt_sizes"].field.keep_on_cpu is False
processor = MiniCPMV4_7MultiModalProcessor.__new__(MiniCPMV4_7MultiModalProcessor)
fields = processor._get_mm_fields_config({}, {})
assert fields["tgt_sizes"].field.keep_on_cpu is True
assert fields["video_tgt_sizes"].field.keep_on_cpu is True
# Other fields keep their previous setting.
assert fields["pixel_values"].field.keep_on_cpu is False
assert fields["video_image_sizes"].field.keep_on_cpu is True
def test_modality_mm_kwargs_resolution():
"""Flat keys reach both modalities; scopes stay scoped, and win on conflict.
`_merge_and_resolve_mm_processor_kwargs` routes a flat key into every scope
whose HF processor declares it, so reading is a plain scope lookup. The
routing itself is upstream's, tested here against the MiniCPM-V schema to
pin the precedence rule (`scoped > flat`).
"""
from vllm.model_executor.models.minicpmv4_6 import (
_flat_processor_kwargs,
_scoped_mm_kwarg,
)
from vllm.multimodal.processing.context import _resolve_mm_processor_kwargs
# What MiniCPMV4_6{Image,Video}ProcessorKwargs declare.
processor_keys = {"downsample_mode", "max_slice_nums"}
schema = {
"text_kwargs": set(),
"images_kwargs": set(processor_keys),
"videos_kwargs": set(processor_keys),
"audio_kwargs": set(),
}
def read(mm_kwargs, modality, key):
merged = _resolve_mm_processor_kwargs(dict(mm_kwargs), schema)
return _scoped_mm_kwarg(merged, modality, key)
assert read({}, "image", "downsample_mode") is None
# A flat key is routed into both scopes.
assert read({"downsample_mode": "4x"}, "image", "downsample_mode") == "4x"
assert read({"downsample_mode": "4x"}, "video", "downsample_mode") == "4x"
# A scoped key stays in its scope.
assert (
read({"images_kwargs": {"downsample_mode": "16x"}}, "image", "downsample_mode")
== "16x"
)
assert (
read({"images_kwargs": {"downsample_mode": "16x"}}, "video", "downsample_mode")
is None
)
videos_only = {"videos_kwargs": {"max_slice_nums": 1}}
assert read(videos_only, "video", "max_slice_nums") == 1
# Scoped to videos, so image must not see it.
assert read(videos_only, "image", "max_slice_nums") is None
# An explicit scope wins over the flat key it is merged with.
assert (
read(
{"downsample_mode": "16x", "images_kwargs": {"downsample_mode": "4x"}},
"image",
"downsample_mode",
)
== "4x"
)
# A non-mapping scope is ignored rather than raising.
assert read({"images_kwargs": None}, "image", "downsample_mode") is None
# `_flat_processor_kwargs` drops the scoped keys and pins the mode.
flat = _flat_processor_kwargs(
{
"downsample_mode": "4x",
"images_kwargs": {"a": 1},
"videos_kwargs": {"b": 2},
"audio_kwargs": {"c": 3},
},
"16x",
)
assert flat == {"downsample_mode": "16x"}
def test_vit_merger_qkv_is_stacked_before_load():
mapped = list(
_stack_vit_merger_qkv(
[
(
"model.vision_tower.vit_merger.self_attn.q_proj.weight",
torch.zeros(1),
),
(
"model.vision_tower.vit_merger.self_attn.k_proj.weight",
torch.zeros(1),
),
(
"model.vision_tower.vit_merger.self_attn.v_proj.weight",
torch.zeros(1),
),
("vit_merger.self_attn.qkv_proj.weight", torch.zeros(1)),
]
)
)
names = [name for name, _ in mapped]
shards = [getattr(tensor, "shard_id", None) for _, tensor in mapped]
assert names == [
"model.vision_tower.vit_merger.self_attn.qkv_proj.weight",
"model.vision_tower.vit_merger.self_attn.qkv_proj.weight",
"model.vision_tower.vit_merger.self_attn.qkv_proj.weight",
"vit_merger.self_attn.qkv_proj.weight",
]
assert shards == ["q", "k", "v", None]
# Canvas M-RoPE goldens, cross-checked against the reference implementation in
# transformers PR 48979 (`modular_minicpmv4_7.py`). Synthetic ids keep the token
# sequences readable: the canvas math only compares ids for equality.
_IM_START = 10
_IM_END = 11
_SLICE_START = 12
_SLICE_END = 13
_NEWLINE = 14
_IMAGE_PAD = 20
_VIDEO_PAD = 21
_TEXT = 100
_CANVAS_IDS = {
"im_start_id": _IM_START,
"im_end_id": _IM_END,
"slice_start_id": _SLICE_START,
"slice_end_id": _SLICE_END,
"newline_id": _NEWLINE,
}
def _canvas_positions(tokens, target_sizes):
input_ids = torch.tensor([tokens], dtype=torch.long)
targets = torch.tensor(target_sizes, dtype=torch.long)
bounds = build_image_bounds(input_ids[0], _CANVAS_IDS)
positions = _compute_canvas_single(
input_ids[0],
torch.arange(len(tokens), dtype=torch.long),
bounds,
targets,
_CANVAS_IDS,
)
return positions.tolist(), canvas_rope_delta(positions, len(tokens))
def test_canvas_mrope_trailing_newline_stays_inside_the_canvas():
"""A newline right after `</image>` belongs to the canvas, not to the text.
The chat template joins content parts with ``\\n``, so an image followed by
text always leaves one there. The transformers reference closes a frame over
every marker including newlines; leaving it out pushes every later position
one step forward and makes `rope_delta` differ by one.
"""
tokens = [_TEXT, _IM_START, _IMAGE_PAD, _IM_END, _NEWLINE, _TEXT, _TEXT]
positions, delta = _canvas_positions(tokens, [[4, 4]])
assert positions == [
[0, 1, 1, 1, 1, 3, 4],
[0, 0, 1, 2, 1, 3, 4],
[0, 0, 1, 2, 1, 3, 4],
]
assert delta == -2
def test_canvas_mrope_thumbnail_matches_reference():
# <text> <image> PAD </image> <text>; 16x turns a 4x4 patch grid into 1 token.
tokens = [_TEXT, _IM_START, _IMAGE_PAD, _IM_END, _TEXT]
positions, delta = _canvas_positions(tokens, [[4, 4]])
assert positions == [
[0, 1, 1, 1, 3],
[0, 0, 1, 2, 3],
[0, 0, 1, 2, 3],
]
assert delta == -1
def test_canvas_mrope_slice_row_matches_reference():
# Thumbnail then a 1x2 slice row; the slices tile the thumbnail's canvas.
tokens = [
_TEXT,
_IM_START,
_IMAGE_PAD,
_IM_END,
_SLICE_START,
_IMAGE_PAD,
_SLICE_END,
_SLICE_START,
_IMAGE_PAD,
_SLICE_END,
_TEXT,
]
positions, delta = _canvas_positions(tokens, [[4, 4], [4, 4], [4, 4]])
assert positions == [
[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 4],
[0, 0, 1, 2, 1, 1, 1, 1, 1, 1, 4],
[0, 0, 1, 3, 1, 1, 1, 2, 2, 2, 4],
]
assert delta == -6
def test_canvas_mrope_resamples_thumbnail_over_canvas():
# A 8x8 patch grid is 2x2 LLM tokens, so the canvas is resampled (not arange).
tokens = [_TEXT, _IM_START] + [_IMAGE_PAD] * 4 + [_IM_END, _TEXT]
positions, delta = _canvas_positions(tokens, [[8, 8]])
assert positions == [
[0, 1, 1, 1, 1, 1, 1, 4],
[0, 0, 1, 1, 2, 2, 3, 4],
[0, 0, 1, 2, 1, 2, 3, 4],
]
assert delta == -3
def test_canvas_mrope_video_frames_share_one_canvas():
tokens = [
_TEXT,
_IM_START,
_VIDEO_PAD,
_IM_END,
_IM_START,
_VIDEO_PAD,
_IM_END,
_TEXT,
]
positions, delta = _canvas_positions(tokens, [[4, 4], [4, 4]])
assert positions == [
[0, 1, 1, 1, 3, 3, 3, 5],
[0, 0, 1, 2, 2, 3, 4, 5],
[0, 0, 1, 2, 2, 3, 4, 5],
]
assert delta == -2
def test_canvas_mrope_interleaved_image_and_video():
tokens = (
[_TEXT, _IM_START]
+ [_IMAGE_PAD] * 4
+ [_IM_END, _IM_START]
+ [_VIDEO_PAD] * 4
+ [_IM_END, _TEXT]
)
positions, delta = _canvas_positions(tokens, [[8, 8], [8, 8]])
assert positions == [
[0, 1, 1, 1, 1, 1, 1, 4, 4, 4, 4, 4, 4, 7],
[0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7],
[0, 0, 1, 2, 1, 2, 3, 3, 4, 5, 4, 5, 6, 7],
]
assert delta == -6
def test_canvas_span_is_shared_by_image_and_video():
"""A video frame and a same-geometry image must place identical offsets.
Both paths go through `_assign_canvas_span`; this pins that they cannot
drift apart now that the duplicated geometry is gone.
"""
thumb = [_IM_START] + [_IMAGE_PAD] * 4 + [_IM_END]
img_pos, _ = _canvas_positions([_TEXT, *thumb, _TEXT], [[8, 8]])
vid_pos, _ = _canvas_positions([_TEXT, *thumb, *thumb, _TEXT], [[8, 8], [8, 8]])
def offsets(pos, start):
# The <im_start> halo is one below the canvas base.
base = pos[1][start] + 1
return [[c[start + k] - base for k in range(3)] for c in pos]
assert offsets(img_pos, 0) == offsets(vid_pos, 0)
def test_canvas_mrope_delta_leaves_room_for_the_first_decoded_token():
# First decoded token must land on max_prefill_position + 1, matching
# transformers and the other native M-RoPE models.
tokens = [_TEXT, _IM_START, _IMAGE_PAD, _IM_END, _TEXT]
positions, delta = _canvas_positions(tokens, [[4, 4]])
max_position = max(max(channel) for channel in positions)
assert delta == max_position + 1 - len(tokens)
@@ -1,146 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
import torch.nn as nn
from vllm.config.compilation import CompilationMode
from vllm.model_executor.models import deepseek_v2 as deepseek_mod
from vllm.model_executor.models import mistral_large_3_eagle as eagle_mod
class DummyPPGroup:
world_size = 1
is_first_rank = True
is_last_rank = True
class DummyEmbedding(nn.Module):
def __init__(self, vocab_size, hidden_size, *args, **kwargs):
super().__init__()
self.hidden_size = hidden_size
def forward(self, input_ids):
return torch.zeros(
(*input_ids.shape, self.hidden_size),
dtype=torch.float32,
device=input_ids.device,
)
class DummyLinear(nn.Module):
def __init__(self, in_features, out_features, *args, **kwargs):
super().__init__()
self.out_features = out_features
def forward(self, x):
return torch.zeros(
(*x.shape[:-1], self.out_features),
dtype=x.dtype,
device=x.device,
)
class DummyNorm(nn.Module):
def __init__(self, *args, **kwargs):
super().__init__()
def forward(self, hidden_states, residual=None):
return hidden_states, residual
class DummyDecoderLayer(nn.Module):
def __init__(self, *args, **kwargs):
super().__init__()
def forward(self, positions, hidden_states, residual, llama_4_scaling=None):
return hidden_states, residual
def make_vllm_config(
*, model_type="mistral3", qk_nope_head_dim=128, qk_rope_head_dim=64
):
hf_config = SimpleNamespace(
model_type=model_type,
first_k_dense_replace=0,
vocab_size=32000,
hidden_size=16,
num_hidden_layers=1,
rms_norm_eps=1e-5,
qk_nope_head_dim=qk_nope_head_dim,
qk_rope_head_dim=qk_rope_head_dim,
)
return SimpleNamespace(
model_config=SimpleNamespace(hf_config=hf_config),
quant_config=None,
parallel_config=SimpleNamespace(
eplb_config=SimpleNamespace(num_redundant_experts=0),
),
scheduler_config=SimpleNamespace(max_num_batched_tokens=8),
cache_config=None,
compilation_config=SimpleNamespace(mode=CompilationMode.NONE),
)
@pytest.fixture(autouse=True)
def patch_heavy_modules(monkeypatch):
monkeypatch.setattr(eagle_mod, "get_pp_group", lambda: DummyPPGroup())
monkeypatch.setattr(deepseek_mod, "get_pp_group", lambda: DummyPPGroup())
monkeypatch.setattr(eagle_mod, "VocabParallelEmbedding", DummyEmbedding)
monkeypatch.setattr(eagle_mod, "RowParallelLinear", DummyLinear)
monkeypatch.setattr(eagle_mod, "RMSNorm", DummyNorm)
monkeypatch.setattr(eagle_mod, "DeepseekV2DecoderLayer", DummyDecoderLayer)
@pytest.mark.cpu_test
@pytest.mark.parametrize(
("model_type", "qk_nope_head_dim", "qk_rope_head_dim", "expected_use_mha"),
[
# MLA-style config: should not use MHA.
("mistral3", 128, 64, False),
# No MLA dims: should use MHA, matching DeepseekV2Model.__init__ logic.
("mistral3", 0, 0, True),
# DeepSeek model type always uses MHA by the parent logic.
("deepseek", 128, 64, True),
],
)
def test_eagle_mistral_large3_initializes_deepseek_runtime_attrs(
model_type,
qk_nope_head_dim,
qk_rope_head_dim,
expected_use_mha,
):
vllm_config = make_vllm_config(
model_type=model_type,
qk_nope_head_dim=qk_nope_head_dim,
qk_rope_head_dim=qk_rope_head_dim,
)
model = eagle_mod.EagleMistralLarge3Model(vllm_config=vllm_config)
assert model.aux_hidden_state_layers == ()
assert model.use_mha is expected_use_mha
# Add this if your fix also copies num_redundant_experts from
# DeepseekV2Model.__init__.
assert model.num_redundant_experts == 0
@pytest.mark.cpu_test
def test_eagle_mistral_large3_forward_reuses_deepseek_parent_forward():
vllm_config = make_vllm_config()
model = eagle_mod.EagleMistralLarge3Model(vllm_config=vllm_config)
input_ids = torch.tensor([[1, 2, 3]])
positions = torch.tensor([[0, 1, 2]])
hidden_states = torch.zeros((1, 3, 16))
output = model(input_ids, positions, hidden_states)
assert isinstance(output, torch.Tensor)
assert output.shape == hidden_states.shape
@@ -1,133 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
import pytest
from vllm.model_executor.layers.pooler import DispatchPooler
from vllm.model_executor.layers.pooler.seqwise import CLSPool, MeanPool
from vllm.model_executor.models.bert import BertEmbeddingModel
from vllm.model_executor.models.roberta import RobertaEmbeddingModel
from vllm.platforms import current_platform
MAX_MODEL_LEN = 128
MODEL_NAME = os.environ.get("MODEL_NAME", "BAAI/bge-base-en-v1.5")
REVISION = os.environ.get("REVISION", "main")
MODEL_NAME_ROBERTA = os.environ.get("MODEL_NAME", "intfloat/multilingual-e5-base")
REVISION_ROBERTA = os.environ.get("REVISION", "main")
@pytest.mark.skipif(
current_platform.is_rocm(), reason="Xformers backend is not supported on ROCm."
)
def test_model_loading_with_params(vllm_runner, monkeypatch):
"""Test parameter weight loading with tp>1."""
# to use apply_model
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
with vllm_runner(
model_name=MODEL_NAME,
revision=REVISION,
dtype="float16",
max_model_len=MAX_MODEL_LEN,
) as vllm_model:
output = vllm_model.embed(
"Write a short story about a robot that dreams for the first time.\n"
)
model_config = vllm_model.llm.llm_engine.model_config
model_tokenizer = vllm_model.llm.llm_engine.tokenizer
# asserts on the bert model config file
assert model_config.encoder_config["max_seq_length"] == 512
assert model_config.encoder_config["do_lower_case"]
# asserts on the pooling config files
assert model_config.pooler_config.seq_pooling_type == "CLS"
assert model_config.pooler_config.tok_pooling_type == "ALL"
assert model_config.pooler_config.use_activation
# asserts on the tokenizer loaded
assert model_config.tokenizer == "BAAI/bge-base-en-v1.5"
assert model_tokenizer.model_max_length == 512
def check_model(model):
assert isinstance(model, BertEmbeddingModel)
assert isinstance(pooler := model.pooler, DispatchPooler)
assert isinstance(pooler.poolers_by_task["embed"].pooling, CLSPool)
vllm_model.apply_model(check_model)
assert output
@pytest.mark.skipif(
current_platform.is_rocm(), reason="Xformers backend is not supported on ROCm."
)
def test_roberta_model_loading_with_params(vllm_runner, monkeypatch):
"""Test parameter weight loading with tp>1."""
# to use apply_model
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
with vllm_runner(
model_name=MODEL_NAME_ROBERTA,
revision=REVISION_ROBERTA,
dtype="float16",
max_model_len=MAX_MODEL_LEN,
) as vllm_model:
output = vllm_model.embed(
"Write a short story about a robot that dreams for the first time.\n"
)
model_config = vllm_model.llm.llm_engine.model_config
model_tokenizer = vllm_model.llm.llm_engine.tokenizer
# asserts on the bert model config file
assert model_config.encoder_config["max_seq_length"] == 512
assert not model_config.encoder_config["do_lower_case"]
# asserts on the pooling config files
assert model_config.pooler_config.seq_pooling_type == "MEAN"
assert model_config.pooler_config.tok_pooling_type == "ALL"
assert model_config.pooler_config.use_activation
# asserts on the tokenizer loaded
assert model_config.tokenizer == "intfloat/multilingual-e5-base"
assert model_tokenizer.model_max_length == 512
def check_model(model):
assert isinstance(model, RobertaEmbeddingModel)
assert isinstance(pooler := model.pooler, DispatchPooler)
assert isinstance(pooler.poolers_by_task["embed"].pooling, MeanPool)
vllm_model.apply_model(check_model)
assert output
@pytest.mark.skipif(
current_platform.is_rocm(), reason="Xformers backend is not supported on ROCm."
)
def test_facebook_roberta_model_loading_with_params(vllm_runner, monkeypatch):
"""Test loading roberta-base model with no lm_head."""
# to use apply_model
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
model_name = "FacebookAI/roberta-base"
with vllm_runner(
model_name=model_name, dtype="float16", max_model_len=MAX_MODEL_LEN
) as vllm_model:
output = vllm_model.embed(
"Write a short story about a robot that dreams for the first time.\n"
)
assert vllm_model.llm.llm_engine.model_config.tokenizer == model_name
def check_model(model):
assert isinstance(model, RobertaEmbeddingModel)
assert not hasattr(model, "lm_head")
assert isinstance(pooler := model.pooler, DispatchPooler)
assert isinstance(pooler.poolers_by_task["embed"].pooling, CLSPool)
vllm_model.apply_model(check_model)
assert output
@@ -1,61 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import Mock, patch
import torch
def test_nemotron_h_lm_head_receives_quant_config():
from vllm.model_executor.models.nemotron_h import NemotronHForCausalLM
mock_quant_config = Mock()
mock_hf_config = Mock()
mock_hf_config.vocab_size = 128
mock_hf_config.hidden_size = 64
mock_vllm_config = Mock()
mock_vllm_config.model_config.hf_config = mock_hf_config
mock_vllm_config.model_config.dtype = None
mock_vllm_config.scheduler_config = Mock()
mock_vllm_config.quant_config = mock_quant_config
with (
patch("vllm.model_executor.models.nemotron_h.NemotronHModel") as MockModel,
patch("vllm.model_executor.models.nemotron_h.ParallelLMHead") as MockLMHead,
patch("vllm.model_executor.models.nemotron_h.LogitsProcessor"),
):
MockModel.return_value.make_empty_intermediate_tensors = Mock()
MockModel.return_value.has_moe = False
NemotronHForCausalLM(vllm_config=mock_vllm_config)
MockLMHead.assert_called_once()
call_kwargs = MockLMHead.call_args.kwargs
assert call_kwargs["quant_config"] is mock_quant_config
def test_relu2_fp8_fusion_uses_registry():
from vllm.model_executor.models.nemotron_h import NemotronHMLP
projected = torch.empty((1, 1), dtype=torch.bfloat16)
fused = Mock()
act_fn = Mock()
down_proj = Mock(side_effect=lambda x: (x, None))
mlp = NemotronHMLP.__new__(NemotronHMLP)
torch.nn.Module.__init__(mlp)
mlp.up_proj = Mock(return_value=(projected, None))
mlp.down_proj = down_proj
mlp.act_fn = act_fn
with patch(
"vllm.model_executor.models.nemotron_h.maybe_fused_act_quant",
return_value=fused,
) as maybe_fused:
result = mlp(Mock())
maybe_fused.assert_called_once_with(act_fn, projected, down_proj)
act_fn.assert_not_called()
assert result is fused
@@ -1,210 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass, field
import pytest
import torch
from vllm.model_executor.models.paddleocr_vl import (
PaddleOCRVLForConditionalGeneration,
)
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
MultiModalFieldElem,
MultiModalKwargsItem,
PlaceholderRange,
)
pytestmark = pytest.mark.skip_global_cleanup
@pytest.fixture(autouse=True, scope="module")
def _force_cpu_default_device():
original = torch.get_default_device()
torch.set_default_device("cpu")
yield
torch.set_default_device(original)
@dataclass
class DummyVisionConfig:
spatial_merge_size: int = 2
patch_size: int = 14
@dataclass
class DummyConfig:
image_token_id: int = 151655
video_token_id: int = 151654
vision_start_token_id: int = 151652
vision_end_token_id: int = 151653
vision_config: DummyVisionConfig = field(default_factory=DummyVisionConfig)
def make_model(config: DummyConfig) -> PaddleOCRVLForConditionalGeneration:
model = object.__new__(PaddleOCRVLForConditionalGeneration)
model.config = config
return model
def make_mm_feature(
*,
offset: int,
length: int,
image_grid_thw: tuple[int, int, int],
) -> MultiModalFeatureSpec:
return MultiModalFeatureSpec(
data=MultiModalKwargsItem(
{
"image_grid_thw": MultiModalFieldElem(
data=torch.tensor(image_grid_thw),
field=None,
),
}
),
modality="image",
identifier="DUMMY",
mm_position=PlaceholderRange(offset=offset, length=length),
)
def test_get_mrope_input_positions_text_only():
model = make_model(DummyConfig())
input_tokens = [11, 12, 13, 14, 15]
positions, delta = model.get_mrope_input_positions(
input_tokens=input_tokens,
mm_features=[],
)
expected = torch.tensor(
[
[0, 1, 2, 3, 4],
[0, 1, 2, 3, 4],
[0, 1, 2, 3, 4],
]
)
assert torch.equal(positions, expected)
assert delta == 0
def test_get_mrope_input_positions_single_image():
model = make_model(DummyConfig())
spatial_merge_size = model.config.vision_config.spatial_merge_size
t, h, w = 1, 2, 2
num_image_tokens = t * h * w
input_tokens = (
[10]
+ [model.config.vision_start_token_id]
+ [model.config.image_token_id] * num_image_tokens
+ [model.config.vision_end_token_id]
+ [30, 31]
)
mm_features = [
make_mm_feature(
offset=2, # 1 (text) + 1 (vision_start)
length=num_image_tokens,
image_grid_thw=(t, h * spatial_merge_size, w * spatial_merge_size),
)
]
positions, delta = model.get_mrope_input_positions(
input_tokens=input_tokens,
mm_features=mm_features,
)
expected = torch.tensor(
[
[0, 1, 2, 2, 2, 2, 4, 5, 6],
[0, 1, 2, 2, 3, 3, 4, 5, 6],
[0, 1, 2, 3, 2, 3, 4, 5, 6],
]
)
assert torch.equal(positions, expected)
expected_delta = (positions.max().item() + 1) - len(input_tokens)
assert delta == expected_delta
def test_get_mrope_input_positions_multiple_images():
model = make_model(DummyConfig())
spatial_merge_size = model.config.vision_config.spatial_merge_size
t1, h1, w1 = 1, 2, 2
num1 = t1 * h1 * w1
t2, h2, w2 = 1, 1, 3
num2 = t2 * h2 * w2
input_tokens = (
[10]
+ [model.config.vision_start_token_id]
+ [model.config.image_token_id] * num1
+ [model.config.vision_end_token_id]
+ [20, 21]
+ [model.config.vision_start_token_id]
+ [model.config.image_token_id] * num2
+ [model.config.vision_end_token_id]
+ [30]
)
mm_features = [
make_mm_feature(
offset=2,
length=num1,
image_grid_thw=(t1, h1 * spatial_merge_size, w1 * spatial_merge_size),
),
make_mm_feature(
offset=2 + num1 + 1 + 2 + 1,
length=num2,
image_grid_thw=(t2, h2 * spatial_merge_size, w2 * spatial_merge_size),
),
]
positions, delta = model.get_mrope_input_positions(
input_tokens=input_tokens,
mm_features=mm_features,
)
assert positions.shape == (3, 15)
assert not torch.equal(positions[:, 2:6], torch.arange(4).expand(3, 4) + 2)
assert not torch.equal(positions[:, 10:13], torch.arange(3).expand(3, 3) + 10)
def test_get_mrope_input_positions_image_at_start():
model = make_model(DummyConfig())
spatial_merge_size = model.config.vision_config.spatial_merge_size
t, h, w = 1, 2, 2
num_tokens = t * h * w
input_tokens = (
[model.config.vision_start_token_id]
+ [model.config.image_token_id] * num_tokens
+ [model.config.vision_end_token_id]
+ [10, 11]
)
mm_features = [
make_mm_feature(
offset=1, # start token at index 0
length=num_tokens,
image_grid_thw=(t, h * spatial_merge_size, w * spatial_merge_size),
)
]
positions, delta = model.get_mrope_input_positions(
input_tokens=input_tokens,
mm_features=mm_features,
)
expected = torch.tensor(
[
[0, 1, 1, 1, 1, 3, 4, 5],
[0, 1, 1, 2, 2, 3, 4, 5],
[0, 1, 2, 1, 2, 3, 4, 5],
]
)
assert torch.equal(positions, expected)
@@ -1,28 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.models.phi4mm_utils import MultiHeadedAttention
def test_eager_attention_does_not_retain_probability_matrix():
module = MultiHeadedAttention(
n_head=2,
n_feat=16,
dropout_rate=0.0,
use_pt_scaled_dot_product_attention=False,
)
module.eval()
batch, time, feat = 2, 8, 16
x = torch.randn(batch, time, feat)
with torch.no_grad():
out = module(x, x, x, None, None, None)
assert out.shape == (batch, time, feat)
assert getattr(module, "attn", None) is None
def test_multiheaded_attention_defaults_to_scaled_dot_product():
module = MultiHeadedAttention(n_head=2, n_feat=16, dropout_rate=0.0)
assert module.use_pt_scaled_dot_product_attention is True
-72
View File
@@ -1,72 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from torch import nn
from vllm.model_executor.models import plamo3 as plamo3_mod
from vllm.model_executor.models.interfaces import supports_eagle3
from vllm.v1.worker.gpu.spec_decode.eagle.eagle3_utils import (
set_eagle3_aux_hidden_state_layers,
)
class DummyDecoderLayer(nn.Module):
def forward(self, positions, hidden_states, residual):
del positions
return hidden_states + 1, hidden_states + 2
class DummyNorm(nn.Module):
def forward(self, hidden_states, residual):
return hidden_states + residual, None
@pytest.mark.cpu_test
def test_plamo3_returns_dflash_selected_auxiliary_hidden_states(monkeypatch):
decoder = plamo3_mod.Plamo3Decoder.__new__(plamo3_mod.Plamo3Decoder)
nn.Module.__init__(decoder)
decoder.start_layer = 0
decoder.end_layer = 2
decoder.layers = nn.ModuleList([DummyDecoderLayer() for _ in range(8)])
model = plamo3_mod.Plamo3Model.__new__(plamo3_mod.Plamo3Model)
nn.Module.__init__(model)
model.layers = decoder
model.norm = DummyNorm()
model.do_not_compile = True
target = plamo3_mod.Plamo3ForCausalLM.__new__(plamo3_mod.Plamo3ForCausalLM)
nn.Module.__init__(target)
target.model = model
monkeypatch.setattr(
plamo3_mod,
"get_pp_group",
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
)
assert supports_eagle3(target)
assert target.get_eagle3_default_aux_hidden_state_layers() == (2, 4, 5)
spec_config = SimpleNamespace(
draft_model_config=SimpleNamespace(
hf_config=SimpleNamespace(dflash_config={"target_layer_ids": [0, 1]})
)
)
set_eagle3_aux_hidden_state_layers(target, spec_config)
assert decoder.aux_hidden_state_layers == (1, 2)
output, aux_hidden_states = target(
input_ids=None,
positions=torch.tensor([0]),
inputs_embeds=torch.tensor([[1.0, 2.0]]),
)
torch.testing.assert_close(output, torch.tensor([[7.0, 9.0]]))
assert len(aux_hidden_states) == 2
torch.testing.assert_close(aux_hidden_states[0], torch.tensor([[5.0, 7.0]]))
torch.testing.assert_close(aux_hidden_states[1], torch.tensor([[7.0, 9.0]]))
@@ -1,27 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.models.qwen3_asr import (
Qwen3ASRForConditionalGeneration,
)
from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange
def test_mrope_positions_with_stripped_audio_data() -> None:
input_tokens = list(range(7))
feature = MultiModalFeatureSpec(
data=None,
modality="audio",
identifier="audio",
mm_position=PlaceholderRange(offset=2, length=3),
)
positions, delta = Qwen3ASRForConditionalGeneration.get_mrope_input_positions(
None, input_tokens, [feature]
)
expected = torch.arange(len(input_tokens), dtype=torch.long).expand(3, -1)
torch.testing.assert_close(positions, expected)
assert delta == 0
-424
View File
@@ -1,424 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import Mock, patch
import pytest
import torch
import torch.nn as nn
from transformers import PreTrainedConfig
from vllm.config import ParallelConfig
from vllm.multimodal.processing import InputProcessingContext
# Helper function to print input IDs with coalesced audio/video tokens.
def print_input_ids(input_ids):
"""Print input IDs, compressing consecutive special tokens.
- 151675: <|audio_pad|>
- 151656: <|video_pad|>
"""
if not input_ids:
print("[]")
return
result = []
i = 0
while i < len(input_ids):
current_id = input_ids[i]
# Check if it's a special token that should be compressed
if current_id in [151675, 151656]:
# Count consecutive occurrences
count = 1
while i + count < len(input_ids) and input_ids[i + count] == current_id:
count += 1
# Add compressed representation
token_name = "<|audio_pad|>" if current_id == 151675 else "<|video_pad|>"
result.append(f"{token_name} * {count}")
i += count
else:
# Regular token, just add it
result.append(str(current_id))
i += 1
print(", ".join(result))
@pytest.fixture
def mock_qwen3_omni_config():
"""Create a mock Qwen3OmniMoeThinker config."""
config = Mock(spec=PreTrainedConfig)
# Token IDs from https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Instruct/blob/main/tokenizer_config.json
config.audio_token_id = 151675 # <|audio_pad|>
config.video_token_id = 151656 # <|video_pad|>
config.image_token_id = 151655 # <|image_pad|>
config.audio_start_token_id = 151669 # <|audio_start|>
config.audio_end_token_id = 151670 # <|audio_end|>
config.vision_start_token_id = 151652 # <|vision_start|>
config.position_id_per_seconds = 12.5
# Vision config
vision_config = Mock()
vision_config.spatial_merge_size = 2
config.vision_config = vision_config
return config
@pytest.fixture
def mock_processor():
"""Create a mock HF processor."""
from transformers.models.whisper import WhisperFeatureExtractor
processor = Mock()
processor.audio_token = "<|audio_pad|>"
processor.image_token = "<|image_pad|>"
processor.video_token = "<|video_pad|>"
# Create a real WhisperFeatureExtractor instance for the feature_extractor attribute
feature_extractor = WhisperFeatureExtractor()
processor.feature_extractor = feature_extractor
return processor
@pytest.fixture
def mock_tokenizer():
"""Create a mock tokenizer."""
tokenizer = Mock()
# Token IDs from https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Instruct/blob/main/tokenizer_config.json
tokenizer.get_vocab = Mock(
return_value={
"<|audio_pad|>": 151675,
"<|video_pad|>": 151656,
"<|image_pad|>": 151655,
"<|audio_start|>": 151669,
"<|audio_end|>": 151670,
"<|vision_start|>": 151652,
"<|vision_end|>": 151653,
}
)
tokenizer.encode = Mock(
side_effect=lambda x: {
"<|vision_start|>": [151652],
"<|vision_end|>": [151653],
"<|audio_start|>": [151669],
"<|audio_end|>": [151670],
"<|audio_pad|>": [151675],
"<|image_pad|>": [151655],
"<|video_pad|>": [151656],
}.get(x, [0])
)
tokenizer.vision_bos_token = "<|vision_start|>"
tokenizer.vision_eos_token = "<|vision_end|>"
tokenizer.audio_bos_token = "<|audio_start|>"
tokenizer.audio_eos_token = "<|audio_end|>"
return tokenizer
@pytest.fixture
def mock_image_processor():
"""Create a mock image processor."""
image_processor = Mock()
image_processor.merge_size = 2
return image_processor
def test_qwen3_omni_get_updates_use_audio_in_video(
mock_qwen3_omni_config,
mock_processor,
mock_tokenizer,
mock_image_processor,
):
"""Test the get_updates_use_audio_in_video method directly."""
from vllm.model_executor.models.qwen3_omni_moe_thinker import (
Qwen3OmniMoeThinkerMultiModalProcessor,
Qwen3OmniMoeThinkerProcessingInfo,
)
# Create a mock context
mock_ctx = Mock(spec=InputProcessingContext)
mock_ctx.tokenizer = mock_tokenizer
# `model_config` is an instance attribute, so it is not covered by the
# spec; the data parser reads it for `allow_missing_mm_embeddings`.
mock_ctx.model_config = SimpleNamespace(
multimodal_config=SimpleNamespace(allow_missing_mm_embeddings=False)
)
# Create processing info
info = Qwen3OmniMoeThinkerProcessingInfo(mock_ctx)
info._get_expected_hidden_size = lambda: 100
info.get_hf_config = Mock(return_value=mock_qwen3_omni_config)
info.get_hf_processor = Mock(return_value=mock_processor)
info.get_tokenizer = Mock(return_value=mock_tokenizer)
info.get_image_processor = Mock(return_value=mock_image_processor)
# Create a mock dummy_inputs builder
mock_dummy_inputs = Mock()
# Create the processor
processor = Qwen3OmniMoeThinkerMultiModalProcessor(info, mock_dummy_inputs)
# Test parameters from reference video
# https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Omni/demo/draw.mp4
audio_len = 85
video_grid_thw = [6, 36, 64]
video_second_per_grid_t = 2.0
# Call the method
updates = processor.get_updates_use_audio_in_video(
thinker_config=mock_qwen3_omni_config,
audio_len=audio_len,
video_grid_thw=video_grid_thw,
video_second_per_grid_t=video_second_per_grid_t,
)
# Updated input ids should align with HF implementation.
# 151669,
# <|video_pad|> * 576, <|audio_pad|> * 25,
# <|video_pad|> * 576, <|audio_pad|> * 25,
# <|video_pad|> * 576, <|audio_pad|> * 25,
# <|video_pad|> * 576, <|audio_pad|> * 10,
# <|video_pad|> * 1152,
# 151670
print_input_ids(updates)
# Verify structure
assert isinstance(updates, list)
assert len(updates) > 0
# Verify start and end tokens
audio_start_token_id = mock_qwen3_omni_config.audio_start_token_id
audio_end_token_id = mock_qwen3_omni_config.audio_end_token_id
assert updates[0] == audio_start_token_id
assert updates[-1] == audio_end_token_id
# Verify both audio and video tokens are present
audio_token_id = mock_qwen3_omni_config.audio_token_id
video_token_id = mock_qwen3_omni_config.video_token_id
audio_count = updates.count(audio_token_id)
video_count = updates.count(video_token_id)
assert audio_count == audio_len, (
f"Expected {audio_len} audio tokens, got {audio_count}"
)
# Calculate expected video token count
spatial_merge_size = mock_qwen3_omni_config.vision_config.spatial_merge_size
height = video_grid_thw[1] // spatial_merge_size
width = video_grid_thw[2] // spatial_merge_size
expected_video_count = video_grid_thw[0] * height * width
assert video_count == expected_video_count, (
f"Expected {expected_video_count} video tokens, got {video_count}"
)
# Total tokens should be: 1 (start) + audio_len + video_count + 1 (end)
expected_total = 1 + audio_len + expected_video_count + 1
assert len(updates) == expected_total, (
f"Expected {expected_total} total tokens, got {len(updates)}"
)
@pytest.mark.skip_global_cleanup
def test_qwen3_omni_exposes_eagle3_to_its_text_backbone():
from vllm.model_executor.models.interfaces import EagleModelMixin, supports_eagle3
from vllm.model_executor.models.qwen3_omni_moe_thinker import (
Qwen3OmniMoeThinkerForConditionalGeneration,
)
class DummyBackbone(nn.Module, EagleModelMixin):
def __init__(self):
super().__init__()
self.layers = nn.ModuleList([nn.Identity(), nn.Identity()])
class DummyLanguageModel(nn.Module):
def __init__(self):
super().__init__()
self.model = DummyBackbone()
def embed_input_ids(self, input_ids):
return input_ids
model = Qwen3OmniMoeThinkerForConditionalGeneration.__new__(
Qwen3OmniMoeThinkerForConditionalGeneration
)
nn.Module.__init__(model)
model.language_model = DummyLanguageModel()
assert supports_eagle3(model)
model.set_aux_hidden_state_layers((1, 2))
assert model.language_model.model.aux_hidden_state_layers == (1, 2)
@pytest.mark.skip_global_cleanup
def test_qwen3_omni_text_model_collects_post_deepstack_aux_hidden_states():
from vllm.model_executor.models.qwen3_omni_moe_thinker import Qwen3MoeLLMModel
class DummyLayer(nn.Module):
def forward(self, positions, hidden_states, residual):
return hidden_states + 1, torch.full_like(hidden_states, 10)
class DummyNorm(nn.Module):
def forward(self, hidden_states, residual):
return hidden_states + residual, None
model = Qwen3MoeLLMModel.__new__(Qwen3MoeLLMModel)
nn.Module.__init__(model)
model.start_layer = 0
model.end_layer = 1
model.layers = nn.ModuleList([DummyLayer()])
model.norm = DummyNorm()
model.aux_hidden_state_layers = (1,)
pp_group = Mock(is_first_rank=True, is_last_rank=True)
inputs_embeds = torch.tensor([[1.0]])
deepstack_inputs = {"deepstack_input_embeds_0": torch.tensor([[3.0]])}
with patch(
"vllm.model_executor.models.qwen3_omni_moe_thinker.get_pp_group",
return_value=pp_group,
):
output, aux_hidden_states = model.forward(
input_ids=None,
positions=torch.tensor([0]),
inputs_embeds=inputs_embeds,
deepstack_input_embeds=deepstack_inputs,
)
torch.testing.assert_close(output, torch.tensor([[15.0]]))
assert len(aux_hidden_states) == 1
torch.testing.assert_close(aux_hidden_states[0], torch.tensor([[15.0]]))
def _dspark_vocab_stub(input_vocab_size: int, draft_vocab_size: int):
"""A Qwen3 DSpark model with only what `load_weights` reads."""
from vllm.model_executor.models.qwen3_dspark import Qwen3DSparkForCausalLM
model = Qwen3DSparkForCausalLM.__new__(Qwen3DSparkForCausalLM)
nn.Module.__init__(model)
object.__setattr__(
model,
"config",
SimpleNamespace(
vocab_size=input_vocab_size,
draft_vocab_size=draft_vocab_size,
),
)
object.__setattr__(model, "target_vocab_size", 100)
object.__setattr__(model, "model", Mock(confidence_head=None))
return model
@pytest.mark.skip_global_cleanup
@pytest.mark.parametrize(
("input_vocab_size", "draft_vocab_size", "weights", "error"),
[
(101, 100, [], "must include embed_tokens weights"),
(99, 99, [], "must include lm_head weights"),
(100, 40, [], "must include lm_head weights"),
(
100,
40,
[("lm_head.weight", torch.empty(40, 8))],
"must include a d2t mapping",
),
],
)
def test_qwen3_dspark_rejects_incomplete_vocab_weights(
input_vocab_size, draft_vocab_size, weights, error
):
model = _dspark_vocab_stub(input_vocab_size, draft_vocab_size)
with pytest.raises(ValueError, match=error):
model.load_weights(weights)
@pytest.mark.skip_global_cleanup
@pytest.mark.parametrize("draft_vocab_size", [100, 101])
def test_qwen3_dspark_accepts_draft_vocab_at_least_target(draft_vocab_size):
"""A draft vocab >= the target's maps identically, so needs no lm_head/d2t.
Guards the padded case (draft > target): the draft's unembedding is padded
to its physical size, which is not a reduced vocabulary.
"""
model = _dspark_vocab_stub(100, draft_vocab_size)
model.load_weights([])
@pytest.mark.skip_global_cleanup
def test_dspark_shares_target_embedding_with_smaller_draft_vocabulary():
from vllm.v1.worker.gpu.spec_decode.dspark import utils as dspark_utils
target_embedding = nn.Embedding(100, 8)
draft_embedding = nn.Embedding(99, 8)
target_model = SimpleNamespace(model=SimpleNamespace(embed_tokens=target_embedding))
draft_model = SimpleNamespace(
model=SimpleNamespace(embed_tokens=draft_embedding),
has_own_embed_tokens=False,
)
draft_model_config = SimpleNamespace(
hf_config=SimpleNamespace(model_type="qwen3"),
get_vocab_size=Mock(return_value=99),
)
vllm_config = SimpleNamespace(
speculative_config=SimpleNamespace(
draft_model_config=draft_model_config,
draft_parallel_config=SimpleNamespace(tensor_parallel_size=1),
attention_backend=None,
kv_cache_dtype=None,
draft_load_config=None,
),
parallel_config=ParallelConfig(),
attention_config=SimpleNamespace(backend=None),
cache_config=SimpleNamespace(),
load_config=SimpleNamespace(load_format="auto"),
model_config=SimpleNamespace(get_vocab_size=Mock(return_value=100)),
)
def fake_replace(config, **changes):
values = vars(config).copy()
values.update(changes)
return SimpleNamespace(**values)
with (
patch.object(dspark_utils, "replace", side_effect=fake_replace),
patch(
"vllm.v1.worker.gpu.spec_decode.eagle.utils.get_pp_group",
return_value=SimpleNamespace(world_size=1),
),
patch(
"vllm.v1.worker.gpu.spec_decode.utils.get_pp_group",
return_value=SimpleNamespace(world_size=1),
),
patch(
"vllm.compilation.backends.set_model_tag",
return_value=nullcontext(),
),
patch(
"vllm.model_executor.model_loader.get_model",
return_value=draft_model,
),
patch(
"vllm.model_executor.models.qwen3_dflash.dflash_has_any_non_causal",
return_value=False,
),
patch(
"vllm.model_executor.models.utils.get_draft_quant_config",
return_value=None,
),
):
loaded_model = dspark_utils.load_dspark_model(target_model, vllm_config)
assert loaded_model.model.embed_tokens is target_embedding
if __name__ == "__main__":
pytest.main([__file__, "-v"])
@@ -1,216 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import dataclasses
from dataclasses import dataclass
import pytest
import torch
from vllm.model_executor.models.qwen3_omni_moe_thinker import (
Qwen3OmniMoeThinkerForConditionalGeneration,
)
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
MultiModalFieldElem,
MultiModalKwargsItem,
PlaceholderRange,
)
@pytest.fixture(autouse=True, scope="module")
def _force_cpu_default_device():
# get_mrope_input_positions returns CPU tensors (via torch.from_numpy).
# Ensure the default device is CPU so the rest of the test tensors match.
original = torch.get_default_device()
torch.set_default_device("cpu")
yield
torch.set_default_device(original)
TEXT_TOKEN_ID = 1
VISION_START_TOKEN_ID = 100
VISION_END_TOKEN_ID = 101
IMAGE_TOKEN_ID = 200
VIDEO_TOKEN_ID = 201
AUDIO_TOKEN_ID = 202
AUDIO_START_TOKEN_ID = 300
AUDIO_END_TOKEN_ID = 301
@dataclass
class DummyVisionConfig:
spatial_merge_size: int = 1
@dataclass
class DummyConfig:
position_id_per_seconds: int = 25
vision_config: DummyVisionConfig = dataclasses.field(
default_factory=DummyVisionConfig
)
def make_model() -> Qwen3OmniMoeThinkerForConditionalGeneration:
model = Qwen3OmniMoeThinkerForConditionalGeneration.__new__(
Qwen3OmniMoeThinkerForConditionalGeneration
)
model.config = DummyConfig()
return model
def _elem(data):
return MultiModalFieldElem(
data=torch.tensor(data),
field=None, # HACK.
)
def make_image_feature(offset: int, grid_thw: tuple[int, int, int]):
return MultiModalFeatureSpec(
data=MultiModalKwargsItem({"image_grid_thw": _elem(grid_thw)}),
modality="image",
identifier="DUMMY",
mm_position=PlaceholderRange(
offset=offset, length=int(torch.tensor(grid_thw).prod())
),
)
def make_video_feature(
offset: int, grid_thw: tuple[int, int, int], use_audio_in_video: bool = False
):
data = {"video_grid_thw": _elem(grid_thw)}
if use_audio_in_video:
data["use_audio_in_video"] = _elem(True)
return MultiModalFeatureSpec(
data=MultiModalKwargsItem(data),
modality="video",
identifier="DUMMY",
mm_position=PlaceholderRange(
offset=offset, length=int(torch.tensor(grid_thw).prod())
),
)
def make_audio_feature(offset: int, audio_feature_length: int):
return MultiModalFeatureSpec(
data=MultiModalKwargsItem(
{"audio_feature_lengths": _elem(audio_feature_length)}
),
modality="audio",
identifier="DUMMY",
mm_position=PlaceholderRange(offset=offset, length=0),
)
def test_mrope_positions_image_at_prompt_end():
"""A prompt ending with <|vision_end|> must still produce seq_len positions."""
model = make_model()
# [<|vision_start|>, <|image_pad|> x 4, <|vision_end|>]
input_tokens = (
[VISION_START_TOKEN_ID] + [IMAGE_TOKEN_ID] * 4 + [VISION_END_TOKEN_ID]
)
mm_features = [make_image_feature(offset=1, grid_thw=(1, 2, 2))]
positions, _ = model.get_mrope_input_positions(input_tokens, mm_features)
assert positions.shape == (3, len(input_tokens))
def test_mrope_positions_audio_at_prompt_end():
model = make_model()
# audio_feature_length=16 -> 2 audio tokens.
# [<|audio_start|>, <|audio_pad|> x 2, <|audio_end|>]
input_tokens = [AUDIO_START_TOKEN_ID] + [AUDIO_TOKEN_ID] * 2 + [AUDIO_END_TOKEN_ID]
mm_features = [make_audio_feature(offset=1, audio_feature_length=16)]
positions, _ = model.get_mrope_input_positions(input_tokens, mm_features)
assert positions.shape == (3, len(input_tokens))
def test_mrope_positions_video_at_prompt_end():
model = make_model()
# [T, <|vision_start|>, <|video_pad|> x 8, <|vision_end|>]
input_tokens = (
[TEXT_TOKEN_ID]
+ [VISION_START_TOKEN_ID]
+ [VIDEO_TOKEN_ID] * 8
+ [VISION_END_TOKEN_ID]
)
mm_features = [make_video_feature(offset=2, grid_thw=(2, 2, 2))]
positions, _ = model.get_mrope_input_positions(input_tokens, mm_features)
assert positions.shape == (3, len(input_tokens))
@pytest.mark.parametrize("num_videos", [1, 2])
@pytest.mark.parametrize("trailing_tokens", [0, 3])
def test_mrope_positions_audio_in_video_boundaries(num_videos, trailing_tokens):
"""Interleaved BOS/EOS positions and decode delta match Qwen3-Omni HF."""
model = make_model()
video_tokens = (
[VISION_START_TOKEN_ID, AUDIO_START_TOKEN_ID]
+ [VIDEO_TOKEN_ID] * 4
+ [AUDIO_TOKEN_ID] * 2
+ [AUDIO_END_TOKEN_ID, VISION_END_TOKEN_ID]
)
input_tokens = [TEXT_TOKEN_ID] * 2
mm_features = []
expected = [torch.tensor([[0, 1]]).expand(3, -1)]
video_positions = torch.tensor(
[
[2, 3, 4, 4, 4, 4, 4, 5, 6, 7],
[2, 3, 4, 4, 5, 5, 4, 5, 6, 7],
[2, 3, 4, 5, 4, 5, 4, 5, 6, 7],
]
)
for index in range(num_videos):
offset = len(input_tokens) + 1
mm_features.extend(
[
make_video_feature(offset, (1, 2, 2), use_audio_in_video=True),
make_audio_feature(offset, audio_feature_length=16),
]
)
input_tokens.extend(video_tokens)
expected.append(video_positions + index * 6)
input_tokens.extend([TEXT_TOKEN_ID] * trailing_tokens)
next_position = 2 + num_videos * 6
expected.append((torch.arange(trailing_tokens) + next_position).expand(3, -1))
positions, delta = model.get_mrope_input_positions(input_tokens, mm_features)
torch.testing.assert_close(positions, torch.cat(expected, dim=1))
assert delta == -4 * num_videos
def test_mrope_positions_image_then_text():
"""Media followed by text: positions must match the expected layout exactly.
Before the fix this case passed the length guard but every position from
the first <|image_pad|> onward was silently shifted by one slot.
"""
model = make_model()
# [T, T, <|vision_start|>, <|image_pad|> x 4, <|vision_end|>, T, T, T]
input_tokens = (
[TEXT_TOKEN_ID] * 2
+ [VISION_START_TOKEN_ID]
+ [IMAGE_TOKEN_ID] * 4
+ [VISION_END_TOKEN_ID]
+ [TEXT_TOKEN_ID] * 3
)
mm_features = [make_image_feature(offset=3, grid_thw=(1, 2, 2))]
positions, delta = model.get_mrope_input_positions(input_tokens, mm_features)
expected = torch.tensor(
[
[0, 1, 2, 3, 3, 3, 3, 5, 6, 7, 8],
[0, 1, 2, 3, 3, 4, 4, 5, 6, 7, 8],
[0, 1, 2, 3, 4, 3, 4, 5, 6, 7, 8],
]
)
torch.testing.assert_close(positions, expected)
assert delta == 9 - len(input_tokens)
-237
View File
@@ -1,237 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import dataclasses
import random
from dataclasses import dataclass
import pytest
import torch
from vllm.model_executor.models.qwen3_vl import Qwen3VLForConditionalGeneration
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
MultiModalFieldElem,
MultiModalKwargsItem,
PlaceholderRange,
)
@pytest.fixture(autouse=True, scope="module")
def _force_cpu_default_device():
# _get_mrope_input_positions returns CPU tensors (via torch.from_numpy).
# Ensure the default device is CPU so the rest of the test tensors match.
original = torch.get_default_device()
torch.set_default_device("cpu")
yield
torch.set_default_device(original)
IMAGE_TOKEN_ID = 999
VIDEO_TOKEN_ID = 888
VISION_START_TOKEN_ID = 777
VISION_END_TOKEN_ID = 778
@dataclass
class DummyVisionConfig:
spatial_merge_size: int = 1
@dataclass
class DummyConfig:
image_token_id: int = IMAGE_TOKEN_ID
video_token_id: int = VIDEO_TOKEN_ID
vision_start_token_id: int = VISION_START_TOKEN_ID
vision_end_token_id: int = VISION_END_TOKEN_ID
vision_config: DummyVisionConfig = dataclasses.field(
default_factory=DummyVisionConfig
)
def make_video_embedding(
t, h, w, interleave_text_tokens: tuple[int, int], video_pruning_rate: float = 0.0
):
"""Make a video embedding for a given video size and pruning rate.
Args:
t: Number of frames.
h: Number of rows.
w: Number of columns.
interleave_text_tokens: Tuple of minimum and maximum number of text tokens to
interleave with the video.
video_pruning_rate: Pruning rate for the video.
Returns:
Tuple of (unpruned_tokens_sequence, pruned_tokens_sequence, retention_mask)
"""
unpruned_tokens_sequence = []
population = list(range(1, 100))
for _ in range(t):
num_prefix_tokens = random.randint(
interleave_text_tokens[0], interleave_text_tokens[1]
)
prefix_tokens = random.choices(population, k=num_prefix_tokens)
vision_tokens = (
[VISION_START_TOKEN_ID] + [VIDEO_TOKEN_ID] * h * w + [VISION_END_TOKEN_ID]
)
unpruned_tokens_sequence.extend(prefix_tokens)
unpruned_tokens_sequence.extend(vision_tokens)
unpruned_tokens_sequence = torch.tensor(unpruned_tokens_sequence, dtype=torch.long)
video_token_mask = unpruned_tokens_sequence == VIDEO_TOKEN_ID
pruning_mask = torch.bernoulli(video_token_mask.float() * video_pruning_rate).bool() # type: ignore[attr-defined]
# Sanity check that we don't prune what should not be pruned.
assert not pruning_mask[~video_token_mask].any()
retention_mask = ~pruning_mask
pruned_tokens_sequence = unpruned_tokens_sequence[retention_mask]
return unpruned_tokens_sequence, pruned_tokens_sequence, retention_mask
@pytest.mark.parametrize("spatial_merge_size", [1, 2])
@pytest.mark.parametrize("grid_thw", [[3, 8, 7], [128, 10, 12]])
@pytest.mark.parametrize("num_prefix_tokens", [1, 11])
@pytest.mark.parametrize("num_suffix_tokens", [0, 7])
@pytest.mark.parametrize("video_pruning_rate", [0, 0.25, 0.75])
@pytest.mark.parametrize("interleave_text_tokens", [(0, 0), (1, 4)])
def test_match_qwen3vl_mrope_evs_on(
spatial_merge_size: int,
num_prefix_tokens: int,
grid_thw: tuple[int, int, int],
num_suffix_tokens: int,
video_pruning_rate: float,
interleave_text_tokens: tuple[int, int],
):
hf_config = DummyConfig()
hf_config.vision_config.spatial_merge_size = spatial_merge_size
t, h, w = grid_thw
population = list(range(1, 100))
prefix_tokens = random.choices(population, k=num_prefix_tokens)
suffix_tokens = random.choices(population, k=num_suffix_tokens)
video_tokens, video_tokens_pruned, retention_mask = make_video_embedding(
t,
h // spatial_merge_size,
w // spatial_merge_size,
interleave_text_tokens=interleave_text_tokens,
video_pruning_rate=video_pruning_rate,
)
assert len(video_tokens) == len(retention_mask)
input_tokens = prefix_tokens + video_tokens.tolist() + suffix_tokens
input_tokens_pruned = prefix_tokens + video_tokens_pruned.tolist() + suffix_tokens
whole_sequence_retention_mask = torch.cat(
[
torch.ones(len(prefix_tokens), dtype=torch.bool),
retention_mask,
torch.ones(len(suffix_tokens), dtype=torch.bool),
],
dim=0,
)
# Build the GT mrope for unpruned input.
mm_feature = MultiModalFeatureSpec(
data=MultiModalKwargsItem(
{
"video_grid_thw": MultiModalFieldElem(
data=torch.tensor(grid_thw),
field=None, # HACK.
),
}
),
modality="video",
identifier="DUMMY",
mm_position=PlaceholderRange(offset=0, length=len(input_tokens)),
)
expected_mrope, _ = Qwen3VLForConditionalGeneration._get_mrope_input_positions(
input_tokens=input_tokens,
mm_features=[mm_feature],
config=hf_config,
)
# Compute mrope for a video-only media (unpruned).
mm_feature = MultiModalFeatureSpec(
data=MultiModalKwargsItem(
{
"video_grid_thw": MultiModalFieldElem(
data=torch.tensor(grid_thw),
field=None, # HACK.
),
}
),
modality="video",
identifier="DUMMY",
mm_position=PlaceholderRange(offset=0, length=video_tokens.numel()),
)
video_mrope, _ = Qwen3VLForConditionalGeneration._get_mrope_input_positions(
input_tokens=video_tokens.tolist(),
mm_features=[mm_feature],
config=hf_config,
)
video_mrope = video_mrope.permute(1, 0) # [N, 3]
hidden_size = 16
is_video_embed = torch.isin(
video_tokens_pruned, torch.tensor([VIDEO_TOKEN_ID], dtype=torch.long)
)
expanded_positions = torch.full(
(len(video_tokens_pruned), 5),
fill_value=-100,
device=video_mrope.device,
dtype=torch.long,
)
expanded_positions[is_video_embed, :3] = video_mrope[retention_mask][is_video_embed]
expanded_positions[~is_video_embed, :3] = video_mrope[retention_mask][
~is_video_embed
]
is_vision_start = video_tokens_pruned == VISION_START_TOKEN_ID
expanded_positions[..., 3] = is_vision_start
expanded_positions[..., 4] = is_video_embed
# Check that all positions were filled, since we initialized them as negative.
assert (expanded_positions >= 0).all()
video_embeddings = torch.empty(
(len(video_tokens_pruned), hidden_size), device=video_mrope.device
)
video_embeddings = torch.cat(
[
video_embeddings,
expanded_positions.float(),
],
dim=1,
)
multimodal_embeddings = [video_embeddings]
expected_mrope_masked = expected_mrope[:, whole_sequence_retention_mask]
# Initialize computed_mrope with sequential positions for all prefix tokens
computed_mrope = torch.empty((3, len(input_tokens_pruned)), dtype=torch.long)
computed_mrope[:, 0 : len(prefix_tokens)] = expected_mrope[
:, 0 : len(prefix_tokens)
]
# Paranoia check that computed_mrope is wrong.
assert not torch.equal(computed_mrope, expected_mrope_masked)
_, actual_mrope, _ = Qwen3VLForConditionalGeneration._recompute_mrope_positions(
input_ids=input_tokens_pruned,
multimodal_embeddings=multimodal_embeddings,
mrope_positions=computed_mrope,
num_computed_tokens=len(prefix_tokens),
vision_start_token_id=hf_config.vision_start_token_id,
image_token_id=hf_config.image_token_id,
video_token_id=hf_config.video_token_id,
)
assert torch.equal(actual_mrope, expected_mrope_masked)
@@ -1,124 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from vllm.model_executor.warmup.qwen_vl_triton_warmup import (
_warm_mrope,
_warm_vision,
)
def test_vision_warmup_calls_only_position_and_rotary_paths() -> None:
from vllm.model_executor.models.qwen3_vl import Qwen3_VisionTransformer
calls: list[tuple[str, object]] = []
class FakeAttention:
num_attention_heads_per_partition = 4
hidden_size_per_attention_head = 8
def apply_rotary_emb(self, qk, cos, sin):
calls.append(("rotary", qk.shape))
class FakeVisual(Qwen3_VisionTransformer):
spatial_merge_size = 2
def __init__(self) -> None:
torch.nn.Module.__init__(self)
self.blocks = [SimpleNamespace(attn=FakeAttention())]
self.pos_embed = torch.nn.Embedding(4, 4, device="cpu")
def fast_pos_embed_interpolate(self, grid_thw):
calls.append(("position", grid_thw[0]))
def rot_pos_emb(self, grid_thw):
_, h, w = grid_thw[0]
shape = (h * w, 4)
return torch.empty(shape), torch.empty(shape)
def forward(self, *args, **kwargs):
raise AssertionError("vision warmup must not run the full tower")
model = torch.nn.Module()
model.visual = FakeVisual()
_warm_vision(model, torch.device("cpu"))
grids = [(1, 16, 16), (1, 16, 2), (1, 2, 16), (1, 2, 2)]
assert [value for name, value in calls if name == "position"] == [
list(grid) for grid in grids
]
assert [value for name, value in calls if name == "rotary"] == [
torch.Size((2, h * w, 4, 8)) for _, h, w in grids
]
# CPU offloading leaves tower weights off the accelerator, so the Triton
# paths must not be launched on host pointers.
calls.clear()
_warm_vision(model, torch.device("cuda"))
assert calls == []
@pytest.mark.parametrize(
"owners_fused, skipped", [((), False), ((True,), True), ((True, False), False)]
)
def test_mrope_warmup_launches_triton_shapes(owners_fused, skipped) -> None:
from vllm.model_executor.layers.rotary_embedding.mrope import MRotaryEmbedding
launched: list[tuple[torch.Size, torch.Size]] = []
class FakeRope(MRotaryEmbedding):
def __init__(self) -> None:
torch.nn.Module.__init__(self)
self.head_size = 8
self.rotary_dim = 8
self.mrope_section = [2, 3, 3]
self.mrope_interleaved = False
self.is_neox_style = True
def forward(self, positions, query, key):
launched.append((positions.shape, query.shape))
return query, key
class FakeMropeModel(torch.nn.Module):
supports_mrope = True
def __init__(self) -> None:
super().__init__()
rope = FakeRope()
if not owners_fused:
self.rotary_emb = rope
for i, fused in enumerate(owners_fused):
owner = torch.nn.Module()
owner.rotary_emb, owner.use_fused_qk_norm_rope_gate = rope, fused
self.add_module(f"attn{i}", owner)
def get_mrope_input_positions(self, input_tokens, mm_features):
raise AssertionError("warmup must not build live M-RoPE positions")
runner = SimpleNamespace(
model_config=SimpleNamespace(
get_num_attention_heads=lambda parallel_config: 4,
get_num_kv_heads=lambda parallel_config: 2,
),
parallel_config=object(),
dtype=torch.bfloat16,
device=torch.device("cpu"),
)
_warm_mrope(runner, FakeMropeModel())
expected = [torch.Size((3, n)) for n in (1, 2, 16)]
assert [shape for shape, _ in launched] == ([] if skipped else expected)
def test_mrope_warmup_skips_models_without_supports_mrope() -> None:
def fail(*_args, **_kwargs):
raise AssertionError("M-RoPE warmup must not run without SupportsMRoPE")
runner = SimpleNamespace(
model_config=SimpleNamespace(get_num_attention_heads=fail),
get_model=fail,
)
_warm_mrope(runner, torch.nn.Module())
@@ -1,339 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from functools import partial
from pathlib import Path
import pytest
import torch
from transformers import Gemma3nTextConfig
from vllm import LLM, SamplingParams
from vllm.inputs import TokensPrompt
from vllm.platforms import current_platform
from ..models.utils import dummy_hf_overrides
from ..utils import create_new_process_for_each_test
@dataclass(frozen=True)
class SleepModelCase:
name: str
architecture: str
model: str | None
tensor_names: tuple[str, ...]
modalities: tuple[str, ...] = ()
revision: str | None = None
trust_remote_code: bool = False
tokenizer_mode: str | None = None
SLEEP_MODEL_CASES = (
SleepModelCase(
name="fireredasr2",
architecture="FireRedASR2ForConditionalGeneration",
model="allendou/FireRedASR2-LLM-vllm",
tensor_names=("pe",),
trust_remote_code=True,
),
SleepModelCase(
name="gemma3n",
architecture="Gemma3nForCausalLM",
model=None,
tensor_names=(
"router_input_scale",
"embed_scale",
"embed_scale_per_layer",
"per_layer_input_scale",
"per_layer_projection_scale",
),
),
SleepModelCase(
name="voxtral",
architecture="VoxtralForConditionalGeneration",
model="mistralai/Voxtral-Mini-3B-2507",
tensor_names=("mel_filters",),
revision="3060fe34b35ba5d44202ce9ff3c097642914f8f3",
tokenizer_mode="mistral",
),
SleepModelCase(
name="ernie45_vl",
architecture="Ernie4_5_VLMoeForConditionalGeneration",
model="baidu/ERNIE-4.5-VL-28B-A3B-PT",
tensor_names=("inv_freq", "_visual_token_ids_tensor_cache"),
revision="refs/pr/17",
trust_remote_code=True,
),
)
def _create_local_model_config(case: SleepModelCase, model_dir: Path) -> str:
if case.name == "gemma3n":
config = Gemma3nTextConfig(
architectures=[case.architecture],
vocab_size=128,
hidden_size=64,
intermediate_size=[128, 128, 128],
num_hidden_layers=3,
num_attention_heads=2,
num_key_value_heads=2,
head_dim=32,
max_position_embeddings=256,
sliding_window=64,
layer_types=[
"sliding_attention",
"full_attention",
"sliding_attention",
],
vocab_size_per_layer_input=128,
hidden_size_per_layer_input=16,
num_kv_shared_layers=1,
laurel_rank=16,
activation_sparsity_pattern=[0.0, 0.0, 0.0],
)
else:
raise AssertionError(f"No local config builder for {case.name}")
model_dir.mkdir()
config.save_pretrained(model_dir)
return str(model_dir)
def _sleep_test_hf_overrides(hf_config, *, model_arch: str):
hf_config = dummy_hf_overrides(hf_config, model_arch=model_arch)
if model_arch == "FireRedASR2ForConditionalGeneration":
audio_encoder_conf = getattr(hf_config, "audio_encoder_conf", {})
audio_encoder_conf.update(
{
"idim": 80,
"n_layers_enc": 1,
"n_head": 1,
"d_model": 32,
"kernel_size": 3,
# MM profiling uses the repository's real feature extractor
# and can produce roughly 750 encoder frames. Keep the real
# positional-encoding capacity instead of shrinking it with
# the number of test decoder tokens.
"pe_maxlen": 5000,
}
)
hf_config.audio_encoder_conf = audio_encoder_conf
if (
model_arch == "Ernie4_5_VLMoeForConditionalGeneration"
and getattr(hf_config, "im_patch_id", None) is None
):
# Some revisions expose this id as image_token_id, while the native
# model intentionally caches it under the older im_patch_id name.
hf_config.im_patch_id = getattr(hf_config, "image_token_id", 100295)
return hf_config
def _get_sleep_tensor_targets(
model_name: str,
) -> tuple[tuple[type[torch.nn.Module], tuple[str, ...]], ...]:
if model_name == "fireredasr2":
from vllm.model_executor.models.conformer_encoder import (
RelPositionalEncoding,
)
return ((RelPositionalEncoding, ("pe",)),)
if model_name == "gemma3n":
from vllm.model_executor.models.gemma3n import (
Gemma3nAltUp,
Gemma3nSelfDecoder,
)
return (
(Gemma3nAltUp, ("router_input_scale",)),
(
Gemma3nSelfDecoder,
(
"embed_scale",
"embed_scale_per_layer",
"per_layer_input_scale",
"per_layer_projection_scale",
),
),
)
if model_name == "voxtral":
from vllm.model_executor.models.voxtral import VoxtralEncoderModel
return ((VoxtralEncoderModel, ("mel_filters",)),)
if model_name == "ernie45_vl":
from vllm.model_executor.models.ernie45_vl import (
Ernie4_5_VisionRotaryEmbedding,
Ernie4_5_VLMoeForConditionalGeneration,
)
return (
(Ernie4_5_VisionRotaryEmbedding, ("inv_freq",)),
(
Ernie4_5_VLMoeForConditionalGeneration,
("_visual_token_ids_tensor_cache",),
),
)
raise AssertionError(f"No sleep tensor targets for {model_name}")
def _snapshot_sleep_tensors(
model: torch.nn.Module,
*,
model_name: str,
tensor_names: tuple[str, ...],
) -> dict[str, tuple[int, torch.Tensor]]:
targets = _get_sleep_tensor_targets(model_name)
snapshot: dict[str, tuple[int, torch.Tensor]] = {}
found_tensor_names = set[str]()
for module_path, module in model.named_modules():
for module_type, target_names in targets:
if not isinstance(module, module_type):
continue
for tensor_name in target_names:
tensor = getattr(module, tensor_name, None)
if tensor is None:
continue
assert isinstance(tensor, torch.Tensor)
qualified_name = (
f"{module_path}.{tensor_name}" if module_path else tensor_name
)
snapshot[qualified_name] = (
tensor.data_ptr(),
tensor.detach().cpu().clone(),
)
found_tensor_names.add(tensor_name)
assert found_tensor_names == set(tensor_names), (
f"{model_name}: expected tensors {tensor_names}, "
f"found {tuple(sorted(found_tensor_names))}"
)
return snapshot
def _save_dummy_weights(worker) -> None:
model = worker.model_runner.model
worker._sleep_test_dummy_weights = [
(name, parameter.detach().cpu().clone())
for name, parameter in model.named_parameters()
]
def _reload_dummy_weights(worker) -> None:
with torch.no_grad():
worker.reload_weights(
weights_iterator=iter(worker._sleep_test_dummy_weights),
is_checkpoint_format=False,
)
def _generation_signature(output) -> tuple[tuple[int, ...], tuple[float, ...]]:
completion = output[0].outputs[0]
token_ids = tuple(completion.token_ids)
assert completion.logprobs is not None
chosen_logprobs = tuple(
step[token_id].logprob
for token_id, step in zip(token_ids, completion.logprobs, strict=True)
)
return token_ids, chosen_logprobs
def _assert_tensor_snapshots_equal(
before: dict[str, tuple[int, torch.Tensor]],
after: dict[str, tuple[int, torch.Tensor]],
model_name: str,
) -> None:
assert after.keys() == before.keys(), model_name
for name, (before_ptr, before_value) in before.items():
after_ptr, after_value = after[name]
assert after_ptr == before_ptr, f"{model_name}: {name} address changed"
torch.testing.assert_close(
after_value,
before_value,
msg=lambda msg, name=name: (f"{model_name}: {name} was corrupted: {msg}"),
)
@create_new_process_for_each_test("spawn")
@pytest.mark.slow_test
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA sleep mode test")
@pytest.mark.parametrize("case", SLEEP_MODEL_CASES, ids=lambda case: case.name)
def test_static_model_tensors_survive_level2_restore(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
case: SleepModelCase,
) -> None:
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
if case.name == "fireredasr2":
# FireRedASR2 has a regular decoder and compute_logits implementation,
# but advertises only the transcription task. Enable its text path so
# this ownership test can exercise the common LLM.generate lifecycle
# without constructing an unrelated audio fixture.
from vllm.model_executor.models.fireredasr2 import (
FireRedASR2ForConditionalGeneration,
)
monkeypatch.setattr(
FireRedASR2ForConditionalGeneration,
"supports_transcription_only",
False,
)
model = case.model or _create_local_model_config(case, tmp_path / case.name)
llm_kwargs = {
"model": model,
"load_format": "dummy",
"enable_sleep_mode": True,
"enforce_eager": True,
"attention_backend": "TRITON_ATTN",
"max_model_len": 128,
"max_num_seqs": 1,
"gpu_memory_utilization": 0.5,
"seed": 0,
"enable_prefix_caching": False,
"disable_log_stats": True,
"trust_remote_code": case.trust_remote_code,
}
if case.revision is not None:
llm_kwargs["revision"] = case.revision
if case.name not in ("fireredasr2", "voxtral", "ernie45_vl"):
llm_kwargs["skip_tokenizer_init"] = True
if case.tokenizer_mode is not None:
llm_kwargs["tokenizer_mode"] = case.tokenizer_mode
if case.model is not None:
llm_kwargs["hf_overrides"] = partial(
_sleep_test_hf_overrides, model_arch=case.architecture
)
if case.modalities:
llm_kwargs["limit_mm_per_prompt"] = {
modality: 0 for modality in case.modalities
}
llm = LLM(**llm_kwargs)
prompt = TokensPrompt(prompt_token_ids=[2, 5, 9, 12, 7, 3])
sampling_params = SamplingParams(temperature=0.0, max_tokens=8, logprobs=1)
before_generation = _generation_signature(llm.generate(prompt, sampling_params))
snapshot_fn = partial(
_snapshot_sleep_tensors,
model_name=case.name,
tensor_names=case.tensor_names,
)
(before_tensors,) = llm.apply_model(snapshot_fn)
llm.collective_rpc(_save_dummy_weights)
llm.sleep(level=2)
llm.wake_up(tags=["weights"])
(after_tensors,) = llm.apply_model(snapshot_fn)
_assert_tensor_snapshots_equal(before_tensors, after_tensors, case.name)
llm.collective_rpc(_reload_dummy_weights)
llm.wake_up(tags=["kv_cache"])
after_generation = _generation_signature(llm.generate(prompt, sampling_params))
assert after_generation[0] == before_generation[0], case.name
assert after_generation[1] == pytest.approx(
before_generation[1], rel=1e-5, abs=1e-5
), case.name