[Mini-vLLM] SM86-only cut: models, backends, Rust, tests
- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries - Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels - Remove inc/quark quant loaders; keep compressed-tensors and Marlin - Drop the Rust frontend, pooling entrypoints, and idle test suites - Rebuild kernels for sm_80;sm_86 and strip the extensions - Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,337 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from safetensors.torch import save_file
|
||||
from transformers import DeepseekV2Config, DeepseekV2ForCausalLM
|
||||
|
||||
from vllm.config import (
|
||||
AttentionConfig,
|
||||
DeviceConfig,
|
||||
KernelConfig,
|
||||
LoadConfig,
|
||||
ModelConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.expert_substitution import (
|
||||
ConstantExpertSubstitution,
|
||||
)
|
||||
from vllm.model_executor.model_loader import get_model_loader
|
||||
from vllm.model_executor.models.expert_substitution import as_expert_substitution_model
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader, PPMissingLayer
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
MODEL_CONFIG = SimpleNamespace(
|
||||
hf_config=SimpleNamespace(approximate_experts={"0": [1, 2], "1": [1, 2]})
|
||||
)
|
||||
|
||||
|
||||
def _compression_config(value_names: dict[int, dict[int, str]]) -> dict:
|
||||
"""Explicitly named constants given as ``{layer: {expert: tensor name}}``."""
|
||||
targets = {
|
||||
f"model.layers.{layer}.mlp.experts": {
|
||||
"num_logical_experts": 3,
|
||||
"weight_layout": "compact_retained_experts",
|
||||
"replacements": {
|
||||
str(expert): {"format": "constant-v1", "tensors": {"value": name}}
|
||||
for expert, name in names.items()
|
||||
},
|
||||
}
|
||||
for layer, names in value_names.items()
|
||||
}
|
||||
router_semantics = {
|
||||
"preserve_logical_expert_ids": True,
|
||||
"preserve_router_weights": True,
|
||||
"renormalize_after_substitution": False,
|
||||
}
|
||||
return {
|
||||
"transform_config": {
|
||||
"expert_substitution": {
|
||||
"version": 1,
|
||||
"router_semantics": router_semantics,
|
||||
"targets": targets,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
# Expert 1 of both layers shares one explicitly named constant.
|
||||
EXPLICIT_MODEL_CONFIG = SimpleNamespace(
|
||||
hf_config=SimpleNamespace(
|
||||
compression_config=_compression_config(
|
||||
{
|
||||
layer: {1: "constants.shared", 2: f"constants.layer_{layer}"}
|
||||
for layer in (0, 1)
|
||||
}
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _approx_value(layer: int, expert: int) -> str:
|
||||
return f"model.layers.{layer}.mlp.experts.{expert}.approx_value"
|
||||
|
||||
|
||||
def _cpu_model(local_layers=(0, 1), tracked=True, model_config=MODEL_CONFIG):
|
||||
"""Two decoder layers; the others are placeholders as on another PP stage."""
|
||||
|
||||
class Model(torch.nn.Module):
|
||||
def __init__(self, prefix=""):
|
||||
super().__init__()
|
||||
self.weight = torch.nn.Parameter(torch.zeros(2), requires_grad=False)
|
||||
self.model = torch.nn.Module()
|
||||
self.model.layers = torch.nn.ModuleList()
|
||||
for index in range(2):
|
||||
if index not in local_layers:
|
||||
self.model.layers.append(PPMissingLayer())
|
||||
continue
|
||||
layer = torch.nn.Module()
|
||||
layer.mlp = torch.nn.Module()
|
||||
layer.mlp.experts = ConstantExpertSubstitution(
|
||||
index, 3, (1, 2), 2, torch.float32
|
||||
)
|
||||
self.model.layers.append(layer)
|
||||
|
||||
def load_weights(self, weights):
|
||||
loaded = AutoWeightsLoader(self).load_weights(weights)
|
||||
return loaded if tracked else None
|
||||
|
||||
return as_expert_substitution_model(Model, model_config)()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("local_layers", [(0, 1), (0,), (1,), ()])
|
||||
def test_loads_local_approx_values_and_passes_through_others(local_layers):
|
||||
"""Other PP stages and MTP drafts own none or some of the configured layers."""
|
||||
model = _cpu_model(local_layers)
|
||||
loaded = model.load_weights(
|
||||
[
|
||||
("weight", torch.tensor([1.0, 2.0])),
|
||||
*(
|
||||
(_approx_value(layer, expert), torch.full((2,), 10.0 * layer + expert))
|
||||
for layer in range(2)
|
||||
for expert in (1, 2)
|
||||
),
|
||||
]
|
||||
)
|
||||
model.process_weights_after_loading()
|
||||
|
||||
assert loaded == {"weight"} | {
|
||||
f"model.layers.{layer}.mlp.experts.values" for layer in local_layers
|
||||
}
|
||||
for layer in local_layers:
|
||||
torch.testing.assert_close(
|
||||
model.model.layers[layer].mlp.experts.values,
|
||||
torch.tensor([[1.0, 1.0], [2.0, 2.0]]) + 10.0 * layer,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("local_layers", [(0, 1), (1,), ()])
|
||||
def test_loads_explicitly_named_values(local_layers):
|
||||
"""Named values are consumed even when their layer is built elsewhere."""
|
||||
model = _cpu_model(local_layers, model_config=EXPLICIT_MODEL_CONFIG)
|
||||
loaded = model.load_weights(
|
||||
[
|
||||
("weight", torch.tensor([1.0, 2.0])),
|
||||
("constants.shared", torch.full((2,), 1.0)),
|
||||
("constants.layer_0", torch.full((2,), 2.0)),
|
||||
("constants.layer_1", torch.full((2,), 3.0)),
|
||||
]
|
||||
)
|
||||
model.process_weights_after_loading()
|
||||
|
||||
assert loaded == {"weight"} | {
|
||||
f"model.layers.{layer}.mlp.experts.values" for layer in local_layers
|
||||
}
|
||||
for layer in local_layers:
|
||||
torch.testing.assert_close(
|
||||
model.model.layers[layer].mlp.experts.values,
|
||||
torch.tensor([[1.0, 1.0], [2.0 + layer, 2.0 + layer]]),
|
||||
)
|
||||
|
||||
|
||||
def test_direct_reload_updates_values_incrementally():
|
||||
model = _cpu_model(local_layers=(0,), tracked=False)
|
||||
values = model.model.layers[0].mlp.experts.values
|
||||
model.load_weights([(_approx_value(0, 2), torch.ones(2))])
|
||||
with pytest.raises(ValueError, match="missing approx_value .* \\[1\\]"):
|
||||
model.process_weights_after_loading()
|
||||
|
||||
assert model.load_weights([(_approx_value(0, 1), torch.zeros(2))]) is None
|
||||
model.process_weights_after_loading()
|
||||
model.load_weights([(_approx_value(0, 2), torch.full((2,), 3.0))])
|
||||
torch.testing.assert_close(values, torch.tensor([[0.0, 0.0], [3.0, 3.0]]))
|
||||
|
||||
with pytest.raises(ValueError, match="not listed"):
|
||||
model.load_weights([(_approx_value(0, 0), torch.ones(2))])
|
||||
|
||||
|
||||
def test_layerwise_reload_is_rejected_and_keeps_weights():
|
||||
from vllm.model_executor.model_loader.reload import (
|
||||
initialize_layerwise_reload,
|
||||
record_metadata_for_reloading,
|
||||
)
|
||||
|
||||
model = _cpu_model(local_layers=(0,))
|
||||
record_metadata_for_reloading(model)
|
||||
loaded = [
|
||||
("weight", torch.tensor([3.0, 4.0])),
|
||||
(_approx_value(0, 1), torch.zeros(2)),
|
||||
(_approx_value(0, 2), torch.ones(2)),
|
||||
]
|
||||
model.load_weights(loaded)
|
||||
model.process_weights_after_loading()
|
||||
substitution = model.model.layers[0].mlp.experts
|
||||
values = substitution.values
|
||||
|
||||
initialize_layerwise_reload(model)
|
||||
with pytest.raises(NotImplementedError, match="layerwise weight reload"):
|
||||
model.load_weights([(name, weight + 1) for name, weight in loaded])
|
||||
|
||||
assert substitution.values is values
|
||||
torch.testing.assert_close(values, torch.tensor([[0.0, 0.0], [1.0, 1.0]]))
|
||||
torch.testing.assert_close(model.weight, torch.tensor([3.0, 4.0]))
|
||||
assert substitution.expert_substitution_routes.tolist() == [0, -1, -2]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("missing_value", [False, True])
|
||||
def test_tensorizer_finalizes_local_substitution_values(monkeypatch, missing_value):
|
||||
"""Raw Tensorizer loading must reject an unloaded local constant row."""
|
||||
from vllm.model_executor.model_loader import tensorizer_loader
|
||||
|
||||
model = _cpu_model(local_layers=(0,))
|
||||
weights = [(_approx_value(0, 1), torch.zeros(2))]
|
||||
if not missing_value:
|
||||
weights.append((_approx_value(0, 2), torch.ones(2)))
|
||||
finalize = Mock(wraps=model.process_weights_after_loading)
|
||||
monkeypatch.setattr(model, "process_weights_after_loading", finalize)
|
||||
monkeypatch.setattr(tensorizer_loader, "initialize_model", lambda **kwargs: model)
|
||||
loader = object.__new__(tensorizer_loader.TensorizerLoader)
|
||||
monkeypatch.setattr(loader, "_get_weights_iterator", lambda: iter(weights))
|
||||
config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(dtype=torch.float32),
|
||||
device_config=SimpleNamespace(device="cpu"),
|
||||
)
|
||||
|
||||
if missing_value:
|
||||
with pytest.raises(ValueError, match=r"missing approx_value .* \[2\]"):
|
||||
loader._load_model_serialized_cpu(config)
|
||||
else:
|
||||
assert loader._load_model_serialized_cpu(config) is model
|
||||
torch.testing.assert_close(
|
||||
model.model.layers[0].mlp.experts.values,
|
||||
torch.tensor([[0.0, 0.0], [1.0, 1.0]]),
|
||||
)
|
||||
finalize.assert_called_once_with()
|
||||
|
||||
|
||||
def _write_tiny_deepseek_mone_checkpoint(
|
||||
model_dir: Path, explicit_names: bool
|
||||
) -> tuple[dict[str, torch.Tensor], str]:
|
||||
config = DeepseekV2Config(
|
||||
architectures=["DeepseekV2ForCausalLM"],
|
||||
vocab_size=32,
|
||||
hidden_size=16,
|
||||
intermediate_size=32,
|
||||
moe_intermediate_size=8,
|
||||
num_hidden_layers=1,
|
||||
num_attention_heads=2,
|
||||
num_key_value_heads=2,
|
||||
max_position_embeddings=32,
|
||||
first_k_dense_replace=0,
|
||||
n_routed_experts=3,
|
||||
n_shared_experts=1,
|
||||
num_experts_per_tok=1,
|
||||
n_group=1,
|
||||
topk_group=1,
|
||||
norm_topk_prob=False,
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=4,
|
||||
qk_nope_head_dim=128,
|
||||
qk_rope_head_dim=64,
|
||||
v_head_dim=128,
|
||||
tie_word_embeddings=False,
|
||||
dtype="float16",
|
||||
)
|
||||
experts = "model.layers.0.mlp.experts"
|
||||
if explicit_names:
|
||||
value_name = "model.layers.0.mlp.expert_replacements.1.value"
|
||||
config.compression_config = _compression_config({0: {1: value_name}})
|
||||
else:
|
||||
value_name = f"{experts}.1.approx_value"
|
||||
config.approximate_experts = {"0": [1]}
|
||||
|
||||
hf_weights = DeepseekV2ForCausalLM(config).to(torch.float16).state_dict()
|
||||
gate_up = hf_weights.pop(f"{experts}.gate_up_proj")
|
||||
down = hf_weights.pop(f"{experts}.down_proj")
|
||||
weights = {name: value.contiguous() for name, value in hf_weights.items()}
|
||||
for expert in (0, 2):
|
||||
gate, up = gate_up[expert].chunk(2, dim=0)
|
||||
weights[f"{experts}.{expert}.gate_proj.weight"] = gate.contiguous()
|
||||
weights[f"{experts}.{expert}.up_proj.weight"] = up.contiguous()
|
||||
weights[f"{experts}.{expert}.down_proj.weight"] = down[expert].contiguous()
|
||||
weights[value_name] = torch.arange(16, dtype=torch.float16)
|
||||
config.save_pretrained(model_dir)
|
||||
save_file(weights, model_dir / "model.safetensors")
|
||||
return weights, value_name
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like device"
|
||||
)
|
||||
@pytest.mark.usefixtures("dist_init", "workspace_init")
|
||||
@pytest.mark.parametrize(
|
||||
"explicit_names", [False, True], ids=["approximate_experts", "expert_substitution"]
|
||||
)
|
||||
def test_loads_mone_checkpoint_into_compact_experts(
|
||||
tmp_path: Path, explicit_names: bool
|
||||
):
|
||||
weights, value_name = _write_tiny_deepseek_mone_checkpoint(tmp_path, explicit_names)
|
||||
model_config = ModelConfig(
|
||||
model=str(tmp_path),
|
||||
tokenizer=str(tmp_path),
|
||||
skip_tokenizer_init=True,
|
||||
dtype="float16",
|
||||
max_model_len=32,
|
||||
enforce_eager=True,
|
||||
)
|
||||
load_config = LoadConfig(load_format="safetensors", use_tqdm_on_load=False)
|
||||
vllm_config = VllmConfig(
|
||||
model_config=model_config,
|
||||
device_config=DeviceConfig(device="cuda"),
|
||||
load_config=load_config,
|
||||
attention_config=AttentionConfig(backend=AttentionBackendEnum.TRITON_MLA),
|
||||
kernel_config=KernelConfig(moe_backend="triton"),
|
||||
)
|
||||
model = get_model_loader(load_config).load_model(vllm_config, model_config)
|
||||
|
||||
experts = "model.layers.0.mlp.experts"
|
||||
routed_experts = model.model.layers[0].mlp.experts.routed_experts
|
||||
# Logical expert 2 is stored in physical row 1.
|
||||
torch.testing.assert_close(
|
||||
routed_experts.w13_weight[1].cpu(),
|
||||
torch.cat(
|
||||
[
|
||||
weights[f"{experts}.2.gate_proj.weight"],
|
||||
weights[f"{experts}.2.up_proj.weight"],
|
||||
]
|
||||
),
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
routed_experts.w2_weight[1].cpu(), weights[f"{experts}.2.down_proj.weight"]
|
||||
)
|
||||
substitution = routed_experts.expert_substitution
|
||||
torch.testing.assert_close(
|
||||
substitution.values.cpu(), weights[value_name].unsqueeze(0)
|
||||
)
|
||||
|
||||
updated = weights[value_name] + 1
|
||||
assert model.load_weights([(value_name, updated)]) == {
|
||||
f"{experts}.routed_experts.expert_substitution.values"
|
||||
}
|
||||
torch.testing.assert_close(substitution.values.cpu(), updated.unsqueeze(0))
|
||||
@@ -1,152 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.gpt_oss import (
|
||||
GptOssForCausalLM,
|
||||
GptOssModel,
|
||||
GptOssRoutedExperts,
|
||||
)
|
||||
|
||||
|
||||
class _TestExperts(GptOssRoutedExperts):
|
||||
def __init__(
|
||||
self,
|
||||
param_name: str,
|
||||
shape: tuple[int, ...],
|
||||
*,
|
||||
tp_rank: int = 0,
|
||||
quant_method_name: str = "TestQuantMethod",
|
||||
weight_dtype: str = "",
|
||||
dtype: torch.dtype = torch.float32,
|
||||
) -> None:
|
||||
torch.nn.Module.__init__(self)
|
||||
self.moe_config = SimpleNamespace(
|
||||
moe_parallel_config=SimpleNamespace(tp_size=2, tp_rank=tp_rank)
|
||||
)
|
||||
self.quant_method = type(
|
||||
quant_method_name, (), {"weight_dtype": weight_dtype}
|
||||
)()
|
||||
|
||||
param = torch.nn.Parameter(torch.zeros(shape, dtype=dtype), requires_grad=False)
|
||||
param.weight_loader = self.weight_loader
|
||||
self.register_parameter(param_name, param)
|
||||
|
||||
@staticmethod
|
||||
def _map_global_expert_id_to_local_expert_id(expert_id: int) -> int:
|
||||
return {3: 0, 7: 1}.get(expert_id, -1)
|
||||
|
||||
|
||||
def _load(
|
||||
experts: _TestExperts,
|
||||
param_name: str,
|
||||
loaded_weight: torch.Tensor,
|
||||
shard_id: str,
|
||||
expert_id: int = 3,
|
||||
) -> bool:
|
||||
param = getattr(experts, param_name)
|
||||
return param.weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
weight_name=f"model.layers.0.mlp.experts.routed_experts.{param_name}",
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("base_layer_prefix", ["", "base_layer."])
|
||||
def test_streamed_expert_resolves_wrapped_parameter(base_layer_prefix: str) -> None:
|
||||
experts = _TestExperts("w13_weight", (2, 4, 3))
|
||||
checkpoint_name = "model.layers.2.mlp.experts.7.gate_up_proj"
|
||||
mapped_name = GptOssForCausalLM.hf_to_vllm_mapper.apply_list([checkpoint_name])[0]
|
||||
fused_name = (
|
||||
f"model.layers.2.mlp.experts.{base_layer_prefix}routed_experts.w13_weight"
|
||||
)
|
||||
params_dict = {fused_name: experts.w13_weight}
|
||||
loaded_params: set[str] = set()
|
||||
loaded_weight = torch.arange(24, dtype=torch.float32).reshape(3, 8)
|
||||
|
||||
assert GptOssModel._get_streamed_expert_info(mapped_name, params_dict) == (
|
||||
7,
|
||||
fused_name,
|
||||
"gpt_oss_w13",
|
||||
)
|
||||
assert GptOssModel._try_load_streamed_expert(
|
||||
mapped_name, loaded_weight, params_dict, loaded_params
|
||||
)
|
||||
assert loaded_params == {fused_name}
|
||||
assert torch.equal(experts.w13_weight[1], loaded_weight[:, :4].t())
|
||||
assert torch.count_nonzero(experts.w13_weight[0]) == 0
|
||||
|
||||
|
||||
def test_streamed_expert_skips_non_local_expert() -> None:
|
||||
experts = _TestExperts("w2_bias", (2, 3))
|
||||
|
||||
assert not _load(experts, "w2_bias", torch.ones(3), "gpt_oss_w2", expert_id=6)
|
||||
assert torch.count_nonzero(experts.w2_bias) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("param_name", ["w13_weight", "w2_weight"])
|
||||
def test_streamed_unquantized_expert_tp_slice(param_name: str) -> None:
|
||||
if param_name == "w13_weight":
|
||||
experts = _TestExperts(param_name, (2, 4, 3), tp_rank=1)
|
||||
loaded_weight = torch.arange(24, dtype=torch.float32).reshape(3, 8)
|
||||
shard_id = "gpt_oss_w13"
|
||||
expected = loaded_weight[:, 4:].t()
|
||||
else:
|
||||
experts = _TestExperts(param_name, (2, 3, 2), tp_rank=1)
|
||||
loaded_weight = torch.arange(12, dtype=torch.float32).reshape(4, 3)
|
||||
shard_id = "gpt_oss_w2"
|
||||
expected = loaded_weight[2:].t()
|
||||
|
||||
assert _load(experts, param_name, loaded_weight, shard_id)
|
||||
assert torch.equal(getattr(experts, param_name)[0], expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_rank", [0, 1])
|
||||
def test_streamed_down_bias_owned_by_tp_rank_zero(tp_rank: int) -> None:
|
||||
experts = _TestExperts("w2_bias", (2, 3), tp_rank=tp_rank)
|
||||
loaded_weight = torch.arange(1, 4, dtype=torch.float32)
|
||||
|
||||
assert _load(experts, "w2_bias", loaded_weight, "gpt_oss_w2")
|
||||
expected = loaded_weight if tp_rank == 0 else torch.zeros_like(loaded_weight)
|
||||
assert torch.equal(experts.w2_bias[0], expected)
|
||||
|
||||
|
||||
def test_streamed_gate_up_bias_tp_slice() -> None:
|
||||
experts = _TestExperts("w13_bias", (2, 4), tp_rank=1)
|
||||
loaded_weight = torch.arange(8, dtype=torch.float32)
|
||||
|
||||
assert _load(experts, "w13_bias", loaded_weight, "gpt_oss_w13")
|
||||
assert torch.equal(experts.w13_bias[0], loaded_weight[4:])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", ["mxfp4", "nvfp4"])
|
||||
def test_streamed_quantized_expert_layout(case: str) -> None:
|
||||
if case == "mxfp4":
|
||||
experts = _TestExperts(
|
||||
"w13_weight",
|
||||
(2, 4, 4),
|
||||
tp_rank=1,
|
||||
weight_dtype="gpt_oss_mxfp4",
|
||||
dtype=torch.uint8,
|
||||
)
|
||||
loaded_weight = torch.arange(32, dtype=torch.uint8).reshape(8, 2, 2)
|
||||
expected = loaded_weight.reshape(8, 4)[4:]
|
||||
else:
|
||||
experts = _TestExperts(
|
||||
"w13_weight",
|
||||
(2, 4, 2),
|
||||
tp_rank=1,
|
||||
quant_method_name="Nvfp4OnlineMoEMethod",
|
||||
dtype=torch.uint8,
|
||||
)
|
||||
loaded_weight = torch.arange(16, dtype=torch.uint8).reshape(8, 2)
|
||||
expected = loaded_weight[4:]
|
||||
|
||||
assert _load(experts, "w13_weight", loaded_weight, "gpt_oss_w13")
|
||||
assert torch.equal(experts.w13_weight[0], expected)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,176 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import copy
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import DeviceConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.rotary_embedding.bailing_mrope import (
|
||||
BailingMRotaryEmbedding,
|
||||
)
|
||||
from vllm.model_executor.models.bailing_moe_v3 import _build_mla_rotary_embedding
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def test_bailing_mrope_selects_alternating_spatial_then_temporal():
|
||||
with set_current_vllm_config(VllmConfig(device_config=DeviceConfig("cpu"))):
|
||||
rotary_emb = BailingMRotaryEmbedding(
|
||||
head_size=12,
|
||||
rotary_dim=12,
|
||||
max_position_embeddings=8,
|
||||
base=10_000,
|
||||
is_neox_style=False,
|
||||
dtype=torch.float32,
|
||||
mrope_section=[2, 2, 2],
|
||||
)
|
||||
|
||||
positions = torch.tensor([[1, 2], [3, 4], [5, 6]])
|
||||
frequencies = torch.arange(6).expand(8, -1)
|
||||
position_values = torch.arange(8).unsqueeze(1) * 10
|
||||
cos = position_values + frequencies
|
||||
sin = cos + 100
|
||||
cos_sin_cache = torch.cat((cos, sin), dim=-1)
|
||||
|
||||
actual_cos, actual_sin = rotary_emb.select_cos_sin(
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
)
|
||||
|
||||
expected_cos = torch.tensor(
|
||||
[
|
||||
[30, 51, 32, 53, 14, 15],
|
||||
[40, 61, 42, 63, 24, 25],
|
||||
]
|
||||
)
|
||||
torch.testing.assert_close(actual_cos, expected_cos)
|
||||
torch.testing.assert_close(actual_sin, expected_cos + 100)
|
||||
|
||||
|
||||
def test_bailing_factory_preserves_mla_width_and_config():
|
||||
config = SimpleNamespace(
|
||||
max_position_embeddings=128,
|
||||
rope_parameters={
|
||||
"rope_type": "default",
|
||||
"rope_theta": 6_000_000,
|
||||
"partial_rotary_factor": 0.5,
|
||||
"mrope_section": [8, 12, 12],
|
||||
},
|
||||
rope_scaling={"partial_rotary_factor": 0.5},
|
||||
)
|
||||
original = copy.deepcopy(vars(config))
|
||||
with set_current_vllm_config(VllmConfig(device_config=DeviceConfig("cpu"))):
|
||||
rope = _build_mla_rotary_embedding(config, head_size=64)
|
||||
assert rope is _build_mla_rotary_embedding(config, head_size=64)
|
||||
generic = get_rope(
|
||||
head_size=64,
|
||||
max_position=128,
|
||||
is_neox_style=False,
|
||||
rope_parameters={
|
||||
"rope_type": "default",
|
||||
"rope_theta": 6_000_000,
|
||||
"mrope_section": [8, 12, 12],
|
||||
},
|
||||
)
|
||||
assert isinstance(rope, BailingMRotaryEmbedding)
|
||||
assert rope is not generic
|
||||
assert rope.rotary_dim == 64
|
||||
assert vars(config) == original
|
||||
|
||||
|
||||
def test_bailing_factory_rejects_unsupported_scaling():
|
||||
config = SimpleNamespace(
|
||||
rope_parameters={
|
||||
"rope_type": "yarn",
|
||||
"factor": 2.0,
|
||||
"mrope_section": [8, 12, 12],
|
||||
}
|
||||
)
|
||||
with pytest.raises(ValueError, match="only supports rope_type='default'"):
|
||||
_build_mla_rotary_embedding(config, head_size=64)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="Skipping CUDA/ROCm-only test.",
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("position_kind", ["text", "multimodal"])
|
||||
@torch.inference_mode()
|
||||
def test_bailing_mrope_cuda_matches_native_for_strided_mla_views(
|
||||
dtype: torch.dtype,
|
||||
position_kind: str,
|
||||
):
|
||||
device = torch.device("cuda")
|
||||
num_tokens = 17
|
||||
num_query_heads = 8
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
rotary_emb = BailingMRotaryEmbedding(
|
||||
head_size=64,
|
||||
rotary_dim=64,
|
||||
max_position_embeddings=64,
|
||||
base=6_000_000,
|
||||
is_neox_style=False,
|
||||
dtype=dtype,
|
||||
mrope_section=[8, 12, 12],
|
||||
).to(device)
|
||||
|
||||
torch.manual_seed(0)
|
||||
multimodal_positions = torch.randint(
|
||||
0,
|
||||
64,
|
||||
(3, 2 * num_tokens),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)[:, ::2]
|
||||
if position_kind == "multimodal":
|
||||
positions = multimodal_positions
|
||||
reference_positions = positions
|
||||
else:
|
||||
positions = torch.randint(
|
||||
0,
|
||||
64,
|
||||
(2 * num_tokens,),
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
)[::2]
|
||||
reference_positions = positions.expand(3, -1)
|
||||
query_buffer = torch.randn(
|
||||
num_tokens,
|
||||
num_query_heads,
|
||||
192,
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
)
|
||||
key_buffer = torch.randn(
|
||||
num_tokens,
|
||||
576,
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
)
|
||||
query = query_buffer[..., 128:]
|
||||
key = key_buffer[:, 512:].unsqueeze(1)
|
||||
assert not query.is_contiguous()
|
||||
assert not key.is_contiguous()
|
||||
|
||||
query_prefix = query_buffer[..., :128].clone()
|
||||
key_prefix = key_buffer[:, :512].clone()
|
||||
expected_query, expected_key = rotary_emb.forward_native(
|
||||
reference_positions,
|
||||
query.clone(),
|
||||
key.clone(),
|
||||
)
|
||||
actual_query, actual_key = rotary_emb.forward_cuda(positions, query, key)
|
||||
assert expected_key is not None
|
||||
assert actual_key is not None
|
||||
|
||||
torch.testing.assert_close(actual_query, expected_query, atol=1e-2, rtol=1.6e-2)
|
||||
torch.testing.assert_close(actual_key, expected_key, atol=1e-2, rtol=1.6e-2)
|
||||
assert torch.equal(query_buffer[..., :128], query_prefix)
|
||||
assert torch.equal(key_buffer[:, :512], key_prefix)
|
||||
if position_kind == "multimodal":
|
||||
assert actual_query.data_ptr() == query.data_ptr()
|
||||
assert actual_key.data_ptr() == key.data_ptr()
|
||||
@@ -1,134 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.deep_gemm import (
|
||||
DeepGemmFp8BlockScaledMMKernel,
|
||||
)
|
||||
from vllm.model_executor.warmup import deep_gemm_warmup
|
||||
|
||||
|
||||
def _block_fp8_layer(n: int, k: int, scale_name: str = "weight_scale"):
|
||||
layer = SimpleNamespace(
|
||||
weight=torch.empty((n, k), dtype=torch.float8_e4m3fn),
|
||||
weight_block_size=[128, 128],
|
||||
deep_gemm_warmup_provider=object.__new__(DeepGemmFp8BlockScaledMMKernel),
|
||||
)
|
||||
setattr(layer, scale_name, torch.empty((n // 128, k // 128), dtype=torch.float32))
|
||||
return layer
|
||||
|
||||
|
||||
def _run_warmup(monkeypatch, layers) -> list[dict]:
|
||||
calls: list[dict] = []
|
||||
monkeypatch.setattr(
|
||||
deep_gemm_warmup,
|
||||
"get_mk_alignment_for_contiguous_layout",
|
||||
lambda: [128, 128],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
deep_gemm_warmup,
|
||||
"_deepgemm_fp8_gemm_nt_warmup",
|
||||
lambda **kwargs: calls.append(kwargs),
|
||||
)
|
||||
model = SimpleNamespace(modules=lambda: iter(layers))
|
||||
deep_gemm_warmup.deepgemm_fp8_gemm_nt_warmup(model, max_tokens=16)
|
||||
return calls
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scale_name", ["weight_scale", "weight_scale_inv"])
|
||||
def test_registered_deep_gemm_layer_is_warmed_up(monkeypatch, scale_name) -> None:
|
||||
"""Any layer stamped by the DeepGEMM kernel is warmed, regardless of the
|
||||
quantization method that owns it or the name of its scale parameter."""
|
||||
layer = _block_fp8_layer(256, 128, scale_name)
|
||||
|
||||
calls = _run_warmup(monkeypatch, [layer])
|
||||
|
||||
assert len(calls) == 1
|
||||
assert calls[0]["w"] is layer.weight
|
||||
assert calls[0]["ws"] is getattr(layer, scale_name)
|
||||
|
||||
|
||||
def test_n_multiple_of_64_matches_kernel_selection(monkeypatch) -> None:
|
||||
"""DeepGEMM accepts N % 64 == 0 (e.g. DeepSeek kv_a_proj_with_mqa, N=576),
|
||||
so warmup must not require N % 128 == 0."""
|
||||
assert len(_run_warmup(monkeypatch, [_block_fp8_layer(576, 256)])) == 1
|
||||
|
||||
|
||||
def test_unstamped_and_mismatched_block_layers_are_skipped(monkeypatch) -> None:
|
||||
mismatched = _block_fp8_layer(256, 128)
|
||||
mismatched.weight_block_size = [1, 32]
|
||||
layers = [SimpleNamespace(), mismatched, _block_fp8_layer(256, 128)]
|
||||
|
||||
calls = _run_warmup(monkeypatch, layers)
|
||||
|
||||
assert len(calls) == 1
|
||||
assert calls[0]["w"] is layers[-1].weight
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_bmm", [False, True])
|
||||
def test_kernel_registers_itself_as_warmup_provider(is_bmm) -> None:
|
||||
"""Bmm layers bypass the kernel at runtime, so they must not be warmed."""
|
||||
kernel = object.__new__(DeepGemmFp8BlockScaledMMKernel)
|
||||
kernel.is_deep_gemm_supported = False
|
||||
layer = torch.nn.Module()
|
||||
layer.weight = torch.nn.Parameter(
|
||||
torch.empty((256, 128), dtype=torch.float8_e4m3fn), requires_grad=False
|
||||
)
|
||||
layer.weight_scale = torch.nn.Parameter(
|
||||
torch.empty((2, 1), dtype=torch.float32), requires_grad=False
|
||||
)
|
||||
layer.weight_block_size = [128, 128]
|
||||
layer.is_bmm = is_bmm
|
||||
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
provider = getattr(layer, "deep_gemm_warmup_provider", None)
|
||||
assert provider is (None if is_bmm else kernel)
|
||||
|
||||
|
||||
def test_mega_moe_predicate_matches_only_native_mega_moe_layers(monkeypatch) -> None:
|
||||
"""Without the MegaMoE model modules loaded, nothing matches and nothing is
|
||||
imported; with them loaded, only DeepSeek-V4 and Kimi K3 MegaMoE layers match."""
|
||||
name = "vllm.models.deepseek_v4.nvidia.model"
|
||||
kimi_name = "vllm.models.kimi_k3.nvidia.model"
|
||||
monkeypatch.delitem(sys.modules, name, raising=False)
|
||||
monkeypatch.delitem(sys.modules, kimi_name, raising=False)
|
||||
other = SimpleNamespace(use_native_mega_moe=True, use_mega_moe=True)
|
||||
assert not deep_gemm_warmup._mega_moe_may_use_deep_gemm(other)
|
||||
assert name not in sys.modules and kimi_name not in sys.modules
|
||||
|
||||
moe_cls = type("DeepseekV4MoE", (), {})
|
||||
monkeypatch.setitem(sys.modules, name, SimpleNamespace(DeepseekV4MoE=moe_cls))
|
||||
native, fused = moe_cls(), moe_cls()
|
||||
native.use_native_mega_moe, fused.use_native_mega_moe = True, False
|
||||
assert deep_gemm_warmup._mega_moe_may_use_deep_gemm(native)
|
||||
assert not deep_gemm_warmup._mega_moe_may_use_deep_gemm(fused)
|
||||
assert not deep_gemm_warmup._mega_moe_may_use_deep_gemm(other)
|
||||
|
||||
kimi_cls = type("KimiMoE", (), {})
|
||||
monkeypatch.setitem(sys.modules, kimi_name, SimpleNamespace(KimiMoE=kimi_cls))
|
||||
kimi_mega, kimi_fused = kimi_cls(), kimi_cls()
|
||||
kimi_mega.use_mega_moe, kimi_fused.use_mega_moe = True, False
|
||||
assert deep_gemm_warmup._mega_moe_may_use_deep_gemm(kimi_mega)
|
||||
assert not deep_gemm_warmup._mega_moe_may_use_deep_gemm(kimi_fused)
|
||||
|
||||
|
||||
def test_mega_moe_m_values_reach_mega_gate(monkeypatch) -> None:
|
||||
"""Each bf16_mega_gate config is warmed with an M above the GateLinear
|
||||
threshold (1 for 128 experts), so the forward actually reaches the kernel."""
|
||||
dg = SimpleNamespace(
|
||||
get_bf16_mega_gate_config=lambda m, *_: {"num_sms": -(-m // 16)},
|
||||
)
|
||||
monkeypatch.setattr(deep_gemm_warmup, "_import_deep_gemm", lambda: dg)
|
||||
monkeypatch.setattr(deep_gemm_warmup, "_mega_moe_uses_mega_gate", lambda _: True)
|
||||
module = SimpleNamespace(
|
||||
gate=SimpleNamespace(input_size=4096, output_size=128),
|
||||
experts=SimpleNamespace(top_k=6, get_block_m=lambda num_tokens: 16),
|
||||
use_sequence_parallel=False,
|
||||
)
|
||||
assert deep_gemm_warmup._get_fp8_fp4_mega_moe_m_values(module, 40) == [16, 32, 40]
|
||||
@@ -1,160 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import LoadConfig, ModelConfig, SpeculativeConfig, VllmConfig
|
||||
from vllm.model_executor.models.utils import get_draft_quant_config
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
DEVICES = (
|
||||
[f"{DEVICE_TYPE}:{i}" for i in range(min(torch.accelerator.device_count(), 2))]
|
||||
if not current_platform.is_cpu()
|
||||
else ["cpu"]
|
||||
)
|
||||
|
||||
|
||||
def test_get_draft_quant_config_with_draft_model():
|
||||
mock_draft_model_config = Mock(spec=ModelConfig)
|
||||
mock_load_config = Mock(spec=LoadConfig)
|
||||
mock_speculative_config = Mock(spec=SpeculativeConfig)
|
||||
mock_speculative_config.draft_model_config = mock_draft_model_config
|
||||
|
||||
mock_vllm_config = Mock(spec=VllmConfig)
|
||||
mock_vllm_config.speculative_config = mock_speculative_config
|
||||
mock_vllm_config.load_config = mock_load_config
|
||||
|
||||
mock_quant_config = Mock()
|
||||
with patch.object(
|
||||
VllmConfig, "get_quantization_config", return_value=mock_quant_config
|
||||
):
|
||||
result = get_draft_quant_config(mock_vllm_config)
|
||||
|
||||
# Verify the function calls get_quantization_config with draft model config
|
||||
VllmConfig.get_quantization_config.assert_called_once_with(
|
||||
mock_draft_model_config, mock_load_config
|
||||
)
|
||||
assert result == mock_quant_config
|
||||
|
||||
|
||||
def test_get_draft_quant_config_without_draft_model():
|
||||
mock_speculative_config = Mock(spec=SpeculativeConfig)
|
||||
mock_speculative_config.draft_model_config = None
|
||||
|
||||
mock_vllm_config = Mock(spec=VllmConfig)
|
||||
mock_vllm_config.speculative_config = mock_speculative_config
|
||||
mock_vllm_config.load_config = Mock(spec=LoadConfig)
|
||||
|
||||
result = get_draft_quant_config(mock_vllm_config)
|
||||
|
||||
assert result is None
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
@pytest.mark.parametrize("device", DEVICES)
|
||||
def test_fc_layer_quant_config_usage(default_vllm_config, dist_init, device) -> None:
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
|
||||
if current_platform.is_cuda_alike():
|
||||
torch.accelerator.set_device_index(device)
|
||||
|
||||
torch.set_default_device(device)
|
||||
|
||||
input_size = 256
|
||||
output_size = 128
|
||||
|
||||
fc_no_quant = ReplicatedLinear(
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
bias=False,
|
||||
params_dtype=torch.float16,
|
||||
quant_config=None,
|
||||
prefix="fc",
|
||||
)
|
||||
|
||||
assert fc_no_quant.quant_config is None
|
||||
assert fc_no_quant.input_size == input_size
|
||||
assert fc_no_quant.output_size == output_size
|
||||
|
||||
mock_quant_config = Mock()
|
||||
mock_quant_config.online_quantization_config = None
|
||||
fc_with_quant = ReplicatedLinear(
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
bias=False,
|
||||
params_dtype=torch.float16,
|
||||
quant_config=mock_quant_config,
|
||||
prefix="fc",
|
||||
)
|
||||
|
||||
assert fc_with_quant.quant_config == mock_quant_config
|
||||
|
||||
# Check forward pass
|
||||
x = torch.randn(2, input_size, dtype=torch.float16)
|
||||
output, _ = fc_no_quant(x)
|
||||
assert output.shape == (2, output_size)
|
||||
|
||||
|
||||
def test_maybe_remap_kv_scale_name():
|
||||
from vllm.model_executor.model_loader.weight_utils import maybe_remap_kv_scale_name
|
||||
|
||||
params_dict = {
|
||||
"layers.0.self_attn.kv_scale": Mock(),
|
||||
"layers.1.self_attn.kv_scale": Mock(),
|
||||
}
|
||||
|
||||
name = "layers.0.self_attn.some_scale"
|
||||
remapped = maybe_remap_kv_scale_name(name, params_dict)
|
||||
|
||||
assert remapped in params_dict or remapped == name or remapped is None
|
||||
|
||||
|
||||
def test_eagle3_lm_head_receives_quant_config():
|
||||
"""Eagle3LlamaForCausalLM must pass quant_config to ParallelLMHead.
|
||||
|
||||
Without quant_config, quantized lm_head weights (e.g. INT8 per-channel)
|
||||
in Eagle3 drafter checkpoints fail to load because ParallelLMHead doesn't
|
||||
expect weight_packed tensors.
|
||||
"""
|
||||
from vllm.model_executor.models.llama_eagle3 import Eagle3LlamaForCausalLM
|
||||
|
||||
mock_quant_config = Mock()
|
||||
|
||||
mock_hf_config = Mock()
|
||||
mock_hf_config.draft_vocab_size = 1000
|
||||
mock_hf_config.hidden_size = 256
|
||||
mock_hf_config.vocab_size = 32000
|
||||
mock_hf_config.logit_scale = 1.0
|
||||
|
||||
mock_vllm_config = Mock()
|
||||
mock_vllm_config.speculative_config.draft_model_config.hf_config = mock_hf_config
|
||||
mock_vllm_config.model_config.get_num_layers.return_value = 32
|
||||
mock_vllm_config.speculative_config.parallel_drafting = False
|
||||
|
||||
with (
|
||||
patch("vllm.model_executor.models.llama_eagle3.LlamaModel") as MockModel,
|
||||
patch("vllm.model_executor.models.llama_eagle3.ParallelLMHead") as MockLMHead,
|
||||
patch("vllm.model_executor.models.llama_eagle3.LogitsProcessor"),
|
||||
patch(
|
||||
"vllm.model_executor.models.llama_eagle3.get_draft_quant_config",
|
||||
return_value=mock_quant_config,
|
||||
),
|
||||
):
|
||||
MockModel.return_value.use_aux_hidden_state = True
|
||||
|
||||
Eagle3LlamaForCausalLM(vllm_config=mock_vllm_config)
|
||||
|
||||
MockLMHead.assert_called_once()
|
||||
call_kwargs = MockLMHead.call_args.kwargs
|
||||
assert "quant_config" in call_kwargs, (
|
||||
"ParallelLMHead must receive quant_config for quantized lm_head weights"
|
||||
)
|
||||
assert call_kwargs["quant_config"] is mock_quant_config, (
|
||||
"ParallelLMHead must receive the draft model's quant_config"
|
||||
)
|
||||
@@ -1,143 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.ernie45_vl import (
|
||||
Ernie4_5_VLMoeForConditionalGeneration,
|
||||
)
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldElem,
|
||||
MultiModalKwargsItem,
|
||||
PlaceholderRange,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True, scope="module")
|
||||
def _force_cpu_default_device():
|
||||
original = torch.get_default_device()
|
||||
torch.set_default_device("cpu")
|
||||
yield
|
||||
torch.set_default_device(original)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DummyConfig:
|
||||
spatial_conv_size: int = 2
|
||||
temporal_conv_size: int = 2
|
||||
|
||||
|
||||
def make_model(config: DummyConfig) -> Ernie4_5_VLMoeForConditionalGeneration:
|
||||
model = object.__new__(Ernie4_5_VLMoeForConditionalGeneration)
|
||||
model.config = config
|
||||
return model
|
||||
|
||||
|
||||
def make_mm_feature(
|
||||
*,
|
||||
modality: str,
|
||||
offset: int,
|
||||
length: int,
|
||||
grid_thw: tuple[int, int, int],
|
||||
) -> MultiModalFeatureSpec:
|
||||
field_name = "image_grid_thw" if modality == "image" else "video_grid_thw"
|
||||
return MultiModalFeatureSpec(
|
||||
data=MultiModalKwargsItem(
|
||||
{
|
||||
field_name: MultiModalFieldElem(
|
||||
data=torch.tensor(grid_thw),
|
||||
field=None, # HACK.
|
||||
),
|
||||
}
|
||||
),
|
||||
modality=modality,
|
||||
identifier="DUMMY",
|
||||
mm_position=PlaceholderRange(offset=offset, length=length),
|
||||
)
|
||||
|
||||
|
||||
def test_get_mrope_input_positions_text_only():
|
||||
model = make_model(DummyConfig())
|
||||
|
||||
positions, delta = model.get_mrope_input_positions(
|
||||
input_tokens=[11, 12, 13, 14, 15],
|
||||
mm_features=[],
|
||||
)
|
||||
|
||||
expected = torch.tensor(
|
||||
[
|
||||
[0, 1, 2, 3, 4],
|
||||
[0, 1, 2, 3, 4],
|
||||
[0, 1, 2, 3, 4],
|
||||
]
|
||||
)
|
||||
|
||||
assert torch.equal(positions, expected)
|
||||
assert delta == 0
|
||||
|
||||
|
||||
def test_get_mrope_input_positions_single_image():
|
||||
model = make_model(DummyConfig())
|
||||
mm_features = [
|
||||
make_mm_feature(
|
||||
modality="image",
|
||||
offset=1,
|
||||
length=4,
|
||||
grid_thw=(1, 4, 4),
|
||||
)
|
||||
]
|
||||
|
||||
positions, delta = model.get_mrope_input_positions(
|
||||
input_tokens=[10, 20, 21, 22, 23, 30, 31],
|
||||
mm_features=mm_features,
|
||||
)
|
||||
|
||||
expected = torch.tensor(
|
||||
[
|
||||
[0, 1, 1, 1, 1, 3, 4],
|
||||
[0, 1, 1, 2, 2, 3, 4],
|
||||
[0, 1, 2, 1, 2, 3, 4],
|
||||
]
|
||||
)
|
||||
|
||||
assert torch.equal(positions, expected)
|
||||
assert delta == -2
|
||||
|
||||
|
||||
def test_get_mrope_input_positions_interleaved_image_and_video():
|
||||
model = make_model(DummyConfig())
|
||||
mm_features = [
|
||||
make_mm_feature(
|
||||
modality="image",
|
||||
offset=1,
|
||||
length=4,
|
||||
grid_thw=(1, 4, 4),
|
||||
),
|
||||
make_mm_feature(
|
||||
modality="video",
|
||||
offset=7,
|
||||
length=2,
|
||||
grid_thw=(2, 4, 2),
|
||||
),
|
||||
]
|
||||
|
||||
positions, delta = model.get_mrope_input_positions(
|
||||
input_tokens=[10, 20, 21, 22, 23, 30, 31, 40, 41, 50, 51],
|
||||
mm_features=mm_features,
|
||||
)
|
||||
|
||||
expected = torch.tensor(
|
||||
[
|
||||
[0, 1, 1, 1, 1, 3, 4, 5, 5, 7, 8],
|
||||
[0, 1, 1, 2, 2, 3, 4, 5, 6, 7, 8],
|
||||
[0, 1, 2, 1, 2, 3, 4, 5, 5, 7, 8],
|
||||
]
|
||||
)
|
||||
|
||||
assert torch.equal(positions, expected)
|
||||
assert delta == -2
|
||||
@@ -1,682 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from inspect import signature
|
||||
from pathlib import Path
|
||||
from types import ModuleType, SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import Mock, call, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.warmup.kernel_warmup as warmup
|
||||
import vllm.utils.flashinfer as fi_utils
|
||||
from vllm.model_executor.warmup.kernel_warmup import (
|
||||
_flashinfer_autotune_token_counts,
|
||||
_run_flashinfer_autotune_dummy_runs,
|
||||
flashinfer_autotune,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
class _FakeMoERunner:
|
||||
"""Minimal MoERunner stand-in for token-count discovery tests."""
|
||||
|
||||
moe_config: Any
|
||||
|
||||
|
||||
def _make_moe(*, max_deferred_tokens: int = 128, enabled: bool = True):
|
||||
"""Create a fake MoE layer with a deferred-finalize token limit."""
|
||||
moe = _FakeMoERunner()
|
||||
moe.moe_config = SimpleNamespace(
|
||||
use_deferred_moe_finalize=enabled,
|
||||
defer_moe_finalize_max_num_tokens=max_deferred_tokens,
|
||||
)
|
||||
return moe
|
||||
|
||||
|
||||
def _make_runner(modules, *, max_tokens: int = 8192):
|
||||
"""Create a runner carrying only the state used by FlashInfer warmup."""
|
||||
return SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=max_tokens),
|
||||
vllm_config=SimpleNamespace(
|
||||
attention_config=SimpleNamespace(hisparse_config=None),
|
||||
parallel_config=SimpleNamespace(
|
||||
enable_elastic_ep=False,
|
||||
nnodes=1,
|
||||
pipeline_parallel_size=1,
|
||||
data_parallel_rank=0,
|
||||
),
|
||||
),
|
||||
get_model=Mock(
|
||||
return_value=SimpleNamespace(modules=Mock(return_value=modules))
|
||||
),
|
||||
_dummy_run=Mock(),
|
||||
)
|
||||
|
||||
|
||||
def test_flashinfer_autotune_token_counts_include_deferred_moe_limits():
|
||||
runner = _make_runner(
|
||||
[
|
||||
object(),
|
||||
_make_moe(max_deferred_tokens=128),
|
||||
_make_moe(max_deferred_tokens=128),
|
||||
_make_moe(max_deferred_tokens=64, enabled=False),
|
||||
_make_moe(max_deferred_tokens=-1),
|
||||
]
|
||||
)
|
||||
|
||||
with patch("vllm.model_executor.layers.fused_moe.MoERunner", _FakeMoERunner):
|
||||
token_counts = _flashinfer_autotune_token_counts(runner)
|
||||
|
||||
assert token_counts == (8192, 128)
|
||||
|
||||
|
||||
def test_flashinfer_autotune_token_counts_are_bounded_and_deduplicated():
|
||||
runner = _make_runner(
|
||||
[_make_moe(max_deferred_tokens=4096)],
|
||||
max_tokens=32,
|
||||
)
|
||||
|
||||
with patch("vllm.model_executor.layers.fused_moe.MoERunner", _FakeMoERunner):
|
||||
token_counts = _flashinfer_autotune_token_counts(runner)
|
||||
|
||||
assert token_counts == (32,)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("skip_attn", [False, True])
|
||||
def test_flashinfer_autotune_uses_token_buckets_for_each_dummy_run(skip_attn):
|
||||
from vllm.v1.worker.gpu.model_runner import GPUModelRunner as V2Runner
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner as V1Runner
|
||||
|
||||
runner = _make_runner([])
|
||||
dummy_run_signature = signature((V2Runner if skip_attn else V1Runner)._dummy_run)
|
||||
runner._dummy_run.side_effect = lambda **kwargs: dummy_run_signature.bind(
|
||||
None, **kwargs
|
||||
)
|
||||
max_buckets = (1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 8192)
|
||||
deferred_buckets = (1, 2, 4, 8, 16, 32, 64, 128)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.model_executor.warmup.kernel_warmup."
|
||||
"_flashinfer_autotune_token_counts",
|
||||
return_value=(8192, 128),
|
||||
),
|
||||
patch(
|
||||
"vllm.utils.flashinfer.flashinfer_get_hybrid_num_tokens_buckets",
|
||||
side_effect=(max_buckets, deferred_buckets),
|
||||
) as get_buckets,
|
||||
patch("vllm.utils.flashinfer.autotune") as autotune,
|
||||
):
|
||||
_run_flashinfer_autotune_dummy_runs(runner, skip_attn=skip_attn)
|
||||
|
||||
assert get_buckets.call_args_list == [call(8192), call(128)]
|
||||
assert autotune.call_args_list == [
|
||||
call(tuning_buckets=max_buckets, round_up=True),
|
||||
call(tuning_buckets=deferred_buckets, round_up=True),
|
||||
]
|
||||
assert runner._dummy_run.call_args_list == [
|
||||
call(
|
||||
num_tokens=8192,
|
||||
skip_eplb=True,
|
||||
is_profile=True,
|
||||
randomize_inputs=True,
|
||||
**({"skip_attn": True} if skip_attn else {}),
|
||||
),
|
||||
call(
|
||||
num_tokens=128,
|
||||
skip_eplb=True,
|
||||
is_profile=True,
|
||||
randomize_inputs=True,
|
||||
**({"skip_attn": True} if skip_attn else {}),
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"has_v2,elastic_ep,node_count",
|
||||
[(False, False, 1), (True, False, 1), (True, True, 1), (True, False, 2)],
|
||||
ids=["v1", "v2", "elastic-v1", "multi-node-v1"],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"hisparse,world_size,rank,skip_ops,failure",
|
||||
[
|
||||
(False, 1, 0, [], None),
|
||||
(False, 2, 0, [], None),
|
||||
(False, 2, 1, [], None),
|
||||
(True, 2, 0, ["fp4_gemm", "bmm_fp8"], None),
|
||||
(False, 2, 1, ["fp4_gemm"], "dummy"),
|
||||
(False, 2, 1, [], "replayssm"),
|
||||
(True, 2, 0, [], "kimi"),
|
||||
],
|
||||
)
|
||||
def test_flashinfer_autotune_lifecycle(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
has_v2,
|
||||
elastic_ep,
|
||||
node_count,
|
||||
hisparse,
|
||||
world_size,
|
||||
rank,
|
||||
skip_ops,
|
||||
failure,
|
||||
):
|
||||
use_v2 = has_v2 and not elastic_ep and node_count == 1 and world_size == 1
|
||||
runner = _make_runner([])
|
||||
runner.vllm_config.parallel_config.enable_elastic_ep = elastic_ep
|
||||
runner.vllm_config.attention_config.hisparse_config = object() if hisparse else None
|
||||
model = runner.get_model()
|
||||
cache_path = tmp_path / "autotune.json"
|
||||
rank_cache_path = cache_path.with_name(f"autotune_dp0_rank{rank}.json")
|
||||
rank_cache_path.write_text(json.dumps({"_metadata": {}, "op": 1}))
|
||||
events = Mock()
|
||||
events.tuner.load_configs.return_value = True
|
||||
events.tuner._dirty = True
|
||||
world = SimpleNamespace(
|
||||
rank_in_group=rank,
|
||||
world_size=world_size,
|
||||
cpu_group=object(),
|
||||
broadcast_object=events.broadcast,
|
||||
barrier=events.barrier,
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def autotune_context(**kwargs):
|
||||
assert torch.is_inference_mode_enabled()
|
||||
events.enter()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
events.exit()
|
||||
|
||||
events.autotune.side_effect = autotune_context
|
||||
if failure is not None:
|
||||
getattr(events, failure).side_effect = RuntimeError("warmup failed")
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"flashinfer.autotuner",
|
||||
SimpleNamespace(
|
||||
AutoTuner=SimpleNamespace(get=lambda: events.tuner),
|
||||
set_autotune_process_group=events.set_group,
|
||||
),
|
||||
)
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"flashinfer",
|
||||
SimpleNamespace(autotune_v2=events.autotune, autotune_v2_reload=events.reload),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.parallel_state.get_world_group", lambda: world
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.parallel_state.get_node_count", lambda: node_count
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.parallel_state.get_pp_group",
|
||||
lambda: SimpleNamespace(world_size=1),
|
||||
)
|
||||
monkeypatch.setattr(fi_utils, "has_flashinfer_autotune_v2", lambda: has_v2)
|
||||
monkeypatch.setattr(fi_utils, "autotune", events.autotune)
|
||||
monkeypatch.setattr(warmup.envs, "VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS", skip_ops)
|
||||
resolve_file = Mock(return_value=cache_path)
|
||||
resolve_root = Mock(return_value=str(tmp_path))
|
||||
monkeypatch.setattr(warmup, "resolve_flashinfer_autotune_file", resolve_file)
|
||||
monkeypatch.setattr(warmup, "resolve_flashinfer_autotune_v2_root", resolve_root)
|
||||
|
||||
def gather(out, value, *, group):
|
||||
out[:] = [value] * world_size
|
||||
|
||||
monkeypatch.setattr(torch.distributed, "all_gather_object", gather)
|
||||
monkeypatch.setattr(warmup, "_run_flashinfer_autotune_dummy_runs", events.dummy)
|
||||
monkeypatch.setattr(warmup, "replayssm_autotune_warmup", events.replayssm)
|
||||
monkeypatch.setattr(warmup, "_autotune_kimi_k3_kda_qkvg", events.kimi)
|
||||
monkeypatch.setattr(
|
||||
warmup, "autotune_hisparse_flashinfer_attention", events.hisparse
|
||||
)
|
||||
|
||||
if failure is None:
|
||||
warmup.flashinfer_autotune(runner)
|
||||
else:
|
||||
with pytest.raises(RuntimeError, match="warmup failed"):
|
||||
warmup.flashinfer_autotune(runner)
|
||||
|
||||
kwargs = {"skip_ops": set(skip_ops)} if skip_ops else {}
|
||||
expected = []
|
||||
if use_v2:
|
||||
resolve_file.assert_not_called()
|
||||
resolve_root.assert_called_once_with()
|
||||
expected.append(call.autotune(mode="tune", cache_root=str(tmp_path), **kwargs))
|
||||
else:
|
||||
resolve_root.assert_not_called()
|
||||
resolve_file.assert_called_once_with(runner)
|
||||
expected.extend(
|
||||
[
|
||||
call.tuner.load_configs(str(rank_cache_path)),
|
||||
call.autotune(tune_mode=True, **kwargs),
|
||||
]
|
||||
)
|
||||
expected.extend(
|
||||
[call.set_group(world.cpu_group if world_size > 1 else None), call.enter()]
|
||||
)
|
||||
if hisparse:
|
||||
expected.append(call.hisparse(runner))
|
||||
for name, arg, kwargs in [
|
||||
("dummy", runner, {"skip_attn": hisparse}),
|
||||
("replayssm", runner, {}),
|
||||
("kimi", model, {}),
|
||||
]:
|
||||
expected.append(getattr(call, name)(arg, **kwargs))
|
||||
if failure == name:
|
||||
break
|
||||
expected.append(call.exit())
|
||||
expected.append(call.set_group(None))
|
||||
if failure is None:
|
||||
if world_size > 1:
|
||||
expected.append(call.barrier())
|
||||
if use_v2 and world_size > 1:
|
||||
expected.extend([call.reload(), call.barrier()])
|
||||
elif not use_v2:
|
||||
expected.append(call.tuner.save_configs(str(rank_cache_path)))
|
||||
assert events.mock_calls == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"node_count,elastic_ep,pp_size",
|
||||
[(1, False, 1), (2, False, 1), (1, True, 1), (1, False, 2)],
|
||||
)
|
||||
@pytest.mark.parametrize("world_size,rank", [(1, 0), (2, 0), (2, 1)])
|
||||
@pytest.mark.parametrize("model_runner_v2,max_reqs", [(False, 2), (True, 1), (True, 2)])
|
||||
def test_sparse_autotune_respects_cache_selection(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
node_count,
|
||||
elastic_ep,
|
||||
pp_size,
|
||||
world_size,
|
||||
rank,
|
||||
model_runner_v2,
|
||||
max_reqs,
|
||||
):
|
||||
import vllm.model_executor.warmup.flashinfer_sparse_mla_warmup as sparse
|
||||
|
||||
runner = _make_runner([])
|
||||
runner.max_num_reqs = max_reqs
|
||||
runner.vllm_config.use_v2_model_runner = model_runner_v2
|
||||
runner.vllm_config.kernel_config = SimpleNamespace(enable_flashinfer_autotune=True)
|
||||
runner.vllm_config.parallel_config.enable_elastic_ep = elastic_ep
|
||||
runner.vllm_config.parallel_config.pipeline_parallel_size = pp_size
|
||||
worker = SimpleNamespace(
|
||||
model_runner=runner,
|
||||
vllm_config=runner.vllm_config,
|
||||
execute_model=Mock(),
|
||||
sample_tokens=Mock(),
|
||||
)
|
||||
world = SimpleNamespace(
|
||||
rank_in_group=rank,
|
||||
world_size=world_size,
|
||||
broadcast_object=Mock(return_value=b"cached"),
|
||||
barrier=Mock(),
|
||||
)
|
||||
managed = Mock(return_value=nullcontext())
|
||||
legacy = Mock(return_value=nullcontext())
|
||||
reload = Mock()
|
||||
tuner = Mock()
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"flashinfer",
|
||||
SimpleNamespace(autotune_v2=managed, autotune_v2_reload=reload),
|
||||
)
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"flashinfer.autotuner",
|
||||
SimpleNamespace(AutoTuner=SimpleNamespace(get=lambda: tuner)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.parallel_state.get_world_group", lambda: world
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.parallel_state.get_node_count", lambda: node_count
|
||||
)
|
||||
monkeypatch.setattr(fi_utils, "has_flashinfer_autotune_v2", lambda: True)
|
||||
monkeypatch.setattr(sparse, "has_flashinfer", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
sparse.current_platform, "is_device_capability_family", lambda _: True
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sparse, "_flashinfer_sparse_mla_decode_label", lambda *_: "DSv3.2"
|
||||
)
|
||||
monkeypatch.setattr(sparse, "resolve_flashinfer_autotune_v2_root", lambda: tmp_path)
|
||||
cache_path = tmp_path / "autotune.json"
|
||||
cache_path.write_bytes(b"cached")
|
||||
monkeypatch.setattr(
|
||||
sparse, "resolve_flashinfer_autotune_file", lambda _: cache_path
|
||||
)
|
||||
monkeypatch.setattr(sparse, "flashinfer_autotune", legacy)
|
||||
|
||||
def mixed_warmup(*args, mixed_step_context=None, **kwargs):
|
||||
context = (
|
||||
mixed_step_context if mixed_step_context is not None else nullcontext()
|
||||
)
|
||||
with context:
|
||||
assert torch.is_inference_mode_enabled()
|
||||
return True
|
||||
|
||||
mixed = Mock(side_effect=mixed_warmup)
|
||||
monkeypatch.setattr(sparse, "run_mixed_prefill_decode_warmup", mixed)
|
||||
assert sparse._run_flashinfer_sparse_mla_decode_autotune(worker, 16, frozenset())
|
||||
|
||||
if node_count == 1 and not elastic_ep and pp_size == 1 and world_size == 1:
|
||||
managed.assert_called_once_with(mode="tune", cache_root=tmp_path)
|
||||
legacy.assert_not_called()
|
||||
world.broadcast_object.assert_not_called()
|
||||
tuner.load_configs.assert_not_called()
|
||||
reload.assert_not_called()
|
||||
world.barrier.assert_not_called()
|
||||
else:
|
||||
managed.assert_not_called()
|
||||
reload.assert_not_called()
|
||||
assert legacy.call_count == (1 if rank == 0 else 0)
|
||||
world.broadcast_object.assert_called_once_with(
|
||||
b"cached" if rank == 0 else None, src=0
|
||||
)
|
||||
tuner.load_configs.assert_called_once_with(str(cache_path))
|
||||
assert mixed.call_count == (1 if model_runner_v2 and max_reqs >= 2 else 0)
|
||||
assert runner._dummy_run.call_count == (0 if mixed.called else 1)
|
||||
|
||||
|
||||
def test_flashinfer_autotune_buckets_cover_drafter_tokens():
|
||||
"""The drafter's M can exceed the pass size; buckets must include it."""
|
||||
runner = _make_runner([])
|
||||
runner.max_num_reqs = 256
|
||||
runner.max_num_tokens = 8192
|
||||
runner.speculator = SimpleNamespace(num_query_per_req=6)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.model_executor.warmup.kernel_warmup."
|
||||
"_flashinfer_autotune_token_counts",
|
||||
return_value=(8192, 128),
|
||||
),
|
||||
patch(
|
||||
"vllm.utils.flashinfer.flashinfer_get_hybrid_num_tokens_buckets"
|
||||
) as get_buckets,
|
||||
patch("vllm.utils.flashinfer.autotune"),
|
||||
):
|
||||
_run_flashinfer_autotune_dummy_runs(runner)
|
||||
|
||||
assert get_buckets.call_args_list == [call(8192), call(128 * 6)]
|
||||
assert [c.kwargs["num_tokens"] for c in runner._dummy_run.call_args_list] == [
|
||||
8192,
|
||||
128,
|
||||
]
|
||||
|
||||
|
||||
class _AutotuneGroup:
|
||||
def __init__(self, run, ranks):
|
||||
self.run = run
|
||||
self.ranks = tuple(ranks)
|
||||
self.world_size = len(self.ranks)
|
||||
self.rank_in_group = self.ranks.index(run.rank)
|
||||
self.cpu_group = self
|
||||
|
||||
def record(self, operation):
|
||||
self.run.collectives[self.ranks][self.run.rank].append(operation)
|
||||
|
||||
def all_gather_object(self, out, obj):
|
||||
# Ranks run sequentially: snapshot every rank's file before any saves.
|
||||
self.record(("all_gather",))
|
||||
if isinstance(obj, bool):
|
||||
out[:] = [rank not in self.run.rejected_load_ranks for rank in self.ranks]
|
||||
assert out[self.rank_in_group] == obj
|
||||
return
|
||||
out[:] = self.run.gathered.setdefault(
|
||||
self.ranks,
|
||||
[
|
||||
warmup._autotune_cache_fingerprint(self.run.cache_path(rank))
|
||||
for rank in self.ranks
|
||||
],
|
||||
)
|
||||
|
||||
def barrier(self):
|
||||
self.record(("barrier",))
|
||||
|
||||
|
||||
class _AutotuneTuner:
|
||||
def __init__(self, run):
|
||||
self.run = run
|
||||
self.cache = {}
|
||||
self.loaded = None
|
||||
self._dirty = False
|
||||
|
||||
def load_configs(self, path):
|
||||
if self.run.rank in self.run.rejected_load_ranks:
|
||||
return False
|
||||
self.loaded = json.loads(Path(path).read_text())
|
||||
self.cache.update(self.loaded)
|
||||
return True
|
||||
|
||||
def clear_cache(self):
|
||||
self.cache.clear()
|
||||
self._dirty = False
|
||||
|
||||
def save_configs(self, path):
|
||||
self.run.saves.append((self.run.rank, Path(path), dict(self.cache)))
|
||||
Path(path).write_text(json.dumps(self.cache))
|
||||
|
||||
def profile(self, operation):
|
||||
if operation in self.cache:
|
||||
return
|
||||
group = self.run.tuning_group
|
||||
self.run.profile_groups[self.run.rank].append(
|
||||
None if group is None else group.ranks
|
||||
)
|
||||
for tactic in range(2):
|
||||
if group is not None:
|
||||
group.record(("all_reduce", operation, tactic))
|
||||
self.cache[operation] = self.run.rank // self.run.tp
|
||||
self._dirty = True
|
||||
|
||||
|
||||
class _AutotuneRun:
|
||||
def __init__(self, pp, tp, cache_dir):
|
||||
self.pp, self.tp = pp, tp
|
||||
self.cache_dir = cache_dir
|
||||
self.rank = 0
|
||||
self.tuning_group = None
|
||||
self.collectives: dict[tuple[int, ...], dict[int, list[tuple[Any, ...]]]] = (
|
||||
defaultdict(lambda: defaultdict(list))
|
||||
)
|
||||
self.gathered = {}
|
||||
self.rejected_load_ranks: set[int] = set()
|
||||
self.tuners = {}
|
||||
self.saves = []
|
||||
self.profile_groups = defaultdict(list)
|
||||
|
||||
def cache_path(self, rank):
|
||||
return self.cache_dir / f"autotune_configs_dp0_rank{rank}.json"
|
||||
|
||||
def world(self):
|
||||
return _AutotuneGroup(self, range(self.pp * self.tp))
|
||||
|
||||
def tensor_group(self):
|
||||
start = self.rank // self.tp * self.tp
|
||||
return _AutotuneGroup(self, range(start, start + self.tp))
|
||||
|
||||
def pipeline_group(self):
|
||||
return SimpleNamespace(world_size=self.pp)
|
||||
|
||||
def set_group(self, group):
|
||||
self.tuning_group = group
|
||||
|
||||
def dummy_runs(self, runner, *, skip_attn=False):
|
||||
self.tuners[self.rank].profile("shared_gemm")
|
||||
if self.pp > 1 and self.rank // self.tp == 0:
|
||||
self.tuners[self.rank].profile("pp0_extra_gemm")
|
||||
|
||||
def execute(self):
|
||||
for rank in range(self.pp * self.tp):
|
||||
self.rank = rank
|
||||
self.tuners[rank] = _AutotuneTuner(self)
|
||||
runner = _make_runner([])
|
||||
runner.vllm_config.parallel_config.pipeline_parallel_size = self.pp
|
||||
flashinfer_autotune(runner)
|
||||
return self
|
||||
|
||||
def assert_collectives_match(self):
|
||||
for ranks, traces in self.collectives.items():
|
||||
assert set(traces) == set(ranks)
|
||||
expected = traces[ranks[0]]
|
||||
assert all(trace == expected for trace in traces.values()), dict(traces)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def autotune_run(monkeypatch, tmp_path):
|
||||
import vllm.utils.flashinfer as fi_utils
|
||||
from vllm.distributed import parallel_state
|
||||
|
||||
def make_run(*, pp=2, tp=4):
|
||||
run = _AutotuneRun(pp, tp, tmp_path)
|
||||
monkeypatch.setattr(fi_utils, "has_flashinfer_autotune_v2", lambda: pp > 1)
|
||||
monkeypatch.setattr(parallel_state, "get_node_count", lambda: 1)
|
||||
autotuner = ModuleType("flashinfer.autotuner")
|
||||
monkeypatch.setattr(
|
||||
autotuner,
|
||||
"AutoTuner",
|
||||
SimpleNamespace(get=lambda: run.tuners[run.rank]),
|
||||
raising=False,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
autotuner, "set_autotune_process_group", run.set_group, raising=False
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "flashinfer.autotuner", autotuner)
|
||||
monkeypatch.setattr(parallel_state, "get_world_group", run.world)
|
||||
monkeypatch.setattr(parallel_state, "get_tp_group", run.tensor_group)
|
||||
monkeypatch.setattr(parallel_state, "get_pp_group", run.pipeline_group)
|
||||
monkeypatch.setattr(fi_utils, "autotune", lambda **kwargs: nullcontext())
|
||||
monkeypatch.setattr(
|
||||
"torch.distributed.all_gather_object",
|
||||
lambda out, obj, group: group.all_gather_object(out, obj),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
warmup,
|
||||
"resolve_flashinfer_autotune_file",
|
||||
lambda runner: tmp_path / "autotune_configs.json",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
warmup, "_flashinfer_autotune_skip_ops", lambda runner: None
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
warmup, "_run_flashinfer_autotune_dummy_runs", run.dummy_runs
|
||||
)
|
||||
monkeypatch.setattr(warmup, "replayssm_autotune_warmup", lambda runner: None)
|
||||
monkeypatch.setattr(warmup, "_autotune_kimi_k3_kda_qkvg", lambda model: None)
|
||||
return run
|
||||
|
||||
return make_run
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp", [1, 4])
|
||||
def test_heterogeneous_pp_stages_have_compatible_collectives(autotune_run, tp):
|
||||
run = autotune_run(tp=tp).execute()
|
||||
run.assert_collectives_match()
|
||||
for rank, groups in run.profile_groups.items():
|
||||
expected = tuple(range(rank // tp * tp, (rank // tp + 1) * tp))
|
||||
assert groups and all(
|
||||
group == (expected if tp > 1 else None) for group in groups
|
||||
)
|
||||
|
||||
|
||||
def test_pp_stage_cache_roundtrip_isolated_and_asymmetric_hits_safe(
|
||||
autotune_run, tmp_path
|
||||
):
|
||||
legacy = tmp_path / "autotune_configs.json"
|
||||
legacy.write_text('{"legacy_world_cache": 99}')
|
||||
cold = autotune_run().execute()
|
||||
cold.assert_collectives_match()
|
||||
assert [rank for rank, _, _ in cold.saves] == list(range(8))
|
||||
assert [path for _, path, _ in cold.saves] == [cold.cache_path(r) for r in range(8)]
|
||||
assert json.loads(legacy.read_text()) == {"legacy_world_cache": 99}
|
||||
stage_caches = [{"shared_gemm": 0, "pp0_extra_gemm": 0}, {"shared_gemm": 1}]
|
||||
assert all(cache == stage_caches[r // 4] for r, _, cache in cold.saves)
|
||||
warm = autotune_run().execute()
|
||||
warm.assert_collectives_match()
|
||||
assert not warm.profile_groups and not warm.saves
|
||||
for rank, tuner in warm.tuners.items():
|
||||
assert tuner.loaded == stage_caches[rank // 4]
|
||||
cold.cache_path(5).unlink()
|
||||
mixed = autotune_run().execute()
|
||||
mixed.assert_collectives_match()
|
||||
assert set(mixed.profile_groups) == {4, 5, 6, 7}
|
||||
assert all(mixed.tuners[rank].loaded is not None for rank in range(4))
|
||||
assert all(mixed.tuners[rank].loaded is None for rank in range(4, 8))
|
||||
|
||||
|
||||
def test_pp1_tunes_world_group_and_saves_per_rank(autotune_run):
|
||||
run = autotune_run(pp=1, tp=4).execute()
|
||||
run.assert_collectives_match()
|
||||
assert [(rank, path) for rank, path, _ in run.saves] == [
|
||||
(rank, run.cache_path(rank)) for rank in range(4)
|
||||
]
|
||||
assert all(groups == [(0, 1, 2, 3)] for groups in run.profile_groups.values())
|
||||
|
||||
|
||||
def test_mismatched_rank_caches_are_ignored_by_every_rank(autotune_run):
|
||||
"""One rank loading while others tune deadlocks the per-tactic reduce."""
|
||||
cold = autotune_run(pp=1, tp=4).execute()
|
||||
stale = cold.cache_path(2)
|
||||
stale.write_text(json.dumps({**json.loads(stale.read_text()), "old_gemm": 0}))
|
||||
rerun = autotune_run(pp=1, tp=4).execute()
|
||||
rerun.assert_collectives_match()
|
||||
assert all(tuner.loaded is None for tuner in rerun.tuners.values())
|
||||
assert set(rerun.profile_groups) == {0, 1, 2, 3}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pp, tp", [(1, 1), (1, 4), (2, 4)])
|
||||
def test_rejected_cache_retunes_every_rank_in_its_tuning_group(autotune_run, pp, tp):
|
||||
"""Matching files can still be incompatible with one rank's runtime."""
|
||||
autotune_run(pp=pp, tp=tp).execute()
|
||||
rerun = autotune_run(pp=pp, tp=tp)
|
||||
rerun.rejected_load_ranks = {tp - 1}
|
||||
rerun.execute()
|
||||
rerun.assert_collectives_match()
|
||||
assert set(rerun.profile_groups) == set(range(tp))
|
||||
assert {rank for rank, _, _ in rerun.saves} == set(range(tp))
|
||||
|
||||
|
||||
def test_autotune_cache_key_normalizes_kv_init_fields():
|
||||
"""mamba_block_size/kv_cache_layout only resolve at engine KV-init time;
|
||||
they must not feed the autotune cache key, or nothing can compute the key
|
||||
before startup (the daemon tunes before any KV init). block_size must
|
||||
stay in the key: the daemon replays the engine's block-size resolution
|
||||
before tuning, and different block sizes may select different ops."""
|
||||
from vllm.config.cache import CacheConfig
|
||||
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
|
||||
_normalize_cache_config_for_hash,
|
||||
)
|
||||
|
||||
fresh = CacheConfig()
|
||||
# Simulate the engine's KV-init write-backs (hybrid model values).
|
||||
mutated = CacheConfig()
|
||||
mutated.mamba_block_size = 512
|
||||
mutated.kv_cache_layout = "LBHNC"
|
||||
|
||||
# Negative control: the write-backs are hashed factors.
|
||||
assert mutated.compute_hash() != fresh.compute_hash()
|
||||
normalized = _normalize_cache_config_for_hash(mutated)
|
||||
assert normalized.compute_hash() == fresh.compute_hash()
|
||||
# Factory state is returned untouched.
|
||||
assert _normalize_cache_config_for_hash(fresh) is fresh
|
||||
# block_size stays significant (the daemon keys on the resolved value).
|
||||
resolved = CacheConfig(block_size=64)
|
||||
assert (
|
||||
_normalize_cache_config_for_hash(resolved).compute_hash()
|
||||
!= fresh.compute_hash()
|
||||
)
|
||||
@@ -1,60 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.activation import (
|
||||
GeluAndMul,
|
||||
SiluAndMul,
|
||||
get_act_and_mul_fn,
|
||||
get_act_fn,
|
||||
)
|
||||
from vllm.model_executor.models.gemma3 import Gemma3MLP
|
||||
from vllm.model_executor.models.gemma4 import Gemma4MLP
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("activation_name", "expected_type"),
|
||||
[
|
||||
("gelu_pytorch_tanh", GeluAndMul),
|
||||
("silu", SiluAndMul),
|
||||
("swish", SiluAndMul),
|
||||
],
|
||||
)
|
||||
def test_get_act_and_mul_fn_supports_gemma_hidden_act_aliases(
|
||||
activation_name: str,
|
||||
expected_type: type[torch.nn.Module],
|
||||
default_vllm_config,
|
||||
) -> None:
|
||||
assert isinstance(get_act_and_mul_fn(activation_name), expected_type)
|
||||
|
||||
|
||||
def test_get_act_fn_supports_swish_alias() -> None:
|
||||
assert isinstance(get_act_fn("swish"), torch.nn.SiLU)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mlp_cls", [Gemma3MLP, Gemma4MLP])
|
||||
@pytest.mark.parametrize(
|
||||
("activation_name", "expected_type"),
|
||||
[
|
||||
("gelu_pytorch_tanh", GeluAndMul),
|
||||
("silu", SiluAndMul),
|
||||
("swish", SiluAndMul),
|
||||
],
|
||||
)
|
||||
def test_gemma_mlp_supports_hidden_act_variants(
|
||||
mlp_cls: type[torch.nn.Module],
|
||||
activation_name: str,
|
||||
expected_type: type[torch.nn.Module],
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
) -> None:
|
||||
mlp = mlp_cls(
|
||||
hidden_size=16,
|
||||
intermediate_size=32,
|
||||
hidden_activation=activation_name,
|
||||
)
|
||||
|
||||
assert isinstance(mlp.act_fn, expected_type)
|
||||
assert mlp(torch.randn(3, 16)).shape == (3, 16)
|
||||
@@ -1,636 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
import vllm.model_executor.models.k2_horizon as k2_horizon
|
||||
from tests.quantization.utils import is_quant_method_supported
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
cleanup_dist_env_and_memory,
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
LinearBase,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config, Fp8LinearMethod
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.models.k2_horizon import (
|
||||
K2HorizonDecoderLayer,
|
||||
K2HorizonModel,
|
||||
K2HorizonRMSNorm,
|
||||
_rope_weight_perm,
|
||||
apply_partial_rope,
|
||||
is_rope_weights_folding_supported,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype, set_random_seed
|
||||
|
||||
DEVICE = current_platform.device_type
|
||||
DTYPE = torch.bfloat16
|
||||
SHAPES = [
|
||||
(128, 64),
|
||||
(128, 128),
|
||||
(192, 128),
|
||||
(96, 32),
|
||||
]
|
||||
|
||||
|
||||
def get_gptj_rope(
|
||||
positions: torch.Tensor,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
rope_head_dim: int,
|
||||
max_position: int,
|
||||
base: float,
|
||||
dtype: torch.dtype,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
rotary_emb = get_rope(
|
||||
rope_head_dim,
|
||||
max_position=max_position,
|
||||
is_neox_style=True,
|
||||
rope_parameters={"rope_theta": base},
|
||||
dtype=dtype,
|
||||
)
|
||||
return apply_partial_rope(
|
||||
rotary_emb,
|
||||
positions,
|
||||
q,
|
||||
k,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
head_dim,
|
||||
rope_head_dim,
|
||||
fold_rope_weights=False,
|
||||
)
|
||||
|
||||
|
||||
def _apply_head_perm(
|
||||
x: torch.Tensor, num: int, head_dim: int, idx: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
return (
|
||||
x.reshape(*x.shape[:-1], num, head_dim)[..., idx].reshape(*x.shape).contiguous()
|
||||
)
|
||||
|
||||
|
||||
def get_neox_rope(
|
||||
positions: torch.Tensor,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
rope_head_dim: int,
|
||||
max_position: int,
|
||||
base: float,
|
||||
dtype: torch.dtype,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
rotary_emb = get_rope(
|
||||
head_dim,
|
||||
max_position=max_position,
|
||||
is_neox_style=True,
|
||||
rope_parameters={"rope_theta": base, "rope_dim": rope_head_dim},
|
||||
dtype=dtype,
|
||||
)
|
||||
return apply_partial_rope(
|
||||
rotary_emb,
|
||||
positions,
|
||||
q,
|
||||
k,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
head_dim,
|
||||
rope_head_dim,
|
||||
fold_rope_weights=True,
|
||||
)
|
||||
|
||||
|
||||
def fold_qk_proj_weight(
|
||||
weight: torch.Tensor, head_dim: int, idx: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
hidden = weight.shape[-1]
|
||||
return weight.view(-1, head_dim, hidden)[:, idx, :].reshape(-1, hidden).contiguous()
|
||||
|
||||
|
||||
def fold_qk_channels(
|
||||
vec: torch.Tensor, head_dim: int, idx: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
return vec.view(-1, head_dim)[:, idx].reshape(-1).contiguous()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("head_dim,rope_head_dim", SHAPES)
|
||||
def test_rope_fold_matches_permute_then_rope(
|
||||
head_dim, rope_head_dim, default_vllm_config
|
||||
):
|
||||
dtype = DTYPE
|
||||
set_random_seed(0)
|
||||
num_tokens = 17
|
||||
num_heads = 8
|
||||
num_kv_heads = 2
|
||||
hidden_size = 512
|
||||
max_position = 4096
|
||||
base = 10000.0
|
||||
|
||||
torch.set_default_dtype(dtype)
|
||||
|
||||
positions = torch.randint(0, max_position, (num_tokens,), device=DEVICE)
|
||||
|
||||
scale = hidden_size**-0.5
|
||||
x = torch.randn(num_tokens, hidden_size, dtype=dtype, device=DEVICE)
|
||||
w_q = (
|
||||
torch.randn(num_heads * head_dim, hidden_size, dtype=dtype, device=DEVICE)
|
||||
* scale
|
||||
)
|
||||
w_k = (
|
||||
torch.randn(num_kv_heads * head_dim, hidden_size, dtype=dtype, device=DEVICE)
|
||||
* scale
|
||||
)
|
||||
|
||||
idx = _rope_weight_perm(head_dim, rope_head_dim).to(DEVICE)
|
||||
|
||||
q = torch.nn.functional.linear(x, w_q)
|
||||
k = torch.nn.functional.linear(x, w_k)
|
||||
q_gptj, k_gptj = get_gptj_rope(
|
||||
positions,
|
||||
q,
|
||||
k,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
head_dim,
|
||||
rope_head_dim,
|
||||
max_position,
|
||||
base,
|
||||
dtype,
|
||||
)
|
||||
|
||||
q_folded = torch.nn.functional.linear(x, fold_qk_proj_weight(w_q, head_dim, idx))
|
||||
k_folded = torch.nn.functional.linear(x, fold_qk_proj_weight(w_k, head_dim, idx))
|
||||
q_neox, k_neox = get_neox_rope(
|
||||
positions,
|
||||
q_folded,
|
||||
k_folded,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
head_dim,
|
||||
rope_head_dim,
|
||||
max_position,
|
||||
base,
|
||||
dtype,
|
||||
)
|
||||
|
||||
q_gptj_perm = _apply_head_perm(q_gptj, num_heads, head_dim, idx)
|
||||
k_gptj_perm = _apply_head_perm(k_gptj, num_kv_heads, head_dim, idx)
|
||||
|
||||
atol, rtol = 1e-2, 1e-2
|
||||
|
||||
# Two complementary checks. First, an element-wise check that the folded
|
||||
# NeoX path equals the GPT-J path up to the channel permutation P
|
||||
# (neox_rope(Pq, Pk) == P @ gptj_rope(q, k)), hence permuting q_gptj/k_gptj.
|
||||
# Second, the scores check below: P cancels in q @ k.T.
|
||||
torch.testing.assert_close(q_neox, q_gptj_perm, atol=atol, rtol=rtol)
|
||||
torch.testing.assert_close(k_neox, k_gptj_perm, atol=atol, rtol=rtol)
|
||||
|
||||
def scores(qt, kt):
|
||||
qh = qt.reshape(num_tokens, num_heads, head_dim)
|
||||
kh = kt.reshape(num_tokens, num_kv_heads, head_dim)
|
||||
g = num_heads // num_kv_heads
|
||||
qh = qh.reshape(num_tokens, num_kv_heads, g, head_dim).float()
|
||||
kh = kh.float()
|
||||
return torch.einsum("tkgd,skd->tksg", qh, kh)
|
||||
|
||||
torch.testing.assert_close(
|
||||
scores(q_neox, k_neox), scores(q_gptj, k_gptj), atol=atol, rtol=rtol
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("head_dim,rope_head_dim", SHAPES)
|
||||
@pytest.mark.parametrize("num_heads", [8, 2])
|
||||
def test_qk_bias_fold_matches_permute(
|
||||
head_dim, rope_head_dim, num_heads, default_vllm_config
|
||||
):
|
||||
dtype = DTYPE
|
||||
set_random_seed(0)
|
||||
num_tokens = 17
|
||||
hidden_size = 512
|
||||
torch.set_default_dtype(dtype)
|
||||
|
||||
scale = hidden_size**-0.5
|
||||
x = torch.randn(num_tokens, hidden_size, dtype=dtype, device=DEVICE)
|
||||
w = (
|
||||
torch.randn(num_heads * head_dim, hidden_size, dtype=dtype, device=DEVICE)
|
||||
* scale
|
||||
)
|
||||
b = torch.randn(num_heads * head_dim, dtype=dtype, device=DEVICE)
|
||||
|
||||
idx = _rope_weight_perm(head_dim, rope_head_dim).to(DEVICE)
|
||||
|
||||
ref = _apply_head_perm(
|
||||
torch.nn.functional.linear(x, w, b), num_heads, head_dim, idx
|
||||
)
|
||||
folded = torch.nn.functional.linear(
|
||||
x,
|
||||
fold_qk_proj_weight(w, head_dim, idx),
|
||||
fold_qk_channels(b, head_dim, idx),
|
||||
)
|
||||
|
||||
atol, rtol = 1e-2, 1e-2
|
||||
torch.testing.assert_close(folded, ref, atol=atol, rtol=rtol)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a GPU (CUDA/ROCm)"
|
||||
)
|
||||
@pytest.mark.parametrize("head_dim,rope_head_dim", SHAPES)
|
||||
@pytest.mark.parametrize("num_heads", [8, 2])
|
||||
def test_qk_norm_scale_fold_matches_permute(
|
||||
head_dim, rope_head_dim, num_heads, default_vllm_config
|
||||
):
|
||||
dtype = DTYPE
|
||||
set_random_seed(0)
|
||||
num_tokens = 13
|
||||
torch.set_default_dtype(dtype)
|
||||
|
||||
hidden = num_heads * head_dim
|
||||
idx = _rope_weight_perm(head_dim, rope_head_dim).to(DEVICE)
|
||||
|
||||
x = torch.randn(num_tokens, hidden, dtype=dtype, device=DEVICE)
|
||||
|
||||
norm = K2HorizonRMSNorm(hidden_size=hidden, n_groups=num_heads).to(DEVICE)
|
||||
with torch.no_grad():
|
||||
norm.weight.copy_(torch.randn(hidden, dtype=dtype, device=DEVICE))
|
||||
|
||||
y_ref = _apply_head_perm(norm(x.clone()), num_heads, head_dim, idx)
|
||||
|
||||
folded_norm = K2HorizonRMSNorm(hidden_size=hidden, n_groups=num_heads).to(DEVICE)
|
||||
with torch.no_grad():
|
||||
folded_norm.weight.copy_(fold_qk_channels(norm.weight, head_dim, idx))
|
||||
# Permute x to emulate the folded q_proj/k_proj: at runtime the norm sees the
|
||||
# already-permuted projection output (P @ x), not the raw activations.
|
||||
y_folded = folded_norm(_apply_head_perm(x.clone(), num_heads, head_dim, idx))
|
||||
|
||||
atol, rtol = 1e-2, 1e-2
|
||||
torch.testing.assert_close(y_folded, y_ref, atol=atol, rtol=rtol)
|
||||
|
||||
|
||||
class _Fp8QKVOnlyConfig(Fp8Config):
|
||||
def get_quant_method(self, layer, prefix):
|
||||
if isinstance(layer, LinearBase) and not (
|
||||
prefix.endswith("qkv_proj") or prefix.endswith("qk_proj")
|
||||
):
|
||||
return UnquantizedLinearMethod()
|
||||
return super().get_quant_method(layer, prefix)
|
||||
|
||||
|
||||
class _DummyAttention(nn.Module):
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__()
|
||||
|
||||
def forward(self, q, k, v):
|
||||
return q
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dist_init():
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
init_distributed_environment(
|
||||
world_size=1,
|
||||
rank=0,
|
||||
distributed_init_method=f"tcp://127.0.0.1:{get_open_port()}",
|
||||
local_rank=0,
|
||||
backend="gloo",
|
||||
)
|
||||
initialize_model_parallel(1, 1)
|
||||
yield
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
def _make_hf_config(
|
||||
*,
|
||||
hidden_size: int,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
rope_head_dim: int,
|
||||
attention_bias: bool,
|
||||
query_key_norm: bool,
|
||||
) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
hidden_size=hidden_size,
|
||||
max_position_embeddings=4096,
|
||||
dual_chunk_attention_config=None,
|
||||
mlp_only_layers=[],
|
||||
num_experts=0, # dense layer: avoids the MoE path
|
||||
decoder_sparse_step=1,
|
||||
mova_num_experts=0,
|
||||
num_attention_heads=num_heads,
|
||||
num_key_value_heads=num_kv_heads,
|
||||
rope_parameters={"rope_theta": 10000.0},
|
||||
rope_head_dim=rope_head_dim,
|
||||
query_key_norm=query_key_norm,
|
||||
rms_norm_eps=1e-6,
|
||||
attention_bias=attention_bias,
|
||||
head_dim=head_dim,
|
||||
attention_gate_func=None,
|
||||
intermediate_size=2 * hidden_size,
|
||||
hidden_act="silu",
|
||||
layernorm_num_groups=1,
|
||||
vocab_size=1000,
|
||||
num_hidden_layers=1,
|
||||
tie_word_embeddings=False,
|
||||
)
|
||||
|
||||
|
||||
def _build_single_layer_model(
|
||||
monkeypatch, hf_config: SimpleNamespace, *, quant_config=None
|
||||
) -> K2HorizonModel:
|
||||
monkeypatch.setattr(k2_horizon, "Attention", _DummyAttention)
|
||||
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.model_config = SimpleNamespace(
|
||||
hf_text_config=hf_config, dtype=torch.get_default_dtype()
|
||||
)
|
||||
vllm_config.quant_config = quant_config
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
layer = K2HorizonDecoderLayer(vllm_config=vllm_config, prefix="model.layers.0")
|
||||
|
||||
model = K2HorizonModel.__new__(K2HorizonModel)
|
||||
nn.Module.__init__(model)
|
||||
model.config = hf_config
|
||||
model.quant_config = quant_config
|
||||
model.num_redundant_experts = 0
|
||||
model.layers = nn.ModuleList([layer])
|
||||
return model
|
||||
|
||||
|
||||
def _fold_rows(weight: torch.Tensor, head_dim: int, idx: torch.Tensor) -> torch.Tensor:
|
||||
hidden = weight.shape[-1]
|
||||
return weight.view(-1, head_dim, hidden)[:, idx, :].reshape(-1, hidden).contiguous()
|
||||
|
||||
|
||||
def _fold_channels(vec: torch.Tensor, head_dim: int, idx: torch.Tensor) -> torch.Tensor:
|
||||
return vec.view(-1, head_dim)[:, idx].reshape(-1).contiguous()
|
||||
|
||||
|
||||
def _non_qk_weights(prefix, *, hidden_size, intermediate_size, q_size, device=None):
|
||||
return [
|
||||
(
|
||||
f"{prefix}.self_attn.o_proj.weight",
|
||||
torch.randn(hidden_size, q_size, device=device),
|
||||
),
|
||||
(
|
||||
f"{prefix}.mlp.gate_proj.weight",
|
||||
torch.randn(intermediate_size, hidden_size, device=device),
|
||||
),
|
||||
(
|
||||
f"{prefix}.mlp.up_proj.weight",
|
||||
torch.randn(intermediate_size, hidden_size, device=device),
|
||||
),
|
||||
(
|
||||
f"{prefix}.mlp.down_proj.weight",
|
||||
torch.randn(hidden_size, intermediate_size, device=device),
|
||||
),
|
||||
(
|
||||
f"{prefix}.input_layernorm.weight",
|
||||
torch.randn(hidden_size, device=device),
|
||||
),
|
||||
(
|
||||
f"{prefix}.post_attention_layernorm.weight",
|
||||
torch.randn(hidden_size, device=device),
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("dist_init")
|
||||
@pytest.mark.parametrize("head_dim,rope_head_dim", SHAPES)
|
||||
@pytest.mark.parametrize("attention_bias", [False, True])
|
||||
def test_load_weights_dense_folds_qk(
|
||||
head_dim,
|
||||
rope_head_dim,
|
||||
attention_bias,
|
||||
monkeypatch,
|
||||
):
|
||||
dtype = DTYPE
|
||||
set_random_seed(0)
|
||||
num_heads = 8
|
||||
num_kv_heads = 2
|
||||
hidden_size = 128
|
||||
intermediate_size = 2 * hidden_size
|
||||
q_size = num_heads * head_dim
|
||||
kv_size = num_kv_heads * head_dim
|
||||
prefix = "layers.0"
|
||||
|
||||
with set_default_torch_dtype(dtype):
|
||||
hf_config = _make_hf_config(
|
||||
hidden_size=hidden_size,
|
||||
num_heads=num_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_dim=head_dim,
|
||||
rope_head_dim=rope_head_dim,
|
||||
attention_bias=attention_bias,
|
||||
query_key_norm=True,
|
||||
)
|
||||
model = _build_single_layer_model(monkeypatch, hf_config, quant_config=None)
|
||||
|
||||
assert model.layers[0].self_attn.fold_rope_weights is True
|
||||
|
||||
raw = {
|
||||
"q_proj.weight": torch.randn(q_size, hidden_size),
|
||||
"k_proj.weight": torch.randn(kv_size, hidden_size),
|
||||
"v_proj.weight": torch.randn(kv_size, hidden_size),
|
||||
"q_norm.weight": torch.randn(q_size),
|
||||
"k_norm.weight": torch.randn(kv_size),
|
||||
}
|
||||
if attention_bias:
|
||||
raw.update(
|
||||
{
|
||||
"q_proj.bias": torch.randn(q_size),
|
||||
"k_proj.bias": torch.randn(kv_size),
|
||||
"v_proj.bias": torch.randn(kv_size),
|
||||
"o_proj.bias": torch.randn(hidden_size),
|
||||
}
|
||||
)
|
||||
|
||||
weights: list[tuple[str, torch.Tensor]] = [
|
||||
(f"{prefix}.self_attn.{name}", tensor) for name, tensor in raw.items()
|
||||
]
|
||||
weights += _non_qk_weights(
|
||||
prefix,
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size=intermediate_size,
|
||||
q_size=q_size,
|
||||
)
|
||||
|
||||
loaded = model.load_weights(iter(weights))
|
||||
|
||||
params = dict(model.named_parameters())
|
||||
assert loaded == set(params)
|
||||
|
||||
idx = _rope_weight_perm(head_dim, rope_head_dim)
|
||||
|
||||
qkv = params[f"{prefix}.self_attn.qkv_proj.weight"]
|
||||
q_part = qkv[:q_size]
|
||||
k_part = qkv[q_size : q_size + kv_size]
|
||||
v_part = qkv[q_size + kv_size :]
|
||||
|
||||
q_norm = params[f"{prefix}.self_attn.q_norm.weight"]
|
||||
k_norm = params[f"{prefix}.self_attn.k_norm.weight"]
|
||||
|
||||
assert torch.equal(q_part, _fold_rows(raw["q_proj.weight"], head_dim, idx))
|
||||
assert torch.equal(k_part, _fold_rows(raw["k_proj.weight"], head_dim, idx))
|
||||
assert torch.equal(v_part, raw["v_proj.weight"]) # v never folded
|
||||
assert torch.equal(q_norm, _fold_channels(raw["q_norm.weight"], head_dim, idx))
|
||||
assert torch.equal(k_norm, _fold_channels(raw["k_norm.weight"], head_dim, idx))
|
||||
assert qkv.dtype == dtype
|
||||
|
||||
if attention_bias:
|
||||
qkv_bias = params[f"{prefix}.self_attn.qkv_proj.bias"]
|
||||
q_bias = qkv_bias[:q_size]
|
||||
k_bias = qkv_bias[q_size : q_size + kv_size]
|
||||
v_bias = qkv_bias[q_size + kv_size :]
|
||||
assert torch.equal(q_bias, _fold_channels(raw["q_proj.bias"], head_dim, idx))
|
||||
assert torch.equal(k_bias, _fold_channels(raw["k_proj.bias"], head_dim, idx))
|
||||
assert torch.equal(v_bias, raw["v_proj.bias"]) # v bias never folded
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("fp8"),
|
||||
reason="fp8 is not supported.",
|
||||
)
|
||||
@pytest.mark.usefixtures("dist_init")
|
||||
@pytest.mark.parametrize("head_dim,rope_head_dim", SHAPES)
|
||||
@pytest.mark.parametrize("attention_bias", [False, True])
|
||||
def test_load_weights_fp8_qk_no_fold(
|
||||
head_dim,
|
||||
rope_head_dim,
|
||||
attention_bias,
|
||||
monkeypatch,
|
||||
):
|
||||
set_random_seed(0)
|
||||
num_heads = 8
|
||||
num_kv_heads = 2
|
||||
hidden_size = 128
|
||||
intermediate_size = 2 * hidden_size
|
||||
q_size = num_heads * head_dim
|
||||
kv_size = num_kv_heads * head_dim
|
||||
prefix = "layers.0"
|
||||
|
||||
def rand_fp8(*shape):
|
||||
return (torch.randn(*shape, device=DEVICE) * 32.0).to(torch.float8_e4m3fn)
|
||||
|
||||
def rand_scale():
|
||||
return torch.rand(1, device=DEVICE, dtype=torch.float32) + 0.1
|
||||
|
||||
with set_default_torch_dtype(torch.bfloat16), torch.device(DEVICE):
|
||||
hf_config = _make_hf_config(
|
||||
hidden_size=hidden_size,
|
||||
num_heads=num_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_dim=head_dim,
|
||||
rope_head_dim=rope_head_dim,
|
||||
attention_bias=attention_bias,
|
||||
query_key_norm=True,
|
||||
)
|
||||
quant_config = _Fp8QKVOnlyConfig(
|
||||
is_checkpoint_fp8_serialized=True, activation_scheme="dynamic"
|
||||
)
|
||||
model = _build_single_layer_model(
|
||||
monkeypatch, hf_config, quant_config=quant_config
|
||||
)
|
||||
|
||||
qkv_proj = model.layers[0].self_attn.qkv_proj
|
||||
assert isinstance(qkv_proj.quant_method, Fp8LinearMethod)
|
||||
assert model.layers[0].self_attn.fold_rope_weights is False
|
||||
|
||||
raw_w = {
|
||||
"q_proj.weight": rand_fp8(q_size, hidden_size),
|
||||
"k_proj.weight": rand_fp8(kv_size, hidden_size),
|
||||
"v_proj.weight": rand_fp8(kv_size, hidden_size),
|
||||
}
|
||||
raw_s = {
|
||||
"q_proj.weight_scale": rand_scale(),
|
||||
"k_proj.weight_scale": rand_scale(),
|
||||
"v_proj.weight_scale": rand_scale(),
|
||||
}
|
||||
q_norm = torch.randn(q_size, device=DEVICE)
|
||||
k_norm = torch.randn(kv_size, device=DEVICE)
|
||||
|
||||
weights: list[tuple[str, torch.Tensor]] = [
|
||||
(f"{prefix}.self_attn.{name}", tensor)
|
||||
for name, tensor in {**raw_w, **raw_s}.items()
|
||||
]
|
||||
weights += [
|
||||
(f"{prefix}.self_attn.q_norm.weight", q_norm),
|
||||
(f"{prefix}.self_attn.k_norm.weight", k_norm),
|
||||
]
|
||||
biases = {}
|
||||
if attention_bias:
|
||||
biases = {
|
||||
"q_proj.bias": torch.randn(q_size, device=DEVICE),
|
||||
"k_proj.bias": torch.randn(kv_size, device=DEVICE),
|
||||
"v_proj.bias": torch.randn(kv_size, device=DEVICE),
|
||||
"o_proj.bias": torch.randn(hidden_size, device=DEVICE),
|
||||
}
|
||||
weights += [
|
||||
(f"{prefix}.self_attn.{name}", tensor)
|
||||
for name, tensor in biases.items()
|
||||
]
|
||||
weights += _non_qk_weights(
|
||||
prefix,
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size=intermediate_size,
|
||||
q_size=q_size,
|
||||
device=DEVICE,
|
||||
)
|
||||
|
||||
loaded = model.load_weights(iter(weights))
|
||||
|
||||
params = dict(model.named_parameters())
|
||||
assert loaded == set(params)
|
||||
|
||||
qkv = params[f"{prefix}.self_attn.qkv_proj.weight"]
|
||||
weight_scale = params[f"{prefix}.self_attn.qkv_proj.weight_scale"]
|
||||
|
||||
assert qkv.dtype == torch.float8_e4m3fn
|
||||
assert torch.equal(qkv[:q_size], raw_w["q_proj.weight"])
|
||||
assert torch.equal(qkv[q_size : q_size + kv_size], raw_w["k_proj.weight"])
|
||||
assert torch.equal(qkv[q_size + kv_size :], raw_w["v_proj.weight"])
|
||||
|
||||
assert torch.equal(
|
||||
weight_scale,
|
||||
torch.cat(
|
||||
[
|
||||
raw_s["q_proj.weight_scale"],
|
||||
raw_s["k_proj.weight_scale"],
|
||||
raw_s["v_proj.weight_scale"],
|
||||
]
|
||||
),
|
||||
)
|
||||
|
||||
assert torch.equal(params[f"{prefix}.self_attn.q_norm.weight"], q_norm)
|
||||
assert torch.equal(params[f"{prefix}.self_attn.k_norm.weight"], k_norm)
|
||||
|
||||
if attention_bias:
|
||||
qkv_bias = params[f"{prefix}.self_attn.qkv_proj.bias"]
|
||||
assert torch.equal(qkv_bias[:q_size], biases["q_proj.bias"])
|
||||
assert torch.equal(qkv_bias[q_size : q_size + kv_size], biases["k_proj.bias"])
|
||||
assert torch.equal(qkv_bias[q_size + kv_size :], biases["v_proj.bias"])
|
||||
|
||||
|
||||
def test_is_rope_weights_folding_supported(default_vllm_config):
|
||||
dense = SimpleNamespace(quant_method=UnquantizedLinearMethod())
|
||||
quantized = SimpleNamespace(quant_method=object())
|
||||
|
||||
assert is_rope_weights_folding_supported(dense, None) is True
|
||||
assert is_rope_weights_folding_supported(quantized, None) is False
|
||||
assert is_rope_weights_folding_supported(dense, {"chunk_size": 8}) is False
|
||||
assert is_rope_weights_folding_supported(SimpleNamespace(), None) is False
|
||||
@@ -1,145 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.keye import KeyeForConditionalGeneration
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldElem,
|
||||
MultiModalKwargsItem,
|
||||
PlaceholderRange,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True, scope="module")
|
||||
def _force_cpu_default_device():
|
||||
original = torch.get_default_device()
|
||||
torch.set_default_device("cpu")
|
||||
yield
|
||||
torch.set_default_device(original)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DummyVisionConfig:
|
||||
spatial_merge_size: int = 2
|
||||
|
||||
|
||||
@dataclass
|
||||
class DummyConfig:
|
||||
vision_config: DummyVisionConfig = field(default_factory=DummyVisionConfig)
|
||||
|
||||
|
||||
def make_model(config: DummyConfig) -> KeyeForConditionalGeneration:
|
||||
model = object.__new__(KeyeForConditionalGeneration)
|
||||
model.config = config
|
||||
return model
|
||||
|
||||
|
||||
def make_mm_feature(
|
||||
*,
|
||||
modality: str,
|
||||
offset: int,
|
||||
length: int,
|
||||
grid_thw: tuple[int, int, int] | list[tuple[int, int, int]],
|
||||
) -> MultiModalFeatureSpec:
|
||||
field_name = "image_grid_thw" if modality == "image" else "video_grid_thw"
|
||||
return MultiModalFeatureSpec(
|
||||
data=MultiModalKwargsItem(
|
||||
{
|
||||
field_name: MultiModalFieldElem(
|
||||
data=torch.tensor(grid_thw),
|
||||
field=None, # HACK.
|
||||
),
|
||||
}
|
||||
),
|
||||
modality=modality,
|
||||
identifier="DUMMY",
|
||||
mm_position=PlaceholderRange(offset=offset, length=length),
|
||||
)
|
||||
|
||||
|
||||
def test_get_mrope_input_positions_text_only():
|
||||
model = make_model(DummyConfig())
|
||||
|
||||
positions, delta = model.get_mrope_input_positions(
|
||||
input_tokens=[11, 12, 13, 14, 15],
|
||||
mm_features=[],
|
||||
)
|
||||
|
||||
expected = torch.tensor(
|
||||
[
|
||||
[0, 1, 2, 3, 4],
|
||||
[0, 1, 2, 3, 4],
|
||||
[0, 1, 2, 3, 4],
|
||||
]
|
||||
)
|
||||
|
||||
assert torch.equal(positions, expected)
|
||||
assert delta == 0
|
||||
|
||||
|
||||
def test_get_mrope_input_positions_single_image():
|
||||
model = make_model(DummyConfig())
|
||||
mm_features = [
|
||||
make_mm_feature(
|
||||
modality="image",
|
||||
offset=1,
|
||||
length=4,
|
||||
grid_thw=(1, 4, 4),
|
||||
)
|
||||
]
|
||||
|
||||
positions, delta = model.get_mrope_input_positions(
|
||||
input_tokens=[10, 20, 21, 22, 23, 30, 31],
|
||||
mm_features=mm_features,
|
||||
)
|
||||
|
||||
expected = torch.tensor(
|
||||
[
|
||||
[0, 1, 1, 1, 1, 3, 4],
|
||||
[0, 1, 1, 2, 2, 3, 4],
|
||||
[0, 1, 2, 1, 2, 3, 4],
|
||||
]
|
||||
)
|
||||
|
||||
assert torch.equal(positions, expected)
|
||||
assert delta == -2
|
||||
|
||||
|
||||
def test_get_mrope_input_positions_interleaved_image_and_video():
|
||||
model = make_model(DummyConfig())
|
||||
mm_features = [
|
||||
make_mm_feature(
|
||||
modality="image",
|
||||
offset=1,
|
||||
length=4,
|
||||
grid_thw=(1, 4, 4),
|
||||
),
|
||||
make_mm_feature(
|
||||
modality="video",
|
||||
offset=7,
|
||||
length=4,
|
||||
grid_thw=[(2, 4, 2)],
|
||||
),
|
||||
]
|
||||
|
||||
positions, delta = model.get_mrope_input_positions(
|
||||
input_tokens=[10, 20, 21, 22, 23, 30, 31, 40, 41, 42, 43, 50, 51],
|
||||
mm_features=mm_features,
|
||||
)
|
||||
|
||||
expected = torch.tensor(
|
||||
[
|
||||
[0, 1, 1, 1, 1, 3, 4, 5, 5, 7, 7, 9, 10],
|
||||
[0, 1, 1, 2, 2, 3, 4, 5, 6, 7, 8, 9, 10],
|
||||
[0, 1, 2, 1, 2, 3, 4, 5, 5, 7, 7, 9, 10],
|
||||
]
|
||||
)
|
||||
|
||||
assert torch.equal(positions, expected)
|
||||
assert delta == -2
|
||||
@@ -1,145 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.keye_vl1_5 import KeyeVL1_5ForConditionalGeneration
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldElem,
|
||||
MultiModalKwargsItem,
|
||||
PlaceholderRange,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True, scope="module")
|
||||
def _force_cpu_default_device():
|
||||
original = torch.get_default_device()
|
||||
torch.set_default_device("cpu")
|
||||
yield
|
||||
torch.set_default_device(original)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DummyVisionConfig:
|
||||
spatial_merge_size: int = 2
|
||||
|
||||
|
||||
@dataclass
|
||||
class DummyConfig:
|
||||
vision_config: DummyVisionConfig = field(default_factory=DummyVisionConfig)
|
||||
|
||||
|
||||
def make_model(config: DummyConfig) -> KeyeVL1_5ForConditionalGeneration:
|
||||
model = object.__new__(KeyeVL1_5ForConditionalGeneration)
|
||||
model.config = config
|
||||
return model
|
||||
|
||||
|
||||
def make_mm_feature(
|
||||
*,
|
||||
modality: str,
|
||||
offset: int,
|
||||
length: int,
|
||||
grid_thw: tuple[int, int, int] | list[tuple[int, int, int]],
|
||||
is_embed: list[bool] | None = None,
|
||||
) -> MultiModalFeatureSpec:
|
||||
field_name = "image_grid_thw" if modality == "image" else "video_grid_thw"
|
||||
return MultiModalFeatureSpec(
|
||||
data=MultiModalKwargsItem(
|
||||
{
|
||||
field_name: MultiModalFieldElem(
|
||||
data=torch.tensor(grid_thw),
|
||||
field=None, # HACK.
|
||||
),
|
||||
}
|
||||
),
|
||||
modality=modality,
|
||||
identifier="DUMMY",
|
||||
mm_position=PlaceholderRange(
|
||||
offset=offset,
|
||||
length=length,
|
||||
is_embed=None if is_embed is None else torch.tensor(is_embed),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_get_mrope_input_positions_text_only():
|
||||
model = make_model(DummyConfig())
|
||||
|
||||
positions, delta = model.get_mrope_input_positions(
|
||||
input_tokens=[11, 12, 13, 14, 15],
|
||||
mm_features=[],
|
||||
)
|
||||
|
||||
expected = torch.tensor(
|
||||
[
|
||||
[0, 1, 2, 3, 4],
|
||||
[0, 1, 2, 3, 4],
|
||||
[0, 1, 2, 3, 4],
|
||||
]
|
||||
)
|
||||
|
||||
assert torch.equal(positions, expected)
|
||||
assert delta == 0
|
||||
|
||||
|
||||
def test_get_mrope_input_positions_single_image():
|
||||
model = make_model(DummyConfig())
|
||||
mm_features = [
|
||||
make_mm_feature(
|
||||
modality="image",
|
||||
offset=1,
|
||||
length=4,
|
||||
grid_thw=(1, 4, 4),
|
||||
)
|
||||
]
|
||||
|
||||
positions, delta = model.get_mrope_input_positions(
|
||||
input_tokens=[10, 20, 21, 22, 23, 30, 31],
|
||||
mm_features=mm_features,
|
||||
)
|
||||
|
||||
expected = torch.tensor(
|
||||
[
|
||||
[0, 1, 1, 1, 1, 3, 4],
|
||||
[0, 1, 1, 2, 2, 3, 4],
|
||||
[0, 1, 2, 1, 2, 3, 4],
|
||||
]
|
||||
)
|
||||
|
||||
assert torch.equal(positions, expected)
|
||||
assert delta == -2
|
||||
|
||||
|
||||
def test_get_mrope_input_positions_video_uses_embed_ranges():
|
||||
model = make_model(DummyConfig())
|
||||
mm_features = [
|
||||
make_mm_feature(
|
||||
modality="video",
|
||||
offset=1,
|
||||
length=8,
|
||||
grid_thw=[(2, 4, 2)],
|
||||
is_embed=[False, False, True, True, False, False, True, True],
|
||||
)
|
||||
]
|
||||
|
||||
positions, delta = model.get_mrope_input_positions(
|
||||
input_tokens=[10, 101, 102, 20, 21, 103, 104, 30, 31, 40, 41],
|
||||
mm_features=mm_features,
|
||||
)
|
||||
|
||||
expected = torch.tensor(
|
||||
[
|
||||
[0, 1, 2, 3, 3, 5, 6, 7, 7, 9, 10],
|
||||
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
|
||||
[0, 1, 2, 3, 3, 5, 6, 7, 7, 9, 10],
|
||||
]
|
||||
)
|
||||
|
||||
assert torch.equal(positions, expected)
|
||||
assert delta == 0
|
||||
@@ -1,170 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.minicpmv import MiniCPMV2_6
|
||||
from vllm.model_executor.models.minicpmv4_6 import (
|
||||
MiniCPMV4_6ForConditionalGeneration,
|
||||
_expand_flags,
|
||||
_select_vision_items,
|
||||
_vision_downsample_mode,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
|
||||
def make_model() -> MiniCPMV2_6:
|
||||
return object.__new__(MiniCPMV2_6)
|
||||
|
||||
|
||||
def make_model_4_6() -> MiniCPMV4_6ForConditionalGeneration:
|
||||
model = object.__new__(MiniCPMV4_6ForConditionalGeneration)
|
||||
model._process_vision_input = lambda vision_input, use_vit_merger=None: [
|
||||
vision_input["image_embeds"]
|
||||
]
|
||||
return model
|
||||
|
||||
|
||||
def test_video_embeds_reach_the_vision_parser():
|
||||
embeds = torch.arange(32, dtype=torch.float32).reshape(1, 4, 8)
|
||||
|
||||
modalities = make_model()._parse_and_validate_multimodal_inputs(video_embeds=embeds)
|
||||
|
||||
video_input = modalities["videos"]
|
||||
assert video_input is not None
|
||||
assert video_input["type"] == "image_embeds"
|
||||
assert torch.equal(video_input["image_embeds"], embeds)
|
||||
|
||||
|
||||
def test_video_embeds_are_embedded_by_4_6():
|
||||
embeds = torch.arange(32, dtype=torch.float32).reshape(1, 4, 8)
|
||||
|
||||
embeddings = make_model_4_6().embed_multimodal(video_embeds=embeds)
|
||||
|
||||
assert len(embeddings) == 1
|
||||
assert torch.equal(embeddings[0], embeds)
|
||||
|
||||
|
||||
def test_image_and_video_embeds_stay_in_their_own_modality():
|
||||
image_embeds = torch.zeros(1, 4, 8)
|
||||
video_embeds = torch.ones(1, 4, 8)
|
||||
|
||||
modalities = make_model()._parse_and_validate_multimodal_inputs(
|
||||
image_embeds=image_embeds,
|
||||
video_embeds=video_embeds,
|
||||
)
|
||||
|
||||
assert torch.equal(modalities["images"]["image_embeds"], image_embeds)
|
||||
assert torch.equal(modalities["videos"]["image_embeds"], video_embeds)
|
||||
|
||||
|
||||
class TestPerItemMergerFlags:
|
||||
"""The merger flag is per request but items from several requests can be
|
||||
batched together, so it has to be applied per item."""
|
||||
|
||||
def test_expand_flags_broadcasts_a_single_value(self):
|
||||
assert _expand_flags(None, 3) == [None, None, None]
|
||||
assert _expand_flags(torch.tensor([True]), 3) == [True, True, True]
|
||||
assert _expand_flags(True, 2) == [True, True]
|
||||
|
||||
def test_expand_flags_keeps_per_item_values(self):
|
||||
flags = [torch.tensor([True]), torch.tensor([False])]
|
||||
|
||||
assert _expand_flags(flags, 2) == [True, False]
|
||||
|
||||
def test_expand_flags_falls_back_on_a_shape_mismatch(self):
|
||||
# Too many values to map onto the items: keep one decision for the
|
||||
# batch rather than failing the request.
|
||||
assert _expand_flags([True, False, True], 2) == [True, True]
|
||||
|
||||
def test_vision_downsample_mode(self):
|
||||
assert _vision_downsample_mode(None) is None
|
||||
assert _vision_downsample_mode(True) == "16x"
|
||||
assert _vision_downsample_mode(False) == "4x"
|
||||
|
||||
def test_select_vision_items_slices_by_item(self):
|
||||
# 3 items holding 2, 1 and 2 slices; one channel, one patch, width 4.
|
||||
pixel_values = [torch.full((1, 1, 4), float(i)) for i in range(5)]
|
||||
tgt_sizes = torch.arange(10, dtype=torch.long).reshape(5, 2)
|
||||
image_input = {
|
||||
"pixel_values": pixel_values,
|
||||
"tgt_sizes": tgt_sizes,
|
||||
"num_slices": torch.tensor([2, 1, 2]),
|
||||
}
|
||||
|
||||
selected = _select_vision_items(image_input, [2], [2, 1, 2])
|
||||
|
||||
assert len(selected["pixel_values"]) == 2
|
||||
assert bool(selected["pixel_values"][0][0, 0, 0] == 3.0)
|
||||
assert bool(selected["pixel_values"][1][0, 0, 0] == 4.0)
|
||||
assert selected["num_slices"].tolist() == [2]
|
||||
assert torch.equal(selected["tgt_sizes"], tgt_sizes[3:5])
|
||||
|
||||
def test_select_vision_items_preserves_order(self):
|
||||
pixel_values = [torch.full((1, 1, 4), float(i)) for i in range(3)]
|
||||
image_input = {
|
||||
"pixel_values": pixel_values,
|
||||
"tgt_sizes": torch.zeros(3, 2, dtype=torch.long),
|
||||
"num_slices": torch.tensor([1, 1, 1]),
|
||||
}
|
||||
|
||||
selected = _select_vision_items(image_input, [2, 0], [1, 1, 1])
|
||||
|
||||
assert selected["num_slices"].tolist() == [1, 1]
|
||||
assert bool(selected["pixel_values"][0][0, 0, 0] == 2.0)
|
||||
assert bool(selected["pixel_values"][1][0, 0, 0] == 0.0)
|
||||
|
||||
def test_process_vision_input_applies_flags_per_item(self):
|
||||
calls: list[str | None] = []
|
||||
|
||||
def fake_get_vision_hidden_states(data, downsample_mode=None):
|
||||
calls.append(downsample_mode)
|
||||
tag = 1.0 if downsample_mode == "16x" else 0.0
|
||||
return [torch.full((1,), tag) for _ in data["pixel_values"]]
|
||||
|
||||
model = object.__new__(MiniCPMV4_6ForConditionalGeneration)
|
||||
model.get_vision_hidden_states = fake_get_vision_hidden_states
|
||||
|
||||
image_input = {
|
||||
"type": "pixel_values",
|
||||
"pixel_values": [torch.zeros(1, 1, 4) for _ in range(3)],
|
||||
"tgt_sizes": torch.zeros(3, 2, dtype=torch.long),
|
||||
"num_slices": torch.tensor([1, 1, 1]),
|
||||
}
|
||||
|
||||
result = model._process_vision_input(
|
||||
image_input,
|
||||
use_vit_merger=[
|
||||
torch.tensor([True]),
|
||||
torch.tensor([False]),
|
||||
torch.tensor([True]),
|
||||
],
|
||||
)
|
||||
|
||||
# Each item keeps its own mode even though they share one call.
|
||||
assert [float(r[0]) for r in result] == [1.0, 0.0, 1.0]
|
||||
# Mixed flags are encoded as separate groups, not one collapsed batch.
|
||||
assert calls == ["16x", "4x"]
|
||||
|
||||
def test_process_vision_input_uses_one_group_for_uniform_flags(self):
|
||||
calls: list[str | None] = []
|
||||
|
||||
def fake_get_vision_hidden_states(data, downsample_mode=None):
|
||||
calls.append(downsample_mode)
|
||||
return [torch.zeros(1) for _ in data["pixel_values"]]
|
||||
|
||||
model = object.__new__(MiniCPMV4_6ForConditionalGeneration)
|
||||
model.get_vision_hidden_states = fake_get_vision_hidden_states
|
||||
|
||||
image_input = {
|
||||
"type": "pixel_values",
|
||||
"pixel_values": [torch.zeros(1, 1, 4) for _ in range(2)],
|
||||
"tgt_sizes": torch.zeros(2, 2, dtype=torch.long),
|
||||
"num_slices": torch.tensor([1, 1]),
|
||||
}
|
||||
|
||||
model._process_vision_input(image_input, use_vit_merger=torch.tensor([True]))
|
||||
|
||||
# The common case still makes a single encoder call.
|
||||
assert calls == ["16x"]
|
||||
@@ -1,310 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.minicpmv4_6 import _stack_vit_merger_qkv
|
||||
from vllm.model_executor.models.minicpmv4_7 import (
|
||||
MiniCPMV4_7MultiModalProcessor,
|
||||
_compute_canvas_single,
|
||||
build_image_bounds,
|
||||
canvas_rope_delta,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
|
||||
def test_canvas_mrope_sizes_survive_prefix_cache_stripping():
|
||||
"""A cached image must keep `tgt_sizes`, or canvas M-RoPE falls back.
|
||||
|
||||
`strip_covered_mm_data` drops the payload of a prefix-cache-covered item and
|
||||
keeps only CPU-side metadata. Canvas M-RoPE places slices from `tgt_sizes`,
|
||||
so 4.7 marks it `keep_on_cpu` while the shared 4.6 config stays as it was.
|
||||
"""
|
||||
from vllm.model_executor.models.minicpmv4_6 import _minicpmv4_6_field_config
|
||||
|
||||
shared = _minicpmv4_6_field_config({})
|
||||
assert shared["tgt_sizes"].field.keep_on_cpu is False
|
||||
assert shared["video_tgt_sizes"].field.keep_on_cpu is False
|
||||
|
||||
processor = MiniCPMV4_7MultiModalProcessor.__new__(MiniCPMV4_7MultiModalProcessor)
|
||||
fields = processor._get_mm_fields_config({}, {})
|
||||
|
||||
assert fields["tgt_sizes"].field.keep_on_cpu is True
|
||||
assert fields["video_tgt_sizes"].field.keep_on_cpu is True
|
||||
# Other fields keep their previous setting.
|
||||
assert fields["pixel_values"].field.keep_on_cpu is False
|
||||
assert fields["video_image_sizes"].field.keep_on_cpu is True
|
||||
|
||||
|
||||
def test_modality_mm_kwargs_resolution():
|
||||
"""Flat keys reach both modalities; scopes stay scoped, and win on conflict.
|
||||
|
||||
`_merge_and_resolve_mm_processor_kwargs` routes a flat key into every scope
|
||||
whose HF processor declares it, so reading is a plain scope lookup. The
|
||||
routing itself is upstream's, tested here against the MiniCPM-V schema to
|
||||
pin the precedence rule (`scoped > flat`).
|
||||
"""
|
||||
from vllm.model_executor.models.minicpmv4_6 import (
|
||||
_flat_processor_kwargs,
|
||||
_scoped_mm_kwarg,
|
||||
)
|
||||
from vllm.multimodal.processing.context import _resolve_mm_processor_kwargs
|
||||
|
||||
# What MiniCPMV4_6{Image,Video}ProcessorKwargs declare.
|
||||
processor_keys = {"downsample_mode", "max_slice_nums"}
|
||||
schema = {
|
||||
"text_kwargs": set(),
|
||||
"images_kwargs": set(processor_keys),
|
||||
"videos_kwargs": set(processor_keys),
|
||||
"audio_kwargs": set(),
|
||||
}
|
||||
|
||||
def read(mm_kwargs, modality, key):
|
||||
merged = _resolve_mm_processor_kwargs(dict(mm_kwargs), schema)
|
||||
return _scoped_mm_kwarg(merged, modality, key)
|
||||
|
||||
assert read({}, "image", "downsample_mode") is None
|
||||
# A flat key is routed into both scopes.
|
||||
assert read({"downsample_mode": "4x"}, "image", "downsample_mode") == "4x"
|
||||
assert read({"downsample_mode": "4x"}, "video", "downsample_mode") == "4x"
|
||||
# A scoped key stays in its scope.
|
||||
assert (
|
||||
read({"images_kwargs": {"downsample_mode": "16x"}}, "image", "downsample_mode")
|
||||
== "16x"
|
||||
)
|
||||
assert (
|
||||
read({"images_kwargs": {"downsample_mode": "16x"}}, "video", "downsample_mode")
|
||||
is None
|
||||
)
|
||||
videos_only = {"videos_kwargs": {"max_slice_nums": 1}}
|
||||
assert read(videos_only, "video", "max_slice_nums") == 1
|
||||
# Scoped to videos, so image must not see it.
|
||||
assert read(videos_only, "image", "max_slice_nums") is None
|
||||
# An explicit scope wins over the flat key it is merged with.
|
||||
assert (
|
||||
read(
|
||||
{"downsample_mode": "16x", "images_kwargs": {"downsample_mode": "4x"}},
|
||||
"image",
|
||||
"downsample_mode",
|
||||
)
|
||||
== "4x"
|
||||
)
|
||||
# A non-mapping scope is ignored rather than raising.
|
||||
assert read({"images_kwargs": None}, "image", "downsample_mode") is None
|
||||
|
||||
# `_flat_processor_kwargs` drops the scoped keys and pins the mode.
|
||||
flat = _flat_processor_kwargs(
|
||||
{
|
||||
"downsample_mode": "4x",
|
||||
"images_kwargs": {"a": 1},
|
||||
"videos_kwargs": {"b": 2},
|
||||
"audio_kwargs": {"c": 3},
|
||||
},
|
||||
"16x",
|
||||
)
|
||||
assert flat == {"downsample_mode": "16x"}
|
||||
|
||||
|
||||
def test_vit_merger_qkv_is_stacked_before_load():
|
||||
mapped = list(
|
||||
_stack_vit_merger_qkv(
|
||||
[
|
||||
(
|
||||
"model.vision_tower.vit_merger.self_attn.q_proj.weight",
|
||||
torch.zeros(1),
|
||||
),
|
||||
(
|
||||
"model.vision_tower.vit_merger.self_attn.k_proj.weight",
|
||||
torch.zeros(1),
|
||||
),
|
||||
(
|
||||
"model.vision_tower.vit_merger.self_attn.v_proj.weight",
|
||||
torch.zeros(1),
|
||||
),
|
||||
("vit_merger.self_attn.qkv_proj.weight", torch.zeros(1)),
|
||||
]
|
||||
)
|
||||
)
|
||||
names = [name for name, _ in mapped]
|
||||
shards = [getattr(tensor, "shard_id", None) for _, tensor in mapped]
|
||||
assert names == [
|
||||
"model.vision_tower.vit_merger.self_attn.qkv_proj.weight",
|
||||
"model.vision_tower.vit_merger.self_attn.qkv_proj.weight",
|
||||
"model.vision_tower.vit_merger.self_attn.qkv_proj.weight",
|
||||
"vit_merger.self_attn.qkv_proj.weight",
|
||||
]
|
||||
assert shards == ["q", "k", "v", None]
|
||||
|
||||
|
||||
# Canvas M-RoPE goldens, cross-checked against the reference implementation in
|
||||
# transformers PR 48979 (`modular_minicpmv4_7.py`). Synthetic ids keep the token
|
||||
# sequences readable: the canvas math only compares ids for equality.
|
||||
_IM_START = 10
|
||||
_IM_END = 11
|
||||
_SLICE_START = 12
|
||||
_SLICE_END = 13
|
||||
_NEWLINE = 14
|
||||
_IMAGE_PAD = 20
|
||||
_VIDEO_PAD = 21
|
||||
_TEXT = 100
|
||||
|
||||
_CANVAS_IDS = {
|
||||
"im_start_id": _IM_START,
|
||||
"im_end_id": _IM_END,
|
||||
"slice_start_id": _SLICE_START,
|
||||
"slice_end_id": _SLICE_END,
|
||||
"newline_id": _NEWLINE,
|
||||
}
|
||||
|
||||
|
||||
def _canvas_positions(tokens, target_sizes):
|
||||
input_ids = torch.tensor([tokens], dtype=torch.long)
|
||||
targets = torch.tensor(target_sizes, dtype=torch.long)
|
||||
bounds = build_image_bounds(input_ids[0], _CANVAS_IDS)
|
||||
positions = _compute_canvas_single(
|
||||
input_ids[0],
|
||||
torch.arange(len(tokens), dtype=torch.long),
|
||||
bounds,
|
||||
targets,
|
||||
_CANVAS_IDS,
|
||||
)
|
||||
return positions.tolist(), canvas_rope_delta(positions, len(tokens))
|
||||
|
||||
|
||||
def test_canvas_mrope_trailing_newline_stays_inside_the_canvas():
|
||||
"""A newline right after `</image>` belongs to the canvas, not to the text.
|
||||
|
||||
The chat template joins content parts with ``\\n``, so an image followed by
|
||||
text always leaves one there. The transformers reference closes a frame over
|
||||
every marker including newlines; leaving it out pushes every later position
|
||||
one step forward and makes `rope_delta` differ by one.
|
||||
"""
|
||||
tokens = [_TEXT, _IM_START, _IMAGE_PAD, _IM_END, _NEWLINE, _TEXT, _TEXT]
|
||||
positions, delta = _canvas_positions(tokens, [[4, 4]])
|
||||
|
||||
assert positions == [
|
||||
[0, 1, 1, 1, 1, 3, 4],
|
||||
[0, 0, 1, 2, 1, 3, 4],
|
||||
[0, 0, 1, 2, 1, 3, 4],
|
||||
]
|
||||
assert delta == -2
|
||||
|
||||
|
||||
def test_canvas_mrope_thumbnail_matches_reference():
|
||||
# <text> <image> PAD </image> <text>; 16x turns a 4x4 patch grid into 1 token.
|
||||
tokens = [_TEXT, _IM_START, _IMAGE_PAD, _IM_END, _TEXT]
|
||||
positions, delta = _canvas_positions(tokens, [[4, 4]])
|
||||
|
||||
assert positions == [
|
||||
[0, 1, 1, 1, 3],
|
||||
[0, 0, 1, 2, 3],
|
||||
[0, 0, 1, 2, 3],
|
||||
]
|
||||
assert delta == -1
|
||||
|
||||
|
||||
def test_canvas_mrope_slice_row_matches_reference():
|
||||
# Thumbnail then a 1x2 slice row; the slices tile the thumbnail's canvas.
|
||||
tokens = [
|
||||
_TEXT,
|
||||
_IM_START,
|
||||
_IMAGE_PAD,
|
||||
_IM_END,
|
||||
_SLICE_START,
|
||||
_IMAGE_PAD,
|
||||
_SLICE_END,
|
||||
_SLICE_START,
|
||||
_IMAGE_PAD,
|
||||
_SLICE_END,
|
||||
_TEXT,
|
||||
]
|
||||
positions, delta = _canvas_positions(tokens, [[4, 4], [4, 4], [4, 4]])
|
||||
|
||||
assert positions == [
|
||||
[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 4],
|
||||
[0, 0, 1, 2, 1, 1, 1, 1, 1, 1, 4],
|
||||
[0, 0, 1, 3, 1, 1, 1, 2, 2, 2, 4],
|
||||
]
|
||||
assert delta == -6
|
||||
|
||||
|
||||
def test_canvas_mrope_resamples_thumbnail_over_canvas():
|
||||
# A 8x8 patch grid is 2x2 LLM tokens, so the canvas is resampled (not arange).
|
||||
tokens = [_TEXT, _IM_START] + [_IMAGE_PAD] * 4 + [_IM_END, _TEXT]
|
||||
positions, delta = _canvas_positions(tokens, [[8, 8]])
|
||||
|
||||
assert positions == [
|
||||
[0, 1, 1, 1, 1, 1, 1, 4],
|
||||
[0, 0, 1, 1, 2, 2, 3, 4],
|
||||
[0, 0, 1, 2, 1, 2, 3, 4],
|
||||
]
|
||||
assert delta == -3
|
||||
|
||||
|
||||
def test_canvas_mrope_video_frames_share_one_canvas():
|
||||
tokens = [
|
||||
_TEXT,
|
||||
_IM_START,
|
||||
_VIDEO_PAD,
|
||||
_IM_END,
|
||||
_IM_START,
|
||||
_VIDEO_PAD,
|
||||
_IM_END,
|
||||
_TEXT,
|
||||
]
|
||||
positions, delta = _canvas_positions(tokens, [[4, 4], [4, 4]])
|
||||
|
||||
assert positions == [
|
||||
[0, 1, 1, 1, 3, 3, 3, 5],
|
||||
[0, 0, 1, 2, 2, 3, 4, 5],
|
||||
[0, 0, 1, 2, 2, 3, 4, 5],
|
||||
]
|
||||
assert delta == -2
|
||||
|
||||
|
||||
def test_canvas_mrope_interleaved_image_and_video():
|
||||
tokens = (
|
||||
[_TEXT, _IM_START]
|
||||
+ [_IMAGE_PAD] * 4
|
||||
+ [_IM_END, _IM_START]
|
||||
+ [_VIDEO_PAD] * 4
|
||||
+ [_IM_END, _TEXT]
|
||||
)
|
||||
positions, delta = _canvas_positions(tokens, [[8, 8], [8, 8]])
|
||||
|
||||
assert positions == [
|
||||
[0, 1, 1, 1, 1, 1, 1, 4, 4, 4, 4, 4, 4, 7],
|
||||
[0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7],
|
||||
[0, 0, 1, 2, 1, 2, 3, 3, 4, 5, 4, 5, 6, 7],
|
||||
]
|
||||
assert delta == -6
|
||||
|
||||
|
||||
def test_canvas_span_is_shared_by_image_and_video():
|
||||
"""A video frame and a same-geometry image must place identical offsets.
|
||||
|
||||
Both paths go through `_assign_canvas_span`; this pins that they cannot
|
||||
drift apart now that the duplicated geometry is gone.
|
||||
"""
|
||||
thumb = [_IM_START] + [_IMAGE_PAD] * 4 + [_IM_END]
|
||||
|
||||
img_pos, _ = _canvas_positions([_TEXT, *thumb, _TEXT], [[8, 8]])
|
||||
vid_pos, _ = _canvas_positions([_TEXT, *thumb, *thumb, _TEXT], [[8, 8], [8, 8]])
|
||||
|
||||
def offsets(pos, start):
|
||||
# The <im_start> halo is one below the canvas base.
|
||||
base = pos[1][start] + 1
|
||||
return [[c[start + k] - base for k in range(3)] for c in pos]
|
||||
|
||||
assert offsets(img_pos, 0) == offsets(vid_pos, 0)
|
||||
|
||||
|
||||
def test_canvas_mrope_delta_leaves_room_for_the_first_decoded_token():
|
||||
# First decoded token must land on max_prefill_position + 1, matching
|
||||
# transformers and the other native M-RoPE models.
|
||||
tokens = [_TEXT, _IM_START, _IMAGE_PAD, _IM_END, _TEXT]
|
||||
positions, delta = _canvas_positions(tokens, [[4, 4]])
|
||||
|
||||
max_position = max(max(channel) for channel in positions)
|
||||
assert delta == max_position + 1 - len(tokens)
|
||||
@@ -1,146 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config.compilation import CompilationMode
|
||||
from vllm.model_executor.models import deepseek_v2 as deepseek_mod
|
||||
from vllm.model_executor.models import mistral_large_3_eagle as eagle_mod
|
||||
|
||||
|
||||
class DummyPPGroup:
|
||||
world_size = 1
|
||||
is_first_rank = True
|
||||
is_last_rank = True
|
||||
|
||||
|
||||
class DummyEmbedding(nn.Module):
|
||||
def __init__(self, vocab_size, hidden_size, *args, **kwargs):
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
|
||||
def forward(self, input_ids):
|
||||
return torch.zeros(
|
||||
(*input_ids.shape, self.hidden_size),
|
||||
dtype=torch.float32,
|
||||
device=input_ids.device,
|
||||
)
|
||||
|
||||
|
||||
class DummyLinear(nn.Module):
|
||||
def __init__(self, in_features, out_features, *args, **kwargs):
|
||||
super().__init__()
|
||||
self.out_features = out_features
|
||||
|
||||
def forward(self, x):
|
||||
return torch.zeros(
|
||||
(*x.shape[:-1], self.out_features),
|
||||
dtype=x.dtype,
|
||||
device=x.device,
|
||||
)
|
||||
|
||||
|
||||
class DummyNorm(nn.Module):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__()
|
||||
|
||||
def forward(self, hidden_states, residual=None):
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
class DummyDecoderLayer(nn.Module):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__()
|
||||
|
||||
def forward(self, positions, hidden_states, residual, llama_4_scaling=None):
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
def make_vllm_config(
|
||||
*, model_type="mistral3", qk_nope_head_dim=128, qk_rope_head_dim=64
|
||||
):
|
||||
hf_config = SimpleNamespace(
|
||||
model_type=model_type,
|
||||
first_k_dense_replace=0,
|
||||
vocab_size=32000,
|
||||
hidden_size=16,
|
||||
num_hidden_layers=1,
|
||||
rms_norm_eps=1e-5,
|
||||
qk_nope_head_dim=qk_nope_head_dim,
|
||||
qk_rope_head_dim=qk_rope_head_dim,
|
||||
)
|
||||
|
||||
return SimpleNamespace(
|
||||
model_config=SimpleNamespace(hf_config=hf_config),
|
||||
quant_config=None,
|
||||
parallel_config=SimpleNamespace(
|
||||
eplb_config=SimpleNamespace(num_redundant_experts=0),
|
||||
),
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=8),
|
||||
cache_config=None,
|
||||
compilation_config=SimpleNamespace(mode=CompilationMode.NONE),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def patch_heavy_modules(monkeypatch):
|
||||
monkeypatch.setattr(eagle_mod, "get_pp_group", lambda: DummyPPGroup())
|
||||
monkeypatch.setattr(deepseek_mod, "get_pp_group", lambda: DummyPPGroup())
|
||||
|
||||
monkeypatch.setattr(eagle_mod, "VocabParallelEmbedding", DummyEmbedding)
|
||||
monkeypatch.setattr(eagle_mod, "RowParallelLinear", DummyLinear)
|
||||
monkeypatch.setattr(eagle_mod, "RMSNorm", DummyNorm)
|
||||
monkeypatch.setattr(eagle_mod, "DeepseekV2DecoderLayer", DummyDecoderLayer)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize(
|
||||
("model_type", "qk_nope_head_dim", "qk_rope_head_dim", "expected_use_mha"),
|
||||
[
|
||||
# MLA-style config: should not use MHA.
|
||||
("mistral3", 128, 64, False),
|
||||
# No MLA dims: should use MHA, matching DeepseekV2Model.__init__ logic.
|
||||
("mistral3", 0, 0, True),
|
||||
# DeepSeek model type always uses MHA by the parent logic.
|
||||
("deepseek", 128, 64, True),
|
||||
],
|
||||
)
|
||||
def test_eagle_mistral_large3_initializes_deepseek_runtime_attrs(
|
||||
model_type,
|
||||
qk_nope_head_dim,
|
||||
qk_rope_head_dim,
|
||||
expected_use_mha,
|
||||
):
|
||||
vllm_config = make_vllm_config(
|
||||
model_type=model_type,
|
||||
qk_nope_head_dim=qk_nope_head_dim,
|
||||
qk_rope_head_dim=qk_rope_head_dim,
|
||||
)
|
||||
|
||||
model = eagle_mod.EagleMistralLarge3Model(vllm_config=vllm_config)
|
||||
|
||||
assert model.aux_hidden_state_layers == ()
|
||||
assert model.use_mha is expected_use_mha
|
||||
|
||||
# Add this if your fix also copies num_redundant_experts from
|
||||
# DeepseekV2Model.__init__.
|
||||
assert model.num_redundant_experts == 0
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_eagle_mistral_large3_forward_reuses_deepseek_parent_forward():
|
||||
vllm_config = make_vllm_config()
|
||||
model = eagle_mod.EagleMistralLarge3Model(vllm_config=vllm_config)
|
||||
|
||||
input_ids = torch.tensor([[1, 2, 3]])
|
||||
positions = torch.tensor([[0, 1, 2]])
|
||||
hidden_states = torch.zeros((1, 3, 16))
|
||||
|
||||
output = model(input_ids, positions, hidden_states)
|
||||
|
||||
assert isinstance(output, torch.Tensor)
|
||||
assert output.shape == hidden_states.shape
|
||||
@@ -1,133 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.model_executor.layers.pooler import DispatchPooler
|
||||
from vllm.model_executor.layers.pooler.seqwise import CLSPool, MeanPool
|
||||
from vllm.model_executor.models.bert import BertEmbeddingModel
|
||||
from vllm.model_executor.models.roberta import RobertaEmbeddingModel
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
MAX_MODEL_LEN = 128
|
||||
MODEL_NAME = os.environ.get("MODEL_NAME", "BAAI/bge-base-en-v1.5")
|
||||
REVISION = os.environ.get("REVISION", "main")
|
||||
|
||||
MODEL_NAME_ROBERTA = os.environ.get("MODEL_NAME", "intfloat/multilingual-e5-base")
|
||||
REVISION_ROBERTA = os.environ.get("REVISION", "main")
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_rocm(), reason="Xformers backend is not supported on ROCm."
|
||||
)
|
||||
def test_model_loading_with_params(vllm_runner, monkeypatch):
|
||||
"""Test parameter weight loading with tp>1."""
|
||||
# to use apply_model
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
with vllm_runner(
|
||||
model_name=MODEL_NAME,
|
||||
revision=REVISION,
|
||||
dtype="float16",
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
) as vllm_model:
|
||||
output = vllm_model.embed(
|
||||
"Write a short story about a robot that dreams for the first time.\n"
|
||||
)
|
||||
|
||||
model_config = vllm_model.llm.llm_engine.model_config
|
||||
model_tokenizer = vllm_model.llm.llm_engine.tokenizer
|
||||
|
||||
# asserts on the bert model config file
|
||||
assert model_config.encoder_config["max_seq_length"] == 512
|
||||
assert model_config.encoder_config["do_lower_case"]
|
||||
|
||||
# asserts on the pooling config files
|
||||
assert model_config.pooler_config.seq_pooling_type == "CLS"
|
||||
assert model_config.pooler_config.tok_pooling_type == "ALL"
|
||||
assert model_config.pooler_config.use_activation
|
||||
|
||||
# asserts on the tokenizer loaded
|
||||
assert model_config.tokenizer == "BAAI/bge-base-en-v1.5"
|
||||
assert model_tokenizer.model_max_length == 512
|
||||
|
||||
def check_model(model):
|
||||
assert isinstance(model, BertEmbeddingModel)
|
||||
assert isinstance(pooler := model.pooler, DispatchPooler)
|
||||
assert isinstance(pooler.poolers_by_task["embed"].pooling, CLSPool)
|
||||
|
||||
vllm_model.apply_model(check_model)
|
||||
|
||||
assert output
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_rocm(), reason="Xformers backend is not supported on ROCm."
|
||||
)
|
||||
def test_roberta_model_loading_with_params(vllm_runner, monkeypatch):
|
||||
"""Test parameter weight loading with tp>1."""
|
||||
# to use apply_model
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
with vllm_runner(
|
||||
model_name=MODEL_NAME_ROBERTA,
|
||||
revision=REVISION_ROBERTA,
|
||||
dtype="float16",
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
) as vllm_model:
|
||||
output = vllm_model.embed(
|
||||
"Write a short story about a robot that dreams for the first time.\n"
|
||||
)
|
||||
|
||||
model_config = vllm_model.llm.llm_engine.model_config
|
||||
model_tokenizer = vllm_model.llm.llm_engine.tokenizer
|
||||
|
||||
# asserts on the bert model config file
|
||||
assert model_config.encoder_config["max_seq_length"] == 512
|
||||
assert not model_config.encoder_config["do_lower_case"]
|
||||
|
||||
# asserts on the pooling config files
|
||||
assert model_config.pooler_config.seq_pooling_type == "MEAN"
|
||||
assert model_config.pooler_config.tok_pooling_type == "ALL"
|
||||
assert model_config.pooler_config.use_activation
|
||||
|
||||
# asserts on the tokenizer loaded
|
||||
assert model_config.tokenizer == "intfloat/multilingual-e5-base"
|
||||
assert model_tokenizer.model_max_length == 512
|
||||
|
||||
def check_model(model):
|
||||
assert isinstance(model, RobertaEmbeddingModel)
|
||||
assert isinstance(pooler := model.pooler, DispatchPooler)
|
||||
assert isinstance(pooler.poolers_by_task["embed"].pooling, MeanPool)
|
||||
|
||||
vllm_model.apply_model(check_model)
|
||||
|
||||
assert output
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_rocm(), reason="Xformers backend is not supported on ROCm."
|
||||
)
|
||||
def test_facebook_roberta_model_loading_with_params(vllm_runner, monkeypatch):
|
||||
"""Test loading roberta-base model with no lm_head."""
|
||||
# to use apply_model
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
model_name = "FacebookAI/roberta-base"
|
||||
with vllm_runner(
|
||||
model_name=model_name, dtype="float16", max_model_len=MAX_MODEL_LEN
|
||||
) as vllm_model:
|
||||
output = vllm_model.embed(
|
||||
"Write a short story about a robot that dreams for the first time.\n"
|
||||
)
|
||||
|
||||
assert vllm_model.llm.llm_engine.model_config.tokenizer == model_name
|
||||
|
||||
def check_model(model):
|
||||
assert isinstance(model, RobertaEmbeddingModel)
|
||||
assert not hasattr(model, "lm_head")
|
||||
assert isinstance(pooler := model.pooler, DispatchPooler)
|
||||
assert isinstance(pooler.poolers_by_task["embed"].pooling, CLSPool)
|
||||
|
||||
vllm_model.apply_model(check_model)
|
||||
|
||||
assert output
|
||||
@@ -1,61 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
def test_nemotron_h_lm_head_receives_quant_config():
|
||||
from vllm.model_executor.models.nemotron_h import NemotronHForCausalLM
|
||||
|
||||
mock_quant_config = Mock()
|
||||
|
||||
mock_hf_config = Mock()
|
||||
mock_hf_config.vocab_size = 128
|
||||
mock_hf_config.hidden_size = 64
|
||||
|
||||
mock_vllm_config = Mock()
|
||||
mock_vllm_config.model_config.hf_config = mock_hf_config
|
||||
mock_vllm_config.model_config.dtype = None
|
||||
mock_vllm_config.scheduler_config = Mock()
|
||||
mock_vllm_config.quant_config = mock_quant_config
|
||||
|
||||
with (
|
||||
patch("vllm.model_executor.models.nemotron_h.NemotronHModel") as MockModel,
|
||||
patch("vllm.model_executor.models.nemotron_h.ParallelLMHead") as MockLMHead,
|
||||
patch("vllm.model_executor.models.nemotron_h.LogitsProcessor"),
|
||||
):
|
||||
MockModel.return_value.make_empty_intermediate_tensors = Mock()
|
||||
MockModel.return_value.has_moe = False
|
||||
|
||||
NemotronHForCausalLM(vllm_config=mock_vllm_config)
|
||||
|
||||
MockLMHead.assert_called_once()
|
||||
call_kwargs = MockLMHead.call_args.kwargs
|
||||
assert call_kwargs["quant_config"] is mock_quant_config
|
||||
|
||||
|
||||
def test_relu2_fp8_fusion_uses_registry():
|
||||
from vllm.model_executor.models.nemotron_h import NemotronHMLP
|
||||
|
||||
projected = torch.empty((1, 1), dtype=torch.bfloat16)
|
||||
fused = Mock()
|
||||
act_fn = Mock()
|
||||
down_proj = Mock(side_effect=lambda x: (x, None))
|
||||
|
||||
mlp = NemotronHMLP.__new__(NemotronHMLP)
|
||||
torch.nn.Module.__init__(mlp)
|
||||
mlp.up_proj = Mock(return_value=(projected, None))
|
||||
mlp.down_proj = down_proj
|
||||
mlp.act_fn = act_fn
|
||||
|
||||
with patch(
|
||||
"vllm.model_executor.models.nemotron_h.maybe_fused_act_quant",
|
||||
return_value=fused,
|
||||
) as maybe_fused:
|
||||
result = mlp(Mock())
|
||||
|
||||
maybe_fused.assert_called_once_with(act_fn, projected, down_proj)
|
||||
act_fn.assert_not_called()
|
||||
assert result is fused
|
||||
@@ -1,210 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.paddleocr_vl import (
|
||||
PaddleOCRVLForConditionalGeneration,
|
||||
)
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldElem,
|
||||
MultiModalKwargsItem,
|
||||
PlaceholderRange,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True, scope="module")
|
||||
def _force_cpu_default_device():
|
||||
original = torch.get_default_device()
|
||||
torch.set_default_device("cpu")
|
||||
yield
|
||||
torch.set_default_device(original)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DummyVisionConfig:
|
||||
spatial_merge_size: int = 2
|
||||
patch_size: int = 14
|
||||
|
||||
|
||||
@dataclass
|
||||
class DummyConfig:
|
||||
image_token_id: int = 151655
|
||||
video_token_id: int = 151654
|
||||
vision_start_token_id: int = 151652
|
||||
vision_end_token_id: int = 151653
|
||||
vision_config: DummyVisionConfig = field(default_factory=DummyVisionConfig)
|
||||
|
||||
|
||||
def make_model(config: DummyConfig) -> PaddleOCRVLForConditionalGeneration:
|
||||
model = object.__new__(PaddleOCRVLForConditionalGeneration)
|
||||
model.config = config
|
||||
return model
|
||||
|
||||
|
||||
def make_mm_feature(
|
||||
*,
|
||||
offset: int,
|
||||
length: int,
|
||||
image_grid_thw: tuple[int, int, int],
|
||||
) -> MultiModalFeatureSpec:
|
||||
return MultiModalFeatureSpec(
|
||||
data=MultiModalKwargsItem(
|
||||
{
|
||||
"image_grid_thw": MultiModalFieldElem(
|
||||
data=torch.tensor(image_grid_thw),
|
||||
field=None,
|
||||
),
|
||||
}
|
||||
),
|
||||
modality="image",
|
||||
identifier="DUMMY",
|
||||
mm_position=PlaceholderRange(offset=offset, length=length),
|
||||
)
|
||||
|
||||
|
||||
def test_get_mrope_input_positions_text_only():
|
||||
model = make_model(DummyConfig())
|
||||
input_tokens = [11, 12, 13, 14, 15]
|
||||
positions, delta = model.get_mrope_input_positions(
|
||||
input_tokens=input_tokens,
|
||||
mm_features=[],
|
||||
)
|
||||
expected = torch.tensor(
|
||||
[
|
||||
[0, 1, 2, 3, 4],
|
||||
[0, 1, 2, 3, 4],
|
||||
[0, 1, 2, 3, 4],
|
||||
]
|
||||
)
|
||||
assert torch.equal(positions, expected)
|
||||
assert delta == 0
|
||||
|
||||
|
||||
def test_get_mrope_input_positions_single_image():
|
||||
model = make_model(DummyConfig())
|
||||
spatial_merge_size = model.config.vision_config.spatial_merge_size
|
||||
|
||||
t, h, w = 1, 2, 2
|
||||
num_image_tokens = t * h * w
|
||||
|
||||
input_tokens = (
|
||||
[10]
|
||||
+ [model.config.vision_start_token_id]
|
||||
+ [model.config.image_token_id] * num_image_tokens
|
||||
+ [model.config.vision_end_token_id]
|
||||
+ [30, 31]
|
||||
)
|
||||
|
||||
mm_features = [
|
||||
make_mm_feature(
|
||||
offset=2, # 1 (text) + 1 (vision_start)
|
||||
length=num_image_tokens,
|
||||
image_grid_thw=(t, h * spatial_merge_size, w * spatial_merge_size),
|
||||
)
|
||||
]
|
||||
|
||||
positions, delta = model.get_mrope_input_positions(
|
||||
input_tokens=input_tokens,
|
||||
mm_features=mm_features,
|
||||
)
|
||||
|
||||
expected = torch.tensor(
|
||||
[
|
||||
[0, 1, 2, 2, 2, 2, 4, 5, 6],
|
||||
[0, 1, 2, 2, 3, 3, 4, 5, 6],
|
||||
[0, 1, 2, 3, 2, 3, 4, 5, 6],
|
||||
]
|
||||
)
|
||||
|
||||
assert torch.equal(positions, expected)
|
||||
expected_delta = (positions.max().item() + 1) - len(input_tokens)
|
||||
assert delta == expected_delta
|
||||
|
||||
|
||||
def test_get_mrope_input_positions_multiple_images():
|
||||
model = make_model(DummyConfig())
|
||||
spatial_merge_size = model.config.vision_config.spatial_merge_size
|
||||
|
||||
t1, h1, w1 = 1, 2, 2
|
||||
num1 = t1 * h1 * w1
|
||||
|
||||
t2, h2, w2 = 1, 1, 3
|
||||
num2 = t2 * h2 * w2
|
||||
|
||||
input_tokens = (
|
||||
[10]
|
||||
+ [model.config.vision_start_token_id]
|
||||
+ [model.config.image_token_id] * num1
|
||||
+ [model.config.vision_end_token_id]
|
||||
+ [20, 21]
|
||||
+ [model.config.vision_start_token_id]
|
||||
+ [model.config.image_token_id] * num2
|
||||
+ [model.config.vision_end_token_id]
|
||||
+ [30]
|
||||
)
|
||||
|
||||
mm_features = [
|
||||
make_mm_feature(
|
||||
offset=2,
|
||||
length=num1,
|
||||
image_grid_thw=(t1, h1 * spatial_merge_size, w1 * spatial_merge_size),
|
||||
),
|
||||
make_mm_feature(
|
||||
offset=2 + num1 + 1 + 2 + 1,
|
||||
length=num2,
|
||||
image_grid_thw=(t2, h2 * spatial_merge_size, w2 * spatial_merge_size),
|
||||
),
|
||||
]
|
||||
|
||||
positions, delta = model.get_mrope_input_positions(
|
||||
input_tokens=input_tokens,
|
||||
mm_features=mm_features,
|
||||
)
|
||||
|
||||
assert positions.shape == (3, 15)
|
||||
assert not torch.equal(positions[:, 2:6], torch.arange(4).expand(3, 4) + 2)
|
||||
assert not torch.equal(positions[:, 10:13], torch.arange(3).expand(3, 3) + 10)
|
||||
|
||||
|
||||
def test_get_mrope_input_positions_image_at_start():
|
||||
model = make_model(DummyConfig())
|
||||
spatial_merge_size = model.config.vision_config.spatial_merge_size
|
||||
|
||||
t, h, w = 1, 2, 2
|
||||
num_tokens = t * h * w
|
||||
|
||||
input_tokens = (
|
||||
[model.config.vision_start_token_id]
|
||||
+ [model.config.image_token_id] * num_tokens
|
||||
+ [model.config.vision_end_token_id]
|
||||
+ [10, 11]
|
||||
)
|
||||
|
||||
mm_features = [
|
||||
make_mm_feature(
|
||||
offset=1, # start token at index 0
|
||||
length=num_tokens,
|
||||
image_grid_thw=(t, h * spatial_merge_size, w * spatial_merge_size),
|
||||
)
|
||||
]
|
||||
|
||||
positions, delta = model.get_mrope_input_positions(
|
||||
input_tokens=input_tokens,
|
||||
mm_features=mm_features,
|
||||
)
|
||||
|
||||
expected = torch.tensor(
|
||||
[
|
||||
[0, 1, 1, 1, 1, 3, 4, 5],
|
||||
[0, 1, 1, 2, 2, 3, 4, 5],
|
||||
[0, 1, 2, 1, 2, 3, 4, 5],
|
||||
]
|
||||
)
|
||||
|
||||
assert torch.equal(positions, expected)
|
||||
@@ -1,28 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.phi4mm_utils import MultiHeadedAttention
|
||||
|
||||
|
||||
def test_eager_attention_does_not_retain_probability_matrix():
|
||||
module = MultiHeadedAttention(
|
||||
n_head=2,
|
||||
n_feat=16,
|
||||
dropout_rate=0.0,
|
||||
use_pt_scaled_dot_product_attention=False,
|
||||
)
|
||||
module.eval()
|
||||
batch, time, feat = 2, 8, 16
|
||||
x = torch.randn(batch, time, feat)
|
||||
with torch.no_grad():
|
||||
out = module(x, x, x, None, None, None)
|
||||
|
||||
assert out.shape == (batch, time, feat)
|
||||
assert getattr(module, "attn", None) is None
|
||||
|
||||
|
||||
def test_multiheaded_attention_defaults_to_scaled_dot_product():
|
||||
module = MultiHeadedAttention(n_head=2, n_feat=16, dropout_rate=0.0)
|
||||
assert module.use_pt_scaled_dot_product_attention is True
|
||||
@@ -1,72 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.model_executor.models import plamo3 as plamo3_mod
|
||||
from vllm.model_executor.models.interfaces import supports_eagle3
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.eagle3_utils import (
|
||||
set_eagle3_aux_hidden_state_layers,
|
||||
)
|
||||
|
||||
|
||||
class DummyDecoderLayer(nn.Module):
|
||||
def forward(self, positions, hidden_states, residual):
|
||||
del positions
|
||||
return hidden_states + 1, hidden_states + 2
|
||||
|
||||
|
||||
class DummyNorm(nn.Module):
|
||||
def forward(self, hidden_states, residual):
|
||||
return hidden_states + residual, None
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_plamo3_returns_dflash_selected_auxiliary_hidden_states(monkeypatch):
|
||||
decoder = plamo3_mod.Plamo3Decoder.__new__(plamo3_mod.Plamo3Decoder)
|
||||
nn.Module.__init__(decoder)
|
||||
decoder.start_layer = 0
|
||||
decoder.end_layer = 2
|
||||
decoder.layers = nn.ModuleList([DummyDecoderLayer() for _ in range(8)])
|
||||
|
||||
model = plamo3_mod.Plamo3Model.__new__(plamo3_mod.Plamo3Model)
|
||||
nn.Module.__init__(model)
|
||||
model.layers = decoder
|
||||
model.norm = DummyNorm()
|
||||
model.do_not_compile = True
|
||||
|
||||
target = plamo3_mod.Plamo3ForCausalLM.__new__(plamo3_mod.Plamo3ForCausalLM)
|
||||
nn.Module.__init__(target)
|
||||
target.model = model
|
||||
|
||||
monkeypatch.setattr(
|
||||
plamo3_mod,
|
||||
"get_pp_group",
|
||||
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
|
||||
)
|
||||
|
||||
assert supports_eagle3(target)
|
||||
assert target.get_eagle3_default_aux_hidden_state_layers() == (2, 4, 5)
|
||||
|
||||
spec_config = SimpleNamespace(
|
||||
draft_model_config=SimpleNamespace(
|
||||
hf_config=SimpleNamespace(dflash_config={"target_layer_ids": [0, 1]})
|
||||
)
|
||||
)
|
||||
set_eagle3_aux_hidden_state_layers(target, spec_config)
|
||||
assert decoder.aux_hidden_state_layers == (1, 2)
|
||||
|
||||
output, aux_hidden_states = target(
|
||||
input_ids=None,
|
||||
positions=torch.tensor([0]),
|
||||
inputs_embeds=torch.tensor([[1.0, 2.0]]),
|
||||
)
|
||||
|
||||
torch.testing.assert_close(output, torch.tensor([[7.0, 9.0]]))
|
||||
assert len(aux_hidden_states) == 2
|
||||
torch.testing.assert_close(aux_hidden_states[0], torch.tensor([[5.0, 7.0]]))
|
||||
torch.testing.assert_close(aux_hidden_states[1], torch.tensor([[7.0, 9.0]]))
|
||||
@@ -1,27 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.qwen3_asr import (
|
||||
Qwen3ASRForConditionalGeneration,
|
||||
)
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange
|
||||
|
||||
|
||||
def test_mrope_positions_with_stripped_audio_data() -> None:
|
||||
input_tokens = list(range(7))
|
||||
feature = MultiModalFeatureSpec(
|
||||
data=None,
|
||||
modality="audio",
|
||||
identifier="audio",
|
||||
mm_position=PlaceholderRange(offset=2, length=3),
|
||||
)
|
||||
|
||||
positions, delta = Qwen3ASRForConditionalGeneration.get_mrope_input_positions(
|
||||
None, input_tokens, [feature]
|
||||
)
|
||||
|
||||
expected = torch.arange(len(input_tokens), dtype=torch.long).expand(3, -1)
|
||||
torch.testing.assert_close(positions, expected)
|
||||
assert delta == 0
|
||||
@@ -1,424 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.multimodal.processing import InputProcessingContext
|
||||
|
||||
|
||||
# Helper function to print input IDs with coalesced audio/video tokens.
|
||||
def print_input_ids(input_ids):
|
||||
"""Print input IDs, compressing consecutive special tokens.
|
||||
- 151675: <|audio_pad|>
|
||||
- 151656: <|video_pad|>
|
||||
"""
|
||||
if not input_ids:
|
||||
print("[]")
|
||||
return
|
||||
|
||||
result = []
|
||||
i = 0
|
||||
|
||||
while i < len(input_ids):
|
||||
current_id = input_ids[i]
|
||||
|
||||
# Check if it's a special token that should be compressed
|
||||
if current_id in [151675, 151656]:
|
||||
# Count consecutive occurrences
|
||||
count = 1
|
||||
while i + count < len(input_ids) and input_ids[i + count] == current_id:
|
||||
count += 1
|
||||
|
||||
# Add compressed representation
|
||||
token_name = "<|audio_pad|>" if current_id == 151675 else "<|video_pad|>"
|
||||
result.append(f"{token_name} * {count}")
|
||||
i += count
|
||||
else:
|
||||
# Regular token, just add it
|
||||
result.append(str(current_id))
|
||||
i += 1
|
||||
|
||||
print(", ".join(result))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_qwen3_omni_config():
|
||||
"""Create a mock Qwen3OmniMoeThinker config."""
|
||||
config = Mock(spec=PreTrainedConfig)
|
||||
# Token IDs from https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Instruct/blob/main/tokenizer_config.json
|
||||
config.audio_token_id = 151675 # <|audio_pad|>
|
||||
config.video_token_id = 151656 # <|video_pad|>
|
||||
config.image_token_id = 151655 # <|image_pad|>
|
||||
config.audio_start_token_id = 151669 # <|audio_start|>
|
||||
config.audio_end_token_id = 151670 # <|audio_end|>
|
||||
config.vision_start_token_id = 151652 # <|vision_start|>
|
||||
config.position_id_per_seconds = 12.5
|
||||
|
||||
# Vision config
|
||||
vision_config = Mock()
|
||||
vision_config.spatial_merge_size = 2
|
||||
config.vision_config = vision_config
|
||||
|
||||
return config
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_processor():
|
||||
"""Create a mock HF processor."""
|
||||
from transformers.models.whisper import WhisperFeatureExtractor
|
||||
|
||||
processor = Mock()
|
||||
processor.audio_token = "<|audio_pad|>"
|
||||
processor.image_token = "<|image_pad|>"
|
||||
processor.video_token = "<|video_pad|>"
|
||||
|
||||
# Create a real WhisperFeatureExtractor instance for the feature_extractor attribute
|
||||
feature_extractor = WhisperFeatureExtractor()
|
||||
processor.feature_extractor = feature_extractor
|
||||
|
||||
return processor
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_tokenizer():
|
||||
"""Create a mock tokenizer."""
|
||||
tokenizer = Mock()
|
||||
# Token IDs from https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Instruct/blob/main/tokenizer_config.json
|
||||
tokenizer.get_vocab = Mock(
|
||||
return_value={
|
||||
"<|audio_pad|>": 151675,
|
||||
"<|video_pad|>": 151656,
|
||||
"<|image_pad|>": 151655,
|
||||
"<|audio_start|>": 151669,
|
||||
"<|audio_end|>": 151670,
|
||||
"<|vision_start|>": 151652,
|
||||
"<|vision_end|>": 151653,
|
||||
}
|
||||
)
|
||||
tokenizer.encode = Mock(
|
||||
side_effect=lambda x: {
|
||||
"<|vision_start|>": [151652],
|
||||
"<|vision_end|>": [151653],
|
||||
"<|audio_start|>": [151669],
|
||||
"<|audio_end|>": [151670],
|
||||
"<|audio_pad|>": [151675],
|
||||
"<|image_pad|>": [151655],
|
||||
"<|video_pad|>": [151656],
|
||||
}.get(x, [0])
|
||||
)
|
||||
tokenizer.vision_bos_token = "<|vision_start|>"
|
||||
tokenizer.vision_eos_token = "<|vision_end|>"
|
||||
tokenizer.audio_bos_token = "<|audio_start|>"
|
||||
tokenizer.audio_eos_token = "<|audio_end|>"
|
||||
return tokenizer
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_image_processor():
|
||||
"""Create a mock image processor."""
|
||||
image_processor = Mock()
|
||||
image_processor.merge_size = 2
|
||||
return image_processor
|
||||
|
||||
|
||||
def test_qwen3_omni_get_updates_use_audio_in_video(
|
||||
mock_qwen3_omni_config,
|
||||
mock_processor,
|
||||
mock_tokenizer,
|
||||
mock_image_processor,
|
||||
):
|
||||
"""Test the get_updates_use_audio_in_video method directly."""
|
||||
from vllm.model_executor.models.qwen3_omni_moe_thinker import (
|
||||
Qwen3OmniMoeThinkerMultiModalProcessor,
|
||||
Qwen3OmniMoeThinkerProcessingInfo,
|
||||
)
|
||||
|
||||
# Create a mock context
|
||||
mock_ctx = Mock(spec=InputProcessingContext)
|
||||
mock_ctx.tokenizer = mock_tokenizer
|
||||
# `model_config` is an instance attribute, so it is not covered by the
|
||||
# spec; the data parser reads it for `allow_missing_mm_embeddings`.
|
||||
mock_ctx.model_config = SimpleNamespace(
|
||||
multimodal_config=SimpleNamespace(allow_missing_mm_embeddings=False)
|
||||
)
|
||||
|
||||
# Create processing info
|
||||
info = Qwen3OmniMoeThinkerProcessingInfo(mock_ctx)
|
||||
info._get_expected_hidden_size = lambda: 100
|
||||
info.get_hf_config = Mock(return_value=mock_qwen3_omni_config)
|
||||
info.get_hf_processor = Mock(return_value=mock_processor)
|
||||
info.get_tokenizer = Mock(return_value=mock_tokenizer)
|
||||
info.get_image_processor = Mock(return_value=mock_image_processor)
|
||||
|
||||
# Create a mock dummy_inputs builder
|
||||
mock_dummy_inputs = Mock()
|
||||
|
||||
# Create the processor
|
||||
processor = Qwen3OmniMoeThinkerMultiModalProcessor(info, mock_dummy_inputs)
|
||||
|
||||
# Test parameters from reference video
|
||||
# https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Omni/demo/draw.mp4
|
||||
audio_len = 85
|
||||
video_grid_thw = [6, 36, 64]
|
||||
video_second_per_grid_t = 2.0
|
||||
|
||||
# Call the method
|
||||
updates = processor.get_updates_use_audio_in_video(
|
||||
thinker_config=mock_qwen3_omni_config,
|
||||
audio_len=audio_len,
|
||||
video_grid_thw=video_grid_thw,
|
||||
video_second_per_grid_t=video_second_per_grid_t,
|
||||
)
|
||||
|
||||
# Updated input ids should align with HF implementation.
|
||||
# 151669,
|
||||
# <|video_pad|> * 576, <|audio_pad|> * 25,
|
||||
# <|video_pad|> * 576, <|audio_pad|> * 25,
|
||||
# <|video_pad|> * 576, <|audio_pad|> * 25,
|
||||
# <|video_pad|> * 576, <|audio_pad|> * 10,
|
||||
# <|video_pad|> * 1152,
|
||||
# 151670
|
||||
print_input_ids(updates)
|
||||
|
||||
# Verify structure
|
||||
assert isinstance(updates, list)
|
||||
assert len(updates) > 0
|
||||
|
||||
# Verify start and end tokens
|
||||
audio_start_token_id = mock_qwen3_omni_config.audio_start_token_id
|
||||
audio_end_token_id = mock_qwen3_omni_config.audio_end_token_id
|
||||
|
||||
assert updates[0] == audio_start_token_id
|
||||
assert updates[-1] == audio_end_token_id
|
||||
|
||||
# Verify both audio and video tokens are present
|
||||
audio_token_id = mock_qwen3_omni_config.audio_token_id
|
||||
video_token_id = mock_qwen3_omni_config.video_token_id
|
||||
|
||||
audio_count = updates.count(audio_token_id)
|
||||
video_count = updates.count(video_token_id)
|
||||
|
||||
assert audio_count == audio_len, (
|
||||
f"Expected {audio_len} audio tokens, got {audio_count}"
|
||||
)
|
||||
|
||||
# Calculate expected video token count
|
||||
spatial_merge_size = mock_qwen3_omni_config.vision_config.spatial_merge_size
|
||||
height = video_grid_thw[1] // spatial_merge_size
|
||||
width = video_grid_thw[2] // spatial_merge_size
|
||||
expected_video_count = video_grid_thw[0] * height * width
|
||||
|
||||
assert video_count == expected_video_count, (
|
||||
f"Expected {expected_video_count} video tokens, got {video_count}"
|
||||
)
|
||||
|
||||
# Total tokens should be: 1 (start) + audio_len + video_count + 1 (end)
|
||||
expected_total = 1 + audio_len + expected_video_count + 1
|
||||
assert len(updates) == expected_total, (
|
||||
f"Expected {expected_total} total tokens, got {len(updates)}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_qwen3_omni_exposes_eagle3_to_its_text_backbone():
|
||||
from vllm.model_executor.models.interfaces import EagleModelMixin, supports_eagle3
|
||||
from vllm.model_executor.models.qwen3_omni_moe_thinker import (
|
||||
Qwen3OmniMoeThinkerForConditionalGeneration,
|
||||
)
|
||||
|
||||
class DummyBackbone(nn.Module, EagleModelMixin):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.layers = nn.ModuleList([nn.Identity(), nn.Identity()])
|
||||
|
||||
class DummyLanguageModel(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.model = DummyBackbone()
|
||||
|
||||
def embed_input_ids(self, input_ids):
|
||||
return input_ids
|
||||
|
||||
model = Qwen3OmniMoeThinkerForConditionalGeneration.__new__(
|
||||
Qwen3OmniMoeThinkerForConditionalGeneration
|
||||
)
|
||||
nn.Module.__init__(model)
|
||||
model.language_model = DummyLanguageModel()
|
||||
|
||||
assert supports_eagle3(model)
|
||||
model.set_aux_hidden_state_layers((1, 2))
|
||||
assert model.language_model.model.aux_hidden_state_layers == (1, 2)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_qwen3_omni_text_model_collects_post_deepstack_aux_hidden_states():
|
||||
from vllm.model_executor.models.qwen3_omni_moe_thinker import Qwen3MoeLLMModel
|
||||
|
||||
class DummyLayer(nn.Module):
|
||||
def forward(self, positions, hidden_states, residual):
|
||||
return hidden_states + 1, torch.full_like(hidden_states, 10)
|
||||
|
||||
class DummyNorm(nn.Module):
|
||||
def forward(self, hidden_states, residual):
|
||||
return hidden_states + residual, None
|
||||
|
||||
model = Qwen3MoeLLMModel.__new__(Qwen3MoeLLMModel)
|
||||
nn.Module.__init__(model)
|
||||
model.start_layer = 0
|
||||
model.end_layer = 1
|
||||
model.layers = nn.ModuleList([DummyLayer()])
|
||||
model.norm = DummyNorm()
|
||||
model.aux_hidden_state_layers = (1,)
|
||||
|
||||
pp_group = Mock(is_first_rank=True, is_last_rank=True)
|
||||
inputs_embeds = torch.tensor([[1.0]])
|
||||
deepstack_inputs = {"deepstack_input_embeds_0": torch.tensor([[3.0]])}
|
||||
with patch(
|
||||
"vllm.model_executor.models.qwen3_omni_moe_thinker.get_pp_group",
|
||||
return_value=pp_group,
|
||||
):
|
||||
output, aux_hidden_states = model.forward(
|
||||
input_ids=None,
|
||||
positions=torch.tensor([0]),
|
||||
inputs_embeds=inputs_embeds,
|
||||
deepstack_input_embeds=deepstack_inputs,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(output, torch.tensor([[15.0]]))
|
||||
assert len(aux_hidden_states) == 1
|
||||
torch.testing.assert_close(aux_hidden_states[0], torch.tensor([[15.0]]))
|
||||
|
||||
|
||||
def _dspark_vocab_stub(input_vocab_size: int, draft_vocab_size: int):
|
||||
"""A Qwen3 DSpark model with only what `load_weights` reads."""
|
||||
from vllm.model_executor.models.qwen3_dspark import Qwen3DSparkForCausalLM
|
||||
|
||||
model = Qwen3DSparkForCausalLM.__new__(Qwen3DSparkForCausalLM)
|
||||
nn.Module.__init__(model)
|
||||
object.__setattr__(
|
||||
model,
|
||||
"config",
|
||||
SimpleNamespace(
|
||||
vocab_size=input_vocab_size,
|
||||
draft_vocab_size=draft_vocab_size,
|
||||
),
|
||||
)
|
||||
object.__setattr__(model, "target_vocab_size", 100)
|
||||
object.__setattr__(model, "model", Mock(confidence_head=None))
|
||||
return model
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
@pytest.mark.parametrize(
|
||||
("input_vocab_size", "draft_vocab_size", "weights", "error"),
|
||||
[
|
||||
(101, 100, [], "must include embed_tokens weights"),
|
||||
(99, 99, [], "must include lm_head weights"),
|
||||
(100, 40, [], "must include lm_head weights"),
|
||||
(
|
||||
100,
|
||||
40,
|
||||
[("lm_head.weight", torch.empty(40, 8))],
|
||||
"must include a d2t mapping",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_qwen3_dspark_rejects_incomplete_vocab_weights(
|
||||
input_vocab_size, draft_vocab_size, weights, error
|
||||
):
|
||||
model = _dspark_vocab_stub(input_vocab_size, draft_vocab_size)
|
||||
|
||||
with pytest.raises(ValueError, match=error):
|
||||
model.load_weights(weights)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
@pytest.mark.parametrize("draft_vocab_size", [100, 101])
|
||||
def test_qwen3_dspark_accepts_draft_vocab_at_least_target(draft_vocab_size):
|
||||
"""A draft vocab >= the target's maps identically, so needs no lm_head/d2t.
|
||||
|
||||
Guards the padded case (draft > target): the draft's unembedding is padded
|
||||
to its physical size, which is not a reduced vocabulary.
|
||||
"""
|
||||
model = _dspark_vocab_stub(100, draft_vocab_size)
|
||||
|
||||
model.load_weights([])
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_dspark_shares_target_embedding_with_smaller_draft_vocabulary():
|
||||
from vllm.v1.worker.gpu.spec_decode.dspark import utils as dspark_utils
|
||||
|
||||
target_embedding = nn.Embedding(100, 8)
|
||||
draft_embedding = nn.Embedding(99, 8)
|
||||
target_model = SimpleNamespace(model=SimpleNamespace(embed_tokens=target_embedding))
|
||||
draft_model = SimpleNamespace(
|
||||
model=SimpleNamespace(embed_tokens=draft_embedding),
|
||||
has_own_embed_tokens=False,
|
||||
)
|
||||
draft_model_config = SimpleNamespace(
|
||||
hf_config=SimpleNamespace(model_type="qwen3"),
|
||||
get_vocab_size=Mock(return_value=99),
|
||||
)
|
||||
vllm_config = SimpleNamespace(
|
||||
speculative_config=SimpleNamespace(
|
||||
draft_model_config=draft_model_config,
|
||||
draft_parallel_config=SimpleNamespace(tensor_parallel_size=1),
|
||||
attention_backend=None,
|
||||
kv_cache_dtype=None,
|
||||
draft_load_config=None,
|
||||
),
|
||||
parallel_config=ParallelConfig(),
|
||||
attention_config=SimpleNamespace(backend=None),
|
||||
cache_config=SimpleNamespace(),
|
||||
load_config=SimpleNamespace(load_format="auto"),
|
||||
model_config=SimpleNamespace(get_vocab_size=Mock(return_value=100)),
|
||||
)
|
||||
|
||||
def fake_replace(config, **changes):
|
||||
values = vars(config).copy()
|
||||
values.update(changes)
|
||||
return SimpleNamespace(**values)
|
||||
|
||||
with (
|
||||
patch.object(dspark_utils, "replace", side_effect=fake_replace),
|
||||
patch(
|
||||
"vllm.v1.worker.gpu.spec_decode.eagle.utils.get_pp_group",
|
||||
return_value=SimpleNamespace(world_size=1),
|
||||
),
|
||||
patch(
|
||||
"vllm.v1.worker.gpu.spec_decode.utils.get_pp_group",
|
||||
return_value=SimpleNamespace(world_size=1),
|
||||
),
|
||||
patch(
|
||||
"vllm.compilation.backends.set_model_tag",
|
||||
return_value=nullcontext(),
|
||||
),
|
||||
patch(
|
||||
"vllm.model_executor.model_loader.get_model",
|
||||
return_value=draft_model,
|
||||
),
|
||||
patch(
|
||||
"vllm.model_executor.models.qwen3_dflash.dflash_has_any_non_causal",
|
||||
return_value=False,
|
||||
),
|
||||
patch(
|
||||
"vllm.model_executor.models.utils.get_draft_quant_config",
|
||||
return_value=None,
|
||||
),
|
||||
):
|
||||
loaded_model = dspark_utils.load_dspark_model(target_model, vllm_config)
|
||||
|
||||
assert loaded_model.model.embed_tokens is target_embedding
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
@@ -1,216 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import dataclasses
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.qwen3_omni_moe_thinker import (
|
||||
Qwen3OmniMoeThinkerForConditionalGeneration,
|
||||
)
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldElem,
|
||||
MultiModalKwargsItem,
|
||||
PlaceholderRange,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True, scope="module")
|
||||
def _force_cpu_default_device():
|
||||
# get_mrope_input_positions returns CPU tensors (via torch.from_numpy).
|
||||
# Ensure the default device is CPU so the rest of the test tensors match.
|
||||
original = torch.get_default_device()
|
||||
torch.set_default_device("cpu")
|
||||
yield
|
||||
torch.set_default_device(original)
|
||||
|
||||
|
||||
TEXT_TOKEN_ID = 1
|
||||
VISION_START_TOKEN_ID = 100
|
||||
VISION_END_TOKEN_ID = 101
|
||||
IMAGE_TOKEN_ID = 200
|
||||
VIDEO_TOKEN_ID = 201
|
||||
AUDIO_TOKEN_ID = 202
|
||||
AUDIO_START_TOKEN_ID = 300
|
||||
AUDIO_END_TOKEN_ID = 301
|
||||
|
||||
|
||||
@dataclass
|
||||
class DummyVisionConfig:
|
||||
spatial_merge_size: int = 1
|
||||
|
||||
|
||||
@dataclass
|
||||
class DummyConfig:
|
||||
position_id_per_seconds: int = 25
|
||||
vision_config: DummyVisionConfig = dataclasses.field(
|
||||
default_factory=DummyVisionConfig
|
||||
)
|
||||
|
||||
|
||||
def make_model() -> Qwen3OmniMoeThinkerForConditionalGeneration:
|
||||
model = Qwen3OmniMoeThinkerForConditionalGeneration.__new__(
|
||||
Qwen3OmniMoeThinkerForConditionalGeneration
|
||||
)
|
||||
model.config = DummyConfig()
|
||||
return model
|
||||
|
||||
|
||||
def _elem(data):
|
||||
return MultiModalFieldElem(
|
||||
data=torch.tensor(data),
|
||||
field=None, # HACK.
|
||||
)
|
||||
|
||||
|
||||
def make_image_feature(offset: int, grid_thw: tuple[int, int, int]):
|
||||
return MultiModalFeatureSpec(
|
||||
data=MultiModalKwargsItem({"image_grid_thw": _elem(grid_thw)}),
|
||||
modality="image",
|
||||
identifier="DUMMY",
|
||||
mm_position=PlaceholderRange(
|
||||
offset=offset, length=int(torch.tensor(grid_thw).prod())
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def make_video_feature(
|
||||
offset: int, grid_thw: tuple[int, int, int], use_audio_in_video: bool = False
|
||||
):
|
||||
data = {"video_grid_thw": _elem(grid_thw)}
|
||||
if use_audio_in_video:
|
||||
data["use_audio_in_video"] = _elem(True)
|
||||
return MultiModalFeatureSpec(
|
||||
data=MultiModalKwargsItem(data),
|
||||
modality="video",
|
||||
identifier="DUMMY",
|
||||
mm_position=PlaceholderRange(
|
||||
offset=offset, length=int(torch.tensor(grid_thw).prod())
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def make_audio_feature(offset: int, audio_feature_length: int):
|
||||
return MultiModalFeatureSpec(
|
||||
data=MultiModalKwargsItem(
|
||||
{"audio_feature_lengths": _elem(audio_feature_length)}
|
||||
),
|
||||
modality="audio",
|
||||
identifier="DUMMY",
|
||||
mm_position=PlaceholderRange(offset=offset, length=0),
|
||||
)
|
||||
|
||||
|
||||
def test_mrope_positions_image_at_prompt_end():
|
||||
"""A prompt ending with <|vision_end|> must still produce seq_len positions."""
|
||||
model = make_model()
|
||||
# [<|vision_start|>, <|image_pad|> x 4, <|vision_end|>]
|
||||
input_tokens = (
|
||||
[VISION_START_TOKEN_ID] + [IMAGE_TOKEN_ID] * 4 + [VISION_END_TOKEN_ID]
|
||||
)
|
||||
mm_features = [make_image_feature(offset=1, grid_thw=(1, 2, 2))]
|
||||
|
||||
positions, _ = model.get_mrope_input_positions(input_tokens, mm_features)
|
||||
|
||||
assert positions.shape == (3, len(input_tokens))
|
||||
|
||||
|
||||
def test_mrope_positions_audio_at_prompt_end():
|
||||
model = make_model()
|
||||
# audio_feature_length=16 -> 2 audio tokens.
|
||||
# [<|audio_start|>, <|audio_pad|> x 2, <|audio_end|>]
|
||||
input_tokens = [AUDIO_START_TOKEN_ID] + [AUDIO_TOKEN_ID] * 2 + [AUDIO_END_TOKEN_ID]
|
||||
mm_features = [make_audio_feature(offset=1, audio_feature_length=16)]
|
||||
|
||||
positions, _ = model.get_mrope_input_positions(input_tokens, mm_features)
|
||||
|
||||
assert positions.shape == (3, len(input_tokens))
|
||||
|
||||
|
||||
def test_mrope_positions_video_at_prompt_end():
|
||||
model = make_model()
|
||||
# [T, <|vision_start|>, <|video_pad|> x 8, <|vision_end|>]
|
||||
input_tokens = (
|
||||
[TEXT_TOKEN_ID]
|
||||
+ [VISION_START_TOKEN_ID]
|
||||
+ [VIDEO_TOKEN_ID] * 8
|
||||
+ [VISION_END_TOKEN_ID]
|
||||
)
|
||||
mm_features = [make_video_feature(offset=2, grid_thw=(2, 2, 2))]
|
||||
|
||||
positions, _ = model.get_mrope_input_positions(input_tokens, mm_features)
|
||||
|
||||
assert positions.shape == (3, len(input_tokens))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_videos", [1, 2])
|
||||
@pytest.mark.parametrize("trailing_tokens", [0, 3])
|
||||
def test_mrope_positions_audio_in_video_boundaries(num_videos, trailing_tokens):
|
||||
"""Interleaved BOS/EOS positions and decode delta match Qwen3-Omni HF."""
|
||||
model = make_model()
|
||||
video_tokens = (
|
||||
[VISION_START_TOKEN_ID, AUDIO_START_TOKEN_ID]
|
||||
+ [VIDEO_TOKEN_ID] * 4
|
||||
+ [AUDIO_TOKEN_ID] * 2
|
||||
+ [AUDIO_END_TOKEN_ID, VISION_END_TOKEN_ID]
|
||||
)
|
||||
input_tokens = [TEXT_TOKEN_ID] * 2
|
||||
mm_features = []
|
||||
expected = [torch.tensor([[0, 1]]).expand(3, -1)]
|
||||
video_positions = torch.tensor(
|
||||
[
|
||||
[2, 3, 4, 4, 4, 4, 4, 5, 6, 7],
|
||||
[2, 3, 4, 4, 5, 5, 4, 5, 6, 7],
|
||||
[2, 3, 4, 5, 4, 5, 4, 5, 6, 7],
|
||||
]
|
||||
)
|
||||
for index in range(num_videos):
|
||||
offset = len(input_tokens) + 1
|
||||
mm_features.extend(
|
||||
[
|
||||
make_video_feature(offset, (1, 2, 2), use_audio_in_video=True),
|
||||
make_audio_feature(offset, audio_feature_length=16),
|
||||
]
|
||||
)
|
||||
input_tokens.extend(video_tokens)
|
||||
expected.append(video_positions + index * 6)
|
||||
|
||||
input_tokens.extend([TEXT_TOKEN_ID] * trailing_tokens)
|
||||
next_position = 2 + num_videos * 6
|
||||
expected.append((torch.arange(trailing_tokens) + next_position).expand(3, -1))
|
||||
positions, delta = model.get_mrope_input_positions(input_tokens, mm_features)
|
||||
|
||||
torch.testing.assert_close(positions, torch.cat(expected, dim=1))
|
||||
assert delta == -4 * num_videos
|
||||
|
||||
|
||||
def test_mrope_positions_image_then_text():
|
||||
"""Media followed by text: positions must match the expected layout exactly.
|
||||
|
||||
Before the fix this case passed the length guard but every position from
|
||||
the first <|image_pad|> onward was silently shifted by one slot.
|
||||
"""
|
||||
model = make_model()
|
||||
# [T, T, <|vision_start|>, <|image_pad|> x 4, <|vision_end|>, T, T, T]
|
||||
input_tokens = (
|
||||
[TEXT_TOKEN_ID] * 2
|
||||
+ [VISION_START_TOKEN_ID]
|
||||
+ [IMAGE_TOKEN_ID] * 4
|
||||
+ [VISION_END_TOKEN_ID]
|
||||
+ [TEXT_TOKEN_ID] * 3
|
||||
)
|
||||
mm_features = [make_image_feature(offset=3, grid_thw=(1, 2, 2))]
|
||||
|
||||
positions, delta = model.get_mrope_input_positions(input_tokens, mm_features)
|
||||
|
||||
expected = torch.tensor(
|
||||
[
|
||||
[0, 1, 2, 3, 3, 3, 3, 5, 6, 7, 8],
|
||||
[0, 1, 2, 3, 3, 4, 4, 5, 6, 7, 8],
|
||||
[0, 1, 2, 3, 4, 3, 4, 5, 6, 7, 8],
|
||||
]
|
||||
)
|
||||
torch.testing.assert_close(positions, expected)
|
||||
assert delta == 9 - len(input_tokens)
|
||||
@@ -1,237 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import dataclasses
|
||||
import random
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.qwen3_vl import Qwen3VLForConditionalGeneration
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldElem,
|
||||
MultiModalKwargsItem,
|
||||
PlaceholderRange,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True, scope="module")
|
||||
def _force_cpu_default_device():
|
||||
# _get_mrope_input_positions returns CPU tensors (via torch.from_numpy).
|
||||
# Ensure the default device is CPU so the rest of the test tensors match.
|
||||
original = torch.get_default_device()
|
||||
torch.set_default_device("cpu")
|
||||
yield
|
||||
torch.set_default_device(original)
|
||||
|
||||
|
||||
IMAGE_TOKEN_ID = 999
|
||||
VIDEO_TOKEN_ID = 888
|
||||
VISION_START_TOKEN_ID = 777
|
||||
VISION_END_TOKEN_ID = 778
|
||||
|
||||
|
||||
@dataclass
|
||||
class DummyVisionConfig:
|
||||
spatial_merge_size: int = 1
|
||||
|
||||
|
||||
@dataclass
|
||||
class DummyConfig:
|
||||
image_token_id: int = IMAGE_TOKEN_ID
|
||||
video_token_id: int = VIDEO_TOKEN_ID
|
||||
vision_start_token_id: int = VISION_START_TOKEN_ID
|
||||
vision_end_token_id: int = VISION_END_TOKEN_ID
|
||||
vision_config: DummyVisionConfig = dataclasses.field(
|
||||
default_factory=DummyVisionConfig
|
||||
)
|
||||
|
||||
|
||||
def make_video_embedding(
|
||||
t, h, w, interleave_text_tokens: tuple[int, int], video_pruning_rate: float = 0.0
|
||||
):
|
||||
"""Make a video embedding for a given video size and pruning rate.
|
||||
|
||||
Args:
|
||||
t: Number of frames.
|
||||
h: Number of rows.
|
||||
w: Number of columns.
|
||||
interleave_text_tokens: Tuple of minimum and maximum number of text tokens to
|
||||
interleave with the video.
|
||||
video_pruning_rate: Pruning rate for the video.
|
||||
|
||||
Returns:
|
||||
Tuple of (unpruned_tokens_sequence, pruned_tokens_sequence, retention_mask)
|
||||
|
||||
"""
|
||||
unpruned_tokens_sequence = []
|
||||
population = list(range(1, 100))
|
||||
|
||||
for _ in range(t):
|
||||
num_prefix_tokens = random.randint(
|
||||
interleave_text_tokens[0], interleave_text_tokens[1]
|
||||
)
|
||||
|
||||
prefix_tokens = random.choices(population, k=num_prefix_tokens)
|
||||
vision_tokens = (
|
||||
[VISION_START_TOKEN_ID] + [VIDEO_TOKEN_ID] * h * w + [VISION_END_TOKEN_ID]
|
||||
)
|
||||
|
||||
unpruned_tokens_sequence.extend(prefix_tokens)
|
||||
unpruned_tokens_sequence.extend(vision_tokens)
|
||||
|
||||
unpruned_tokens_sequence = torch.tensor(unpruned_tokens_sequence, dtype=torch.long)
|
||||
video_token_mask = unpruned_tokens_sequence == VIDEO_TOKEN_ID
|
||||
|
||||
pruning_mask = torch.bernoulli(video_token_mask.float() * video_pruning_rate).bool() # type: ignore[attr-defined]
|
||||
# Sanity check that we don't prune what should not be pruned.
|
||||
assert not pruning_mask[~video_token_mask].any()
|
||||
|
||||
retention_mask = ~pruning_mask
|
||||
pruned_tokens_sequence = unpruned_tokens_sequence[retention_mask]
|
||||
return unpruned_tokens_sequence, pruned_tokens_sequence, retention_mask
|
||||
|
||||
|
||||
@pytest.mark.parametrize("spatial_merge_size", [1, 2])
|
||||
@pytest.mark.parametrize("grid_thw", [[3, 8, 7], [128, 10, 12]])
|
||||
@pytest.mark.parametrize("num_prefix_tokens", [1, 11])
|
||||
@pytest.mark.parametrize("num_suffix_tokens", [0, 7])
|
||||
@pytest.mark.parametrize("video_pruning_rate", [0, 0.25, 0.75])
|
||||
@pytest.mark.parametrize("interleave_text_tokens", [(0, 0), (1, 4)])
|
||||
def test_match_qwen3vl_mrope_evs_on(
|
||||
spatial_merge_size: int,
|
||||
num_prefix_tokens: int,
|
||||
grid_thw: tuple[int, int, int],
|
||||
num_suffix_tokens: int,
|
||||
video_pruning_rate: float,
|
||||
interleave_text_tokens: tuple[int, int],
|
||||
):
|
||||
hf_config = DummyConfig()
|
||||
hf_config.vision_config.spatial_merge_size = spatial_merge_size
|
||||
|
||||
t, h, w = grid_thw
|
||||
population = list(range(1, 100))
|
||||
prefix_tokens = random.choices(population, k=num_prefix_tokens)
|
||||
suffix_tokens = random.choices(population, k=num_suffix_tokens)
|
||||
|
||||
video_tokens, video_tokens_pruned, retention_mask = make_video_embedding(
|
||||
t,
|
||||
h // spatial_merge_size,
|
||||
w // spatial_merge_size,
|
||||
interleave_text_tokens=interleave_text_tokens,
|
||||
video_pruning_rate=video_pruning_rate,
|
||||
)
|
||||
assert len(video_tokens) == len(retention_mask)
|
||||
|
||||
input_tokens = prefix_tokens + video_tokens.tolist() + suffix_tokens
|
||||
input_tokens_pruned = prefix_tokens + video_tokens_pruned.tolist() + suffix_tokens
|
||||
|
||||
whole_sequence_retention_mask = torch.cat(
|
||||
[
|
||||
torch.ones(len(prefix_tokens), dtype=torch.bool),
|
||||
retention_mask,
|
||||
torch.ones(len(suffix_tokens), dtype=torch.bool),
|
||||
],
|
||||
dim=0,
|
||||
)
|
||||
|
||||
# Build the GT mrope for unpruned input.
|
||||
mm_feature = MultiModalFeatureSpec(
|
||||
data=MultiModalKwargsItem(
|
||||
{
|
||||
"video_grid_thw": MultiModalFieldElem(
|
||||
data=torch.tensor(grid_thw),
|
||||
field=None, # HACK.
|
||||
),
|
||||
}
|
||||
),
|
||||
modality="video",
|
||||
identifier="DUMMY",
|
||||
mm_position=PlaceholderRange(offset=0, length=len(input_tokens)),
|
||||
)
|
||||
expected_mrope, _ = Qwen3VLForConditionalGeneration._get_mrope_input_positions(
|
||||
input_tokens=input_tokens,
|
||||
mm_features=[mm_feature],
|
||||
config=hf_config,
|
||||
)
|
||||
|
||||
# Compute mrope for a video-only media (unpruned).
|
||||
mm_feature = MultiModalFeatureSpec(
|
||||
data=MultiModalKwargsItem(
|
||||
{
|
||||
"video_grid_thw": MultiModalFieldElem(
|
||||
data=torch.tensor(grid_thw),
|
||||
field=None, # HACK.
|
||||
),
|
||||
}
|
||||
),
|
||||
modality="video",
|
||||
identifier="DUMMY",
|
||||
mm_position=PlaceholderRange(offset=0, length=video_tokens.numel()),
|
||||
)
|
||||
video_mrope, _ = Qwen3VLForConditionalGeneration._get_mrope_input_positions(
|
||||
input_tokens=video_tokens.tolist(),
|
||||
mm_features=[mm_feature],
|
||||
config=hf_config,
|
||||
)
|
||||
video_mrope = video_mrope.permute(1, 0) # [N, 3]
|
||||
hidden_size = 16
|
||||
|
||||
is_video_embed = torch.isin(
|
||||
video_tokens_pruned, torch.tensor([VIDEO_TOKEN_ID], dtype=torch.long)
|
||||
)
|
||||
|
||||
expanded_positions = torch.full(
|
||||
(len(video_tokens_pruned), 5),
|
||||
fill_value=-100,
|
||||
device=video_mrope.device,
|
||||
dtype=torch.long,
|
||||
)
|
||||
expanded_positions[is_video_embed, :3] = video_mrope[retention_mask][is_video_embed]
|
||||
expanded_positions[~is_video_embed, :3] = video_mrope[retention_mask][
|
||||
~is_video_embed
|
||||
]
|
||||
|
||||
is_vision_start = video_tokens_pruned == VISION_START_TOKEN_ID
|
||||
expanded_positions[..., 3] = is_vision_start
|
||||
expanded_positions[..., 4] = is_video_embed
|
||||
|
||||
# Check that all positions were filled, since we initialized them as negative.
|
||||
assert (expanded_positions >= 0).all()
|
||||
|
||||
video_embeddings = torch.empty(
|
||||
(len(video_tokens_pruned), hidden_size), device=video_mrope.device
|
||||
)
|
||||
|
||||
video_embeddings = torch.cat(
|
||||
[
|
||||
video_embeddings,
|
||||
expanded_positions.float(),
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
multimodal_embeddings = [video_embeddings]
|
||||
|
||||
expected_mrope_masked = expected_mrope[:, whole_sequence_retention_mask]
|
||||
|
||||
# Initialize computed_mrope with sequential positions for all prefix tokens
|
||||
computed_mrope = torch.empty((3, len(input_tokens_pruned)), dtype=torch.long)
|
||||
computed_mrope[:, 0 : len(prefix_tokens)] = expected_mrope[
|
||||
:, 0 : len(prefix_tokens)
|
||||
]
|
||||
|
||||
# Paranoia check that computed_mrope is wrong.
|
||||
assert not torch.equal(computed_mrope, expected_mrope_masked)
|
||||
|
||||
_, actual_mrope, _ = Qwen3VLForConditionalGeneration._recompute_mrope_positions(
|
||||
input_ids=input_tokens_pruned,
|
||||
multimodal_embeddings=multimodal_embeddings,
|
||||
mrope_positions=computed_mrope,
|
||||
num_computed_tokens=len(prefix_tokens),
|
||||
vision_start_token_id=hf_config.vision_start_token_id,
|
||||
image_token_id=hf_config.image_token_id,
|
||||
video_token_id=hf_config.video_token_id,
|
||||
)
|
||||
|
||||
assert torch.equal(actual_mrope, expected_mrope_masked)
|
||||
@@ -1,124 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.warmup.qwen_vl_triton_warmup import (
|
||||
_warm_mrope,
|
||||
_warm_vision,
|
||||
)
|
||||
|
||||
|
||||
def test_vision_warmup_calls_only_position_and_rotary_paths() -> None:
|
||||
from vllm.model_executor.models.qwen3_vl import Qwen3_VisionTransformer
|
||||
|
||||
calls: list[tuple[str, object]] = []
|
||||
|
||||
class FakeAttention:
|
||||
num_attention_heads_per_partition = 4
|
||||
hidden_size_per_attention_head = 8
|
||||
|
||||
def apply_rotary_emb(self, qk, cos, sin):
|
||||
calls.append(("rotary", qk.shape))
|
||||
|
||||
class FakeVisual(Qwen3_VisionTransformer):
|
||||
spatial_merge_size = 2
|
||||
|
||||
def __init__(self) -> None:
|
||||
torch.nn.Module.__init__(self)
|
||||
self.blocks = [SimpleNamespace(attn=FakeAttention())]
|
||||
self.pos_embed = torch.nn.Embedding(4, 4, device="cpu")
|
||||
|
||||
def fast_pos_embed_interpolate(self, grid_thw):
|
||||
calls.append(("position", grid_thw[0]))
|
||||
|
||||
def rot_pos_emb(self, grid_thw):
|
||||
_, h, w = grid_thw[0]
|
||||
shape = (h * w, 4)
|
||||
return torch.empty(shape), torch.empty(shape)
|
||||
|
||||
def forward(self, *args, **kwargs):
|
||||
raise AssertionError("vision warmup must not run the full tower")
|
||||
|
||||
model = torch.nn.Module()
|
||||
model.visual = FakeVisual()
|
||||
_warm_vision(model, torch.device("cpu"))
|
||||
|
||||
grids = [(1, 16, 16), (1, 16, 2), (1, 2, 16), (1, 2, 2)]
|
||||
assert [value for name, value in calls if name == "position"] == [
|
||||
list(grid) for grid in grids
|
||||
]
|
||||
assert [value for name, value in calls if name == "rotary"] == [
|
||||
torch.Size((2, h * w, 4, 8)) for _, h, w in grids
|
||||
]
|
||||
|
||||
# CPU offloading leaves tower weights off the accelerator, so the Triton
|
||||
# paths must not be launched on host pointers.
|
||||
calls.clear()
|
||||
_warm_vision(model, torch.device("cuda"))
|
||||
assert calls == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"owners_fused, skipped", [((), False), ((True,), True), ((True, False), False)]
|
||||
)
|
||||
def test_mrope_warmup_launches_triton_shapes(owners_fused, skipped) -> None:
|
||||
from vllm.model_executor.layers.rotary_embedding.mrope import MRotaryEmbedding
|
||||
|
||||
launched: list[tuple[torch.Size, torch.Size]] = []
|
||||
|
||||
class FakeRope(MRotaryEmbedding):
|
||||
def __init__(self) -> None:
|
||||
torch.nn.Module.__init__(self)
|
||||
self.head_size = 8
|
||||
self.rotary_dim = 8
|
||||
self.mrope_section = [2, 3, 3]
|
||||
self.mrope_interleaved = False
|
||||
self.is_neox_style = True
|
||||
|
||||
def forward(self, positions, query, key):
|
||||
launched.append((positions.shape, query.shape))
|
||||
return query, key
|
||||
|
||||
class FakeMropeModel(torch.nn.Module):
|
||||
supports_mrope = True
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
rope = FakeRope()
|
||||
if not owners_fused:
|
||||
self.rotary_emb = rope
|
||||
for i, fused in enumerate(owners_fused):
|
||||
owner = torch.nn.Module()
|
||||
owner.rotary_emb, owner.use_fused_qk_norm_rope_gate = rope, fused
|
||||
self.add_module(f"attn{i}", owner)
|
||||
|
||||
def get_mrope_input_positions(self, input_tokens, mm_features):
|
||||
raise AssertionError("warmup must not build live M-RoPE positions")
|
||||
|
||||
runner = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
get_num_attention_heads=lambda parallel_config: 4,
|
||||
get_num_kv_heads=lambda parallel_config: 2,
|
||||
),
|
||||
parallel_config=object(),
|
||||
dtype=torch.bfloat16,
|
||||
device=torch.device("cpu"),
|
||||
)
|
||||
_warm_mrope(runner, FakeMropeModel())
|
||||
expected = [torch.Size((3, n)) for n in (1, 2, 16)]
|
||||
assert [shape for shape, _ in launched] == ([] if skipped else expected)
|
||||
|
||||
|
||||
def test_mrope_warmup_skips_models_without_supports_mrope() -> None:
|
||||
def fail(*_args, **_kwargs):
|
||||
raise AssertionError("M-RoPE warmup must not run without SupportsMRoPE")
|
||||
|
||||
runner = SimpleNamespace(
|
||||
model_config=SimpleNamespace(get_num_attention_heads=fail),
|
||||
get_model=fail,
|
||||
)
|
||||
_warm_mrope(runner, torch.nn.Module())
|
||||
@@ -1,339 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import Gemma3nTextConfig
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.inputs import TokensPrompt
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ..models.utils import dummy_hf_overrides
|
||||
from ..utils import create_new_process_for_each_test
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SleepModelCase:
|
||||
name: str
|
||||
architecture: str
|
||||
model: str | None
|
||||
tensor_names: tuple[str, ...]
|
||||
modalities: tuple[str, ...] = ()
|
||||
revision: str | None = None
|
||||
trust_remote_code: bool = False
|
||||
tokenizer_mode: str | None = None
|
||||
|
||||
|
||||
SLEEP_MODEL_CASES = (
|
||||
SleepModelCase(
|
||||
name="fireredasr2",
|
||||
architecture="FireRedASR2ForConditionalGeneration",
|
||||
model="allendou/FireRedASR2-LLM-vllm",
|
||||
tensor_names=("pe",),
|
||||
trust_remote_code=True,
|
||||
),
|
||||
SleepModelCase(
|
||||
name="gemma3n",
|
||||
architecture="Gemma3nForCausalLM",
|
||||
model=None,
|
||||
tensor_names=(
|
||||
"router_input_scale",
|
||||
"embed_scale",
|
||||
"embed_scale_per_layer",
|
||||
"per_layer_input_scale",
|
||||
"per_layer_projection_scale",
|
||||
),
|
||||
),
|
||||
SleepModelCase(
|
||||
name="voxtral",
|
||||
architecture="VoxtralForConditionalGeneration",
|
||||
model="mistralai/Voxtral-Mini-3B-2507",
|
||||
tensor_names=("mel_filters",),
|
||||
revision="3060fe34b35ba5d44202ce9ff3c097642914f8f3",
|
||||
tokenizer_mode="mistral",
|
||||
),
|
||||
SleepModelCase(
|
||||
name="ernie45_vl",
|
||||
architecture="Ernie4_5_VLMoeForConditionalGeneration",
|
||||
model="baidu/ERNIE-4.5-VL-28B-A3B-PT",
|
||||
tensor_names=("inv_freq", "_visual_token_ids_tensor_cache"),
|
||||
revision="refs/pr/17",
|
||||
trust_remote_code=True,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _create_local_model_config(case: SleepModelCase, model_dir: Path) -> str:
|
||||
if case.name == "gemma3n":
|
||||
config = Gemma3nTextConfig(
|
||||
architectures=[case.architecture],
|
||||
vocab_size=128,
|
||||
hidden_size=64,
|
||||
intermediate_size=[128, 128, 128],
|
||||
num_hidden_layers=3,
|
||||
num_attention_heads=2,
|
||||
num_key_value_heads=2,
|
||||
head_dim=32,
|
||||
max_position_embeddings=256,
|
||||
sliding_window=64,
|
||||
layer_types=[
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
],
|
||||
vocab_size_per_layer_input=128,
|
||||
hidden_size_per_layer_input=16,
|
||||
num_kv_shared_layers=1,
|
||||
laurel_rank=16,
|
||||
activation_sparsity_pattern=[0.0, 0.0, 0.0],
|
||||
)
|
||||
else:
|
||||
raise AssertionError(f"No local config builder for {case.name}")
|
||||
|
||||
model_dir.mkdir()
|
||||
config.save_pretrained(model_dir)
|
||||
return str(model_dir)
|
||||
|
||||
|
||||
def _sleep_test_hf_overrides(hf_config, *, model_arch: str):
|
||||
hf_config = dummy_hf_overrides(hf_config, model_arch=model_arch)
|
||||
if model_arch == "FireRedASR2ForConditionalGeneration":
|
||||
audio_encoder_conf = getattr(hf_config, "audio_encoder_conf", {})
|
||||
audio_encoder_conf.update(
|
||||
{
|
||||
"idim": 80,
|
||||
"n_layers_enc": 1,
|
||||
"n_head": 1,
|
||||
"d_model": 32,
|
||||
"kernel_size": 3,
|
||||
# MM profiling uses the repository's real feature extractor
|
||||
# and can produce roughly 750 encoder frames. Keep the real
|
||||
# positional-encoding capacity instead of shrinking it with
|
||||
# the number of test decoder tokens.
|
||||
"pe_maxlen": 5000,
|
||||
}
|
||||
)
|
||||
hf_config.audio_encoder_conf = audio_encoder_conf
|
||||
if (
|
||||
model_arch == "Ernie4_5_VLMoeForConditionalGeneration"
|
||||
and getattr(hf_config, "im_patch_id", None) is None
|
||||
):
|
||||
# Some revisions expose this id as image_token_id, while the native
|
||||
# model intentionally caches it under the older im_patch_id name.
|
||||
hf_config.im_patch_id = getattr(hf_config, "image_token_id", 100295)
|
||||
return hf_config
|
||||
|
||||
|
||||
def _get_sleep_tensor_targets(
|
||||
model_name: str,
|
||||
) -> tuple[tuple[type[torch.nn.Module], tuple[str, ...]], ...]:
|
||||
if model_name == "fireredasr2":
|
||||
from vllm.model_executor.models.conformer_encoder import (
|
||||
RelPositionalEncoding,
|
||||
)
|
||||
|
||||
return ((RelPositionalEncoding, ("pe",)),)
|
||||
if model_name == "gemma3n":
|
||||
from vllm.model_executor.models.gemma3n import (
|
||||
Gemma3nAltUp,
|
||||
Gemma3nSelfDecoder,
|
||||
)
|
||||
|
||||
return (
|
||||
(Gemma3nAltUp, ("router_input_scale",)),
|
||||
(
|
||||
Gemma3nSelfDecoder,
|
||||
(
|
||||
"embed_scale",
|
||||
"embed_scale_per_layer",
|
||||
"per_layer_input_scale",
|
||||
"per_layer_projection_scale",
|
||||
),
|
||||
),
|
||||
)
|
||||
if model_name == "voxtral":
|
||||
from vllm.model_executor.models.voxtral import VoxtralEncoderModel
|
||||
|
||||
return ((VoxtralEncoderModel, ("mel_filters",)),)
|
||||
if model_name == "ernie45_vl":
|
||||
from vllm.model_executor.models.ernie45_vl import (
|
||||
Ernie4_5_VisionRotaryEmbedding,
|
||||
Ernie4_5_VLMoeForConditionalGeneration,
|
||||
)
|
||||
|
||||
return (
|
||||
(Ernie4_5_VisionRotaryEmbedding, ("inv_freq",)),
|
||||
(
|
||||
Ernie4_5_VLMoeForConditionalGeneration,
|
||||
("_visual_token_ids_tensor_cache",),
|
||||
),
|
||||
)
|
||||
raise AssertionError(f"No sleep tensor targets for {model_name}")
|
||||
|
||||
|
||||
def _snapshot_sleep_tensors(
|
||||
model: torch.nn.Module,
|
||||
*,
|
||||
model_name: str,
|
||||
tensor_names: tuple[str, ...],
|
||||
) -> dict[str, tuple[int, torch.Tensor]]:
|
||||
targets = _get_sleep_tensor_targets(model_name)
|
||||
snapshot: dict[str, tuple[int, torch.Tensor]] = {}
|
||||
found_tensor_names = set[str]()
|
||||
|
||||
for module_path, module in model.named_modules():
|
||||
for module_type, target_names in targets:
|
||||
if not isinstance(module, module_type):
|
||||
continue
|
||||
for tensor_name in target_names:
|
||||
tensor = getattr(module, tensor_name, None)
|
||||
if tensor is None:
|
||||
continue
|
||||
assert isinstance(tensor, torch.Tensor)
|
||||
qualified_name = (
|
||||
f"{module_path}.{tensor_name}" if module_path else tensor_name
|
||||
)
|
||||
snapshot[qualified_name] = (
|
||||
tensor.data_ptr(),
|
||||
tensor.detach().cpu().clone(),
|
||||
)
|
||||
found_tensor_names.add(tensor_name)
|
||||
|
||||
assert found_tensor_names == set(tensor_names), (
|
||||
f"{model_name}: expected tensors {tensor_names}, "
|
||||
f"found {tuple(sorted(found_tensor_names))}"
|
||||
)
|
||||
return snapshot
|
||||
|
||||
|
||||
def _save_dummy_weights(worker) -> None:
|
||||
model = worker.model_runner.model
|
||||
worker._sleep_test_dummy_weights = [
|
||||
(name, parameter.detach().cpu().clone())
|
||||
for name, parameter in model.named_parameters()
|
||||
]
|
||||
|
||||
|
||||
def _reload_dummy_weights(worker) -> None:
|
||||
with torch.no_grad():
|
||||
worker.reload_weights(
|
||||
weights_iterator=iter(worker._sleep_test_dummy_weights),
|
||||
is_checkpoint_format=False,
|
||||
)
|
||||
|
||||
|
||||
def _generation_signature(output) -> tuple[tuple[int, ...], tuple[float, ...]]:
|
||||
completion = output[0].outputs[0]
|
||||
token_ids = tuple(completion.token_ids)
|
||||
assert completion.logprobs is not None
|
||||
chosen_logprobs = tuple(
|
||||
step[token_id].logprob
|
||||
for token_id, step in zip(token_ids, completion.logprobs, strict=True)
|
||||
)
|
||||
return token_ids, chosen_logprobs
|
||||
|
||||
|
||||
def _assert_tensor_snapshots_equal(
|
||||
before: dict[str, tuple[int, torch.Tensor]],
|
||||
after: dict[str, tuple[int, torch.Tensor]],
|
||||
model_name: str,
|
||||
) -> None:
|
||||
assert after.keys() == before.keys(), model_name
|
||||
for name, (before_ptr, before_value) in before.items():
|
||||
after_ptr, after_value = after[name]
|
||||
assert after_ptr == before_ptr, f"{model_name}: {name} address changed"
|
||||
torch.testing.assert_close(
|
||||
after_value,
|
||||
before_value,
|
||||
msg=lambda msg, name=name: (f"{model_name}: {name} was corrupted: {msg}"),
|
||||
)
|
||||
|
||||
|
||||
@create_new_process_for_each_test("spawn")
|
||||
@pytest.mark.slow_test
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA sleep mode test")
|
||||
@pytest.mark.parametrize("case", SLEEP_MODEL_CASES, ids=lambda case: case.name)
|
||||
def test_static_model_tensors_survive_level2_restore(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
case: SleepModelCase,
|
||||
) -> None:
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
|
||||
if case.name == "fireredasr2":
|
||||
# FireRedASR2 has a regular decoder and compute_logits implementation,
|
||||
# but advertises only the transcription task. Enable its text path so
|
||||
# this ownership test can exercise the common LLM.generate lifecycle
|
||||
# without constructing an unrelated audio fixture.
|
||||
from vllm.model_executor.models.fireredasr2 import (
|
||||
FireRedASR2ForConditionalGeneration,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
FireRedASR2ForConditionalGeneration,
|
||||
"supports_transcription_only",
|
||||
False,
|
||||
)
|
||||
|
||||
model = case.model or _create_local_model_config(case, tmp_path / case.name)
|
||||
llm_kwargs = {
|
||||
"model": model,
|
||||
"load_format": "dummy",
|
||||
"enable_sleep_mode": True,
|
||||
"enforce_eager": True,
|
||||
"attention_backend": "TRITON_ATTN",
|
||||
"max_model_len": 128,
|
||||
"max_num_seqs": 1,
|
||||
"gpu_memory_utilization": 0.5,
|
||||
"seed": 0,
|
||||
"enable_prefix_caching": False,
|
||||
"disable_log_stats": True,
|
||||
"trust_remote_code": case.trust_remote_code,
|
||||
}
|
||||
if case.revision is not None:
|
||||
llm_kwargs["revision"] = case.revision
|
||||
if case.name not in ("fireredasr2", "voxtral", "ernie45_vl"):
|
||||
llm_kwargs["skip_tokenizer_init"] = True
|
||||
if case.tokenizer_mode is not None:
|
||||
llm_kwargs["tokenizer_mode"] = case.tokenizer_mode
|
||||
if case.model is not None:
|
||||
llm_kwargs["hf_overrides"] = partial(
|
||||
_sleep_test_hf_overrides, model_arch=case.architecture
|
||||
)
|
||||
if case.modalities:
|
||||
llm_kwargs["limit_mm_per_prompt"] = {
|
||||
modality: 0 for modality in case.modalities
|
||||
}
|
||||
|
||||
llm = LLM(**llm_kwargs)
|
||||
prompt = TokensPrompt(prompt_token_ids=[2, 5, 9, 12, 7, 3])
|
||||
sampling_params = SamplingParams(temperature=0.0, max_tokens=8, logprobs=1)
|
||||
|
||||
before_generation = _generation_signature(llm.generate(prompt, sampling_params))
|
||||
snapshot_fn = partial(
|
||||
_snapshot_sleep_tensors,
|
||||
model_name=case.name,
|
||||
tensor_names=case.tensor_names,
|
||||
)
|
||||
(before_tensors,) = llm.apply_model(snapshot_fn)
|
||||
llm.collective_rpc(_save_dummy_weights)
|
||||
|
||||
llm.sleep(level=2)
|
||||
llm.wake_up(tags=["weights"])
|
||||
(after_tensors,) = llm.apply_model(snapshot_fn)
|
||||
_assert_tensor_snapshots_equal(before_tensors, after_tensors, case.name)
|
||||
|
||||
llm.collective_rpc(_reload_dummy_weights)
|
||||
llm.wake_up(tags=["kv_cache"])
|
||||
after_generation = _generation_signature(llm.generate(prompt, sampling_params))
|
||||
|
||||
assert after_generation[0] == before_generation[0], case.name
|
||||
assert after_generation[1] == pytest.approx(
|
||||
before_generation[1], rel=1e-5, abs=1e-5
|
||||
), case.name
|
||||
Reference in New Issue
Block a user