Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-09 17:41:13 +00:00
1122 changed files with 60114 additions and 20956 deletions
@@ -12,13 +12,16 @@ from unittest.mock import patch
import pytest
import torch
from torch import nn
from transformers import Qwen3_5MoeTextConfig
import vllm.config as vllm_config_module
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
from vllm.config.quantization import QuantizationConfigArgs
from vllm.model_executor.layers.fused_moe import utils as fused_moe_utils
from vllm.model_executor.layers.fused_moe.layer import determine_expert_counts
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm.model_executor.layers.quantization.experts_int8 import ExpertsInt8Config
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.model_executor.layers.quantization.online.base import OnlineQuantizationConfig
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
from vllm.model_executor.layers.quantization.utils.config_utils import (
get_quark_ocp_mx_group_size,
@@ -32,7 +35,6 @@ from vllm.models.deepseek_v4 import quant_config as deepseek_v4_quant_config
from vllm.models.minimax_m3.amd import model as minimax_m3_model
from vllm.platforms import current_platform
from vllm.transformers_utils.configs.minimax_m3 import MiniMaxM3TextConfig
from vllm.transformers_utils.configs.qwen3_5_moe import Qwen3_5MoeTextConfig
pytestmark = pytest.mark.skipif(
current_platform.is_xpu(),
@@ -141,8 +143,10 @@ def get_deepseek_v4_quark_config(exclude: list[str]) -> dict[str, Any]:
return quantization_config
def _stub_quant_config() -> QuantizationConfig:
return cast(QuantizationConfig, object())
class _StubGlm5NextAttention(nn.Module):
def __init__(self, *args: Any, **kwargs: Any) -> None:
super().__init__()
self.o_proj = SimpleNamespace(reduce_results=True)
def get_fse_test_model_config(
@@ -279,6 +283,33 @@ def get_fse_test_model_config(
quantization_config=quantization_config,
)
return config, Glm4MoeModel
if model_type == "glm5_next":
from transformers import Glm5NextTextConfig
from vllm.models.glm5next.common.model import Glm5NextModel
config = Glm5NextTextConfig(
vocab_size=256,
hidden_size=128,
intermediate_size=32,
moe_intermediate_size=32,
num_hidden_layers=1,
num_attention_heads=1,
num_key_value_heads=1,
n_routed_experts=2,
n_shared_experts=1,
num_experts_per_tok=1,
layer_types=["linear_attention"],
mlp_layer_types=["sparse"],
index_topk=1,
index_kpool=1,
pad_token_id=None,
scoring_func="sigmoid",
topk_method="noaux_tc",
mhc=False,
quantization_config=quantization_config,
)
return config, Glm5NextModel
raise ValueError(f"Unsupported FSE test model: {model_type}")
@@ -307,7 +338,7 @@ def test_resolve_layer_fused_shared_expert_skips_compatibility_when_disabled(
)
assert not fused_moe_utils.resolve_layer_fused_shared_expert(
_stub_quant_config(), "model.layers.0.mlp"
ExpertsInt8Config(), "model.layers.0.mlp"
)
@@ -322,7 +353,7 @@ def test_resolve_layer_fused_shared_expert_normalizes_unavailable_aiter(
assert (
fused_moe_utils.resolve_layer_fused_shared_expert(
_stub_quant_config(), "model.layers.0.mlp"
ExpertsInt8Config(), "model.layers.0.mlp"
)
is False
)
@@ -331,7 +362,7 @@ def test_resolve_layer_fused_shared_expert_normalizes_unavailable_aiter(
def test_resolve_layer_fused_shared_expert_passes_module_prefixes(
monkeypatch: pytest.MonkeyPatch,
) -> None:
quant_config = _stub_quant_config()
quant_config = ExpertsInt8Config()
monkeypatch.setattr(
fused_moe_utils.rocm_aiter_ops,
"is_fusion_moe_shared_experts_enabled",
@@ -380,7 +411,7 @@ def test_resolve_layer_fused_shared_expert_rejects_incompatible_quantization(
)
assert not fused_moe_utils.resolve_layer_fused_shared_expert(
_stub_quant_config(), "model.layers.0.mlp"
ExpertsInt8Config(), "model.layers.0.mlp"
)
assert "shared experts are excluded" in caplog.text
@@ -390,6 +421,7 @@ def test_deepseek_v4_shared_expert_fse_uses_mtp_quantization_config_prefix(
) -> None:
class DeepseekV4Config:
expert_dtype = "fp4"
online_quantization_config = None
def _is_quark_mxfp4_ocp(self, hf_config: object) -> bool:
return True
@@ -796,7 +828,7 @@ def test_is_model_fused_shared_expert_compatible() -> None:
@pytest.mark.parametrize(
"model_type",
["minimax_m3", "deepseek_v4", "qwen3_5", "glm4_moe", "deepseek_v2"],
["minimax_m3", "deepseek_v4", "qwen3_5", "glm4_moe", "deepseek_v2", "glm5_next"],
)
@pytest.mark.parametrize(
("use_fse", "exclude"),
@@ -850,6 +882,16 @@ def test_models_fse_init(
)
else:
vllm_config.quant_config = QuarkConfig(quantization_config)
if model_type == "glm5_next":
from vllm.models.glm5next.common import model as glm5_next_model
for attention in ("Glm5NextLinearAttention", "Glm5NextMLAAttention"):
monkeypatch.setattr(glm5_next_model, attention, _StubGlm5NextAttention)
# The gate allows fusion only at TP4/TP8 on gfx950, and this test
# runs at TP1; the gate has its own test.
monkeypatch.setattr(
glm5_next_model, "_fused_shared_experts_tuned", lambda _: True
)
import vllm.envs as envs
from vllm._aiter_ops import rocm_aiter_ops
@@ -902,6 +944,10 @@ def test_models_fse_init(
)
mtp = DeepSeekV4MTP(vllm_config=vllm_config)
assert model.is_fused_shared_expert_enabled is (fse_enabled and not exclude)
if model_type == "glm5_next":
assert (model.layers[0].mlp.shared_experts is None) is (
fse_enabled and not exclude
)
# The dummy quant config here uses mixed mxfp4/fp8 for experts/shared_expert
# so should just raise a warning.
@@ -926,6 +972,74 @@ def test_models_fse_init(
rocm_aiter_ops.refresh_env_variables()
@pytest.mark.parametrize(
("setting", "value", "reason"),
[
("tensor_parallel_size", 4, None),
("tensor_parallel_size", 8, None),
("tensor_parallel_size", 2, "tensor_parallel_size is 2"),
("data_parallel_size", 2, "data_parallel_size is 2"),
("prefill_context_parallel_size", 2, "prefill_context_parallel_size is 2"),
("enable_expert_parallel", True, "expert parallelism is enabled"),
("on_gfx950", False, "the GPU is not gfx950"),
],
)
def test_glm5_next_fuses_shared_experts_only_in_tuned_setups(
monkeypatch: pytest.MonkeyPatch,
setting: str,
value: object,
reason: str | None,
) -> None:
from vllm.models.glm5next.common import model as glm5_next_model
parallel_config = SimpleNamespace(
tensor_parallel_size=4,
data_parallel_size=1,
prefill_context_parallel_size=1,
enable_expert_parallel=False,
)
on_gfx950 = value if setting == "on_gfx950" else True
if setting != "on_gfx950":
setattr(parallel_config, setting, value)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: on_gfx950)
with patch.object(glm5_next_model.logger, "warning_once") as warning:
tuned = glm5_next_model._fused_shared_experts_tuned(
cast(ParallelConfig, parallel_config)
)
assert tuned is (reason is None)
if reason is None:
warning.assert_not_called()
else:
warning.assert_called_once()
assert warning.call_args.args[1] == reason
def test_glm5_next_fused_shared_expert_loads_into_the_shared_slot() -> None:
from vllm.models.glm5next.common import model as glm5_next_model
assert glm5_next_model._num_fused_shared_experts(1, True) == 1
assert glm5_next_model._num_fused_shared_experts(1, False) == 0
assert glm5_next_model._num_fused_shared_experts(None, True) == 0
with pytest.raises(NotImplementedError, match="only 1 shared expert"):
glm5_next_model._num_fused_shared_experts(2, True)
for shared, fused in [
(
"model.layers.3.mlp.shared_experts.gate_proj.weight_scale_inv",
"model.layers.3.mlp.experts.288.gate_proj.weight_scale_inv",
),
(
"model.layers.45.mtp_block.mlp.shared_experts.down_proj.weight",
"model.layers.45.mtp_block.mlp.experts.288.down_proj.weight",
),
]:
assert glm5_next_model._fused_shared_expert_name(shared, 288) == fused
routed = "model.layers.3.mlp.experts.7.down_proj.weight"
assert glm5_next_model._fused_shared_expert_name(routed, 288) == routed
@pytest.mark.parametrize(
("exclude", "expected"),
[([], True), ([r"re:.*\.shared_expert\..*"], False)],
@@ -955,6 +1069,73 @@ def test_quark_shared_expert_fse_compatibility(
)
@pytest.mark.parametrize("exclude_shared_expert", [False, True])
def test_online_targets_off_shared_expert_defer_to_checkpoint(
exclude_shared_expert: bool,
) -> None:
"""Unrelated online targets preserve the checkpoint's FSE decision."""
shared_prefix = "model.layers.0.mlp.shared_expert"
quant_config = QuarkConfig(
{
**_QUARK_FSE_CONFIG,
"exclude": (
[f"{shared_prefix}.down_proj"] if exclude_shared_expert else []
),
}
)
quant_config.online_quantization_config = OnlineQuantizationConfig(
QuantizationConfigArgs(targets={"*linear_attn.out_proj": "mxfp4"})
)
compatible, reason = is_shared_expert_quant_fse_compatible(
quant_config,
"model.layers.0.mlp.experts",
shared_prefix,
)
assert compatible is not exclude_shared_expert
assert reason == (
f"Quark excludes shared experts at {shared_prefix}"
if exclude_shared_expert
else None
)
def test_online_targeting_only_one_shared_projection_rejects_fse() -> None:
shared_prefix = "model.layers.0.mlp.shared_expert"
quant_config = QuarkConfig({**_QUARK_FSE_CONFIG, "exclude": []})
quant_config.online_quantization_config = OnlineQuantizationConfig(
QuantizationConfigArgs(targets={f"{shared_prefix}.down_proj": "mxfp4"})
)
assert is_shared_expert_quant_fse_compatible(
quant_config,
"model.layers.0.mlp.experts",
shared_prefix,
) == (
False,
"online quantization targets only part of the shared expert at "
f"{shared_prefix}",
)
def test_online_fp8_shared_expert_rejects_quark_mxfp4_fse() -> None:
shared_prefix = "model.layers.0.mlp.shared_expert"
quant_config = QuarkConfig({**_QUARK_FSE_CONFIG, "exclude": []})
quant_config.online_quantization_config = OnlineQuantizationConfig(
QuantizationConfigArgs(targets={f"{shared_prefix}*": "fp8_per_tensor"})
)
assert is_shared_expert_quant_fse_compatible(
quant_config,
"model.layers.0.mlp.experts",
shared_prefix,
) == (
False,
"online shared-expert quantization keys do not match the routed expert keys",
)
def test_quark_shared_expert_fse_exclude_is_scoped_to_the_layer() -> None:
"""Excluding one layer's shared experts must not disable FSE elsewhere.
@@ -1220,14 +1401,15 @@ def test_quark_packed_layer_config_must_match_global_config() -> None:
def test_non_quark_shared_expert_fse_is_incompatible() -> None:
compatible, reason = is_shared_expert_quant_fse_compatible(
_stub_quant_config(),
ExpertsInt8Config(),
"model.layers.0.mlp.experts",
"model.layers.0.mlp.shared_experts",
)
assert not compatible
assert reason == (
"shared-expert FSE quantization compatibility is not implemented for object"
"shared-expert FSE quantization compatibility is not implemented for "
"ExpertsInt8Config"
)
@@ -6,11 +6,13 @@ import types
from unittest.mock import patch
import pytest
import torch
from vllm import SamplingParams
from vllm.config.load import LoadConfig
from vllm.model_executor.model_loader import get_model_loader
from vllm.model_executor.model_loader import runai_streamer_loader as rsl
from vllm.utils.mem_constants import GiB_bytes
load_format = "runai_streamer"
test_model = "openai-community/gpt2"
@@ -122,3 +124,94 @@ def test_runai_invalid_extra_config_leaves_environ_untouched():
with pytest.raises(ValueError, match="memory_limit must be an integer >= -1"):
_runai_loader({"concurrency": 16, "memory_limit": -5})
assert "RUNAI_STREAMER_CONCURRENCY" not in os.environ
@pytest.mark.parametrize(
"model_config,expected_source",
[
# Object storage: model_weights holds the URI, model the pulled config dir.
(
types.SimpleNamespace(
model="/tmp/pulled-config-files",
model_weights="s3://bucket/weights",
revision="myrev",
),
"s3://bucket/weights",
),
# HF repo or local path: model_weights is empty, model is the source.
(
types.SimpleNamespace(
model="org/model", model_weights="", revision="myrev"
),
"org/model",
),
],
ids=["object_storage", "hf_repo"],
)
def test_runai_get_all_weights_resolves_source(model_config, expected_source):
fake_self = types.SimpleNamespace(
_get_weights_iterator=lambda path, revision: iter(
[(f"{path}@{revision}", None)]
)
)
weights = list(
rsl.RunaiModelStreamerLoader.get_all_weights(
fake_self, model_config, model=None
)
)
assert weights == [(f"{expected_source}@myrev", None)]
@pytest.mark.parametrize(
"base_model,mul_model,add_model",
[
(
"Qwen/Qwen3-0.6B",
"inference-optimization/Qwen3-0.6B-debug-multiply",
"inference-optimization/Qwen3-0.6B-debug-add",
),
],
)
def test_runai_deep_sleep_reload_weights(base_model, mul_model, add_model, vllm_runner):
free, total = torch.accelerator.get_memory_info()
used_bytes_baseline = total - free
def sleep_and_reload(llm, path):
# Level 2 discards the parameter memory, so after wake_up the weights
# can only come from the reload: a tensor the streamer fails to
# deliver changes the output instead of keeping a stale valid value.
llm.get_llm().sleep(level=2)
free, total = torch.accelerator.get_memory_info()
assert total - free - used_bytes_baseline < 3 * GiB_bytes
llm.get_llm().wake_up(tags=["weights"])
llm.collective_rpc("reload_weights", kwargs={"weights_path": path})
free, total = torch.accelerator.get_memory_info()
assert total - free - used_bytes_baseline < 4 * GiB_bytes
llm.get_llm().wake_up(tags=["kv_cache"])
with vllm_runner(
model_name=base_model,
load_format=load_format,
enable_sleep_mode=True,
enable_prefix_caching=False,
max_model_len=16,
max_num_seqs=1,
) as llm:
base_output = llm.generate_greedy(["3 4 ="], max_tokens=4)
sleep_and_reload(llm, mul_model)
mul_perp = llm.generate_prompt_perplexity(["3 4 = 12"], mask=["3 4 ="])[0]
add_perp = llm.generate_prompt_perplexity(["3 4 = 7"], mask=["3 4 ="])[0]
assert mul_perp < add_perp
sleep_and_reload(llm, add_model)
mul_perp = llm.generate_prompt_perplexity(["3 4 = 12"], mask=["3 4 ="])[0]
add_perp = llm.generate_prompt_perplexity(["3 4 = 7"], mask=["3 4 ="])[0]
assert add_perp < mul_perp
# Round trip to the original weights over discarded memory must
# reproduce the pre-sleep output.
sleep_and_reload(llm, base_model)
assert llm.generate_greedy(["3 4 ="], max_tokens=4) == base_output
@@ -3,7 +3,9 @@
from pathlib import Path
import torch
from runai_model_streamer.safetensors_streamer.streamer_mock import StreamerPatcher
from safetensors.torch import load_file, save_file
from vllm.engine.arg_utils import EngineArgs
from vllm.transformers_utils.repo_utils import hf_api
@@ -50,3 +52,72 @@ def test_runai_model_loader_download_files_s3_mocked_with_patch(
executor = RunaiDummyExecutor(vllm_config)
executor.driver_worker.load_model()
def test_runai_reload_weights_path_from_mocked_s3(tmp_path: Path, monkeypatch):
# An engine started from object storage keeps the bucket URI in
# model_config.model_weights. A reload with weights_path must stream the
# new checkpoint, not re-stream the one the engine started from.
patcher = StreamerPatcher(str(tmp_path))
monkeypatch.setattr(
"vllm.transformers_utils.runai_utils.runai_list_safetensors",
patcher.shim_list_safetensors,
)
monkeypatch.setattr(
"vllm.transformers_utils.runai_utils.runai_pull_files",
patcher.shim_pull_files,
)
monkeypatch.setattr(
"vllm.model_executor.model_loader.weight_utils.SafetensorsStreamer",
patcher.create_mock_streamer,
)
base_dir = tmp_path / "gpt2"
hf_api().snapshot_download(
repo_id=test_model,
local_dir=str(base_dir),
allow_patterns=["*.json", "*.txt", "model.safetensors"],
)
# Second checkpoint under another prefix: the final layer norm scaled, so a
# reload that silently re-streams the original bucket is distinguishable.
tensors = load_file(base_dir / "model.safetensors")
original = tensors["ln_f.weight"].clone()
modified = original * 2
tensors["ln_f.weight"] = modified
mod_dir = tmp_path / "gpt2-mod"
mod_dir.mkdir()
save_file(tensors, mod_dir / "model.safetensors", metadata={"format": "pt"})
vllm_config = EngineArgs(
model="s3://my-mock-bucket/gpt2/",
load_format=load_format,
tensor_parallel_size=1,
).create_engine_config()
assert vllm_config.model_config.model_weights == "s3://my-mock-bucket/gpt2/"
executor = RunaiDummyExecutor(vllm_config)
executor.driver_worker.load_model()
model = executor.driver_worker.model_runner.get_model()
(param,) = [p for n, p in model.named_parameters() if n.endswith("ln_f.weight")]
def current() -> torch.Tensor:
return param.detach().float().cpu()
torch.testing.assert_close(current(), original, rtol=1e-2, atol=1e-2)
# No weights_path: re-stream the checkpoint the engine started from, i.e.
# the bucket kept in model_weights. Zero the live tensor first so a reload
# that did nothing is distinguishable from one that streamed the bucket.
param.data.zero_()
executor.driver_worker.reload_weights()
torch.testing.assert_close(current(), original, rtol=1e-2, atol=1e-2)
# weights_path: the new prefix must win over the stale model_weights.
executor.driver_worker.reload_weights(weights_path="s3://my-mock-bucket/gpt2-mod/")
torch.testing.assert_close(current(), modified, rtol=1e-2, atol=1e-2)
# After the override, a bare reload means the new checkpoint, not the
# one the engine was started with.
param.data.zero_()
executor.driver_worker.reload_weights()
torch.testing.assert_close(current(), modified, rtol=1e-2, atol=1e-2)
@@ -579,7 +579,7 @@ async def test_serialize_and_serve_entrypoints(tmp_path):
try:
await asyncio.wait_for(fut, 180)
except asyncio.TimeoutError:
except TimeoutError:
pytest.fail("Server did not start successfully")
finally:
proc.terminate()
@@ -4,11 +4,17 @@
import json
import os
import tempfile
from types import SimpleNamespace
import pytest
import torch
from safetensors.torch import save_file
from vllm.config.load import LoadConfig
from vllm.model_executor.model_loader import DefaultModelLoader
from vllm.model_executor.model_loader.weight_utils import (
filter_duplicate_safetensors_files,
filter_safetensors_files_by_weight_name,
)
@@ -77,3 +83,57 @@ def test_filter_duplicate_safetensors_files_all_exist():
if __name__ == "__main__":
test_filter_duplicate_safetensors_files_missing_weight()
test_filter_duplicate_safetensors_files_all_exist()
def _skip_non_mtp(name: str) -> bool:
return not name.startswith("mtp.")
@pytest.fixture
def mtp_checkpoint(tmp_path):
"""Three shards: target only, target + MTP, MTP only."""
shards = {
"model-00001-of-00003.safetensors": ["model.layers.0.weight"],
"model-00002-of-00003.safetensors": ["model.layers.1.weight", "mtp.a.weight"],
"model-00003-of-00003.safetensors": ["mtp.b.weight"],
}
for filename, names in shards.items():
save_file({name: torch.zeros(1) for name in names}, tmp_path / filename)
return tmp_path
def test_filter_by_weight_name_drops_shards_without_wanted_weights(mtp_checkpoint):
files = sorted(str(f) for f in mtp_checkpoint.glob("*.safetensors"))
assert filter_safetensors_files_by_weight_name(files, _skip_non_mtp) == files[1:]
# A rule that rejects everything must not leave the loader without files.
assert filter_safetensors_files_by_weight_name(files, lambda _: True) == files
def test_loader_does_not_read_shards_the_model_skips(mtp_checkpoint, monkeypatch):
"""Whole-file loaders such as InstantTensor only see the kept shards."""
opened: list[list[str]] = []
def fake_iterator(hf_weights_files, *args, **kwargs):
opened.append(sorted(os.path.basename(f) for f in hf_weights_files))
return iter(())
monkeypatch.setattr(
"vllm.model_executor.model_loader.default_loader."
"instanttensor_weights_iterator",
fake_iterator,
)
class Drafter(torch.nn.Module):
def is_unused_checkpoint_weight(self, name: str) -> bool:
return _skip_non_mtp(name)
loader = DefaultModelLoader(LoadConfig(load_format="instanttensor"))
model_config = SimpleNamespace(model=str(mtp_checkpoint), revision=None)
list(loader.get_all_weights(model_config, Drafter()))
list(loader.get_all_weights(model_config, torch.nn.Module()))
assert opened == [
["model-00002-of-00003.safetensors", "model-00003-of-00003.safetensors"],
sorted(f.name for f in mtp_checkpoint.glob("*.safetensors")),
]
@@ -528,6 +528,7 @@ def test_padded_moe_reload_releases_each_layer(
layer.quant_method = method
layer.expert_map_manager = SimpleNamespace(map_global_to_local=lambda i: i)
layer._loaded_expert_biases = set()
layer._fused_shared_expert_quantizer = None
method.create_weights(
layer,
experts,
@@ -1051,6 +1052,37 @@ def test_hpc_rope_norm_kernel_sees_refit_norm_weights(monkeypatch, hpc_rope_norm
assert not hasattr(rnorm, "qnorm_weight")
def _device_tensor_ptrs(worker) -> dict[str, int]:
"""Addresses of every device tensor held by a module; graphs capture them."""
ptrs = {}
for name, module in worker.get_model().named_modules():
tensors = {**vars(module), **module._parameters, **module._buffers}
for attr, tensor in tensors.items():
if isinstance(tensor, torch.Tensor) and tensor.device.type != "cpu":
ptrs[f"{name}.{attr}"] = tensor.data_ptr()
return ptrs
def _moved_tensors(llm, ptrs: list[dict[str, int]]) -> list[list[str]]:
after = llm.collective_rpc(_device_tensor_ptrs)
return [
sorted(n for n, p in old.items() if new.get(n) != p)
for old, new in zip(ptrs, after)
]
@pytest.mark.parametrize(
"use_aiter",
[
False,
pytest.param(
True,
marks=pytest.mark.skipif(
not current_platform.is_rocm(), reason="AITER is ROCm-only"
),
),
],
)
@pytest.mark.parametrize(
"tp_size", [pytest.param(1), pytest.param(2, marks=[pytest.mark.slow_test])]
)
@@ -1094,13 +1126,19 @@ def test_hpc_rope_norm_kernel_sees_refit_norm_weights(monkeypatch, hpc_rope_norm
),
],
)
def test_reload_weights(base_model, mul_model, add_model, tp_size, vllm_runner):
def test_reload_weights(
base_model, mul_model, add_model, tp_size, use_aiter, vllm_runner, monkeypatch
):
if current_platform.device_count() < tp_size:
pytest.skip(reason="Not enough CUDA devices")
if "FP8" in base_model and _fp8_reload_unsupported():
pytest.skip(reason="Requires FP8 support")
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
if use_aiter:
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
with vllm_runner(
model_name=base_model,
tensor_parallel_size=tp_size,
@@ -1109,12 +1147,16 @@ def test_reload_weights(base_model, mul_model, add_model, tp_size, vllm_runner):
max_model_len=16,
max_num_seqs=1,
) as llm:
# Reloading must update tensors in place: captured graphs keep pointers.
ptrs = llm.collective_rpc(_device_tensor_ptrs)
llm.collective_rpc("reload_weights", kwargs={"weights_path": mul_model})
assert _moved_tensors(llm, ptrs) == [[]] * len(ptrs)
mul_perp = llm.generate_prompt_perplexity(["3 4 = 12"], mask=["3 4 ="])[0]
add_perp = llm.generate_prompt_perplexity(["3 4 = 7"], mask=["3 4 ="])[0]
assert mul_perp < add_perp
llm.collective_rpc("reload_weights", kwargs={"weights_path": add_model})
assert _moved_tensors(llm, ptrs) == [[]] * len(ptrs)
mul_perp = llm.generate_prompt_perplexity(["3 4 = 12"], mask=["3 4 ="])[0]
add_perp = llm.generate_prompt_perplexity(["3 4 = 7"], mask=["3 4 ="])[0]
assert add_perp < mul_perp
@@ -356,6 +356,52 @@ def test_weight_cache_key_distinguishes_dp_ranks():
assert key.mismatched_fields(replace(key, pp_rank=0)) == ["pp_rank"]
@pytest.mark.parametrize("hf_quant_config", [None, {"quant_algo": "NVFP4"}])
def test_weight_cache_key_distinguishes_nvfp4_activation_override(
tmp_path, hf_quant_config
):
"""Cache keys isolate activation modes and reject ambiguous legacy hashes."""
import json
from dataclasses import replace
from types import SimpleNamespace
import torch
from vllm.config.quantization import QuantizationConfigArgs
from vllm.model_executor.model_loader.weight_cache.protocol import WeightCacheKey
from vllm.utils.hashing import safe_hash
model_config = SimpleNamespace(
model=str(tmp_path),
dtype=torch.bfloat16,
quantization="modelopt_fp4",
quantization_config=None,
revision=None,
hf_config=SimpleNamespace(
architectures=["NemotronHForCausalLM"],
quantization_config=hf_quant_config,
),
)
static_key = WeightCacheKey.from_model_config(model_config, tp_size=1, tp_rank=0)
model_config.quantization_config = QuantizationConfigArgs(
moe={"activation": "nvfp4_per_token"}
)
per_token_key = WeightCacheKey.from_model_config(model_config, tp_size=1, tp_rank=0)
assert static_key.mismatched_fields(per_token_key) == ["quant_config_hash"]
legacy_hash = (
""
if hf_quant_config is None
else safe_hash(
json.dumps(hf_quant_config, sort_keys=True).encode(),
usedforsecurity=False,
).hexdigest()
)
assert static_key.mismatched_fields(
replace(static_key, quant_config_hash=legacy_hash)
) == ["quant_config_hash"]
def test_ipc_loader_copy_mode_reports_no_external_weight_memory():
"""Copy mode clones the weights into the engine, so nothing is external
(vllm_config is never touched)."""
@@ -39,13 +39,13 @@ def _make_moe(*, max_deferred_tokens: int = 128, enabled: bool = True):
return moe
def _make_runner(modules, *, max_tokens: int = 8192, linear_backend: str = "auto"):
def _make_runner(modules, *, max_tokens: int = 8192):
"""Create a runner carrying only the state used by FlashInfer warmup."""
return SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=max_tokens),
vllm_config=SimpleNamespace(
kernel_config=SimpleNamespace(linear_backend=linear_backend),
attention_config=SimpleNamespace(hisparse_config=None),
parallel_config=SimpleNamespace(data_parallel_rank=0),
),
get_model=Mock(
return_value=SimpleNamespace(modules=Mock(return_value=modules))
@@ -62,21 +62,19 @@ def test_flashinfer_autotune_token_counts_include_deferred_moe_limits():
_make_moe(max_deferred_tokens=128),
_make_moe(max_deferred_tokens=64, enabled=False),
_make_moe(max_deferred_tokens=-1),
],
linear_backend="flashinfer_cutedsl",
]
)
with patch("vllm.model_executor.layers.fused_moe.MoERunner", _FakeMoERunner):
token_counts = _flashinfer_autotune_token_counts(runner)
assert token_counts == (8192, 32, 128)
assert token_counts == (8192, 128)
def test_flashinfer_autotune_token_counts_are_bounded_and_deduplicated():
runner = _make_runner(
[_make_moe(max_deferred_tokens=4096)],
max_tokens=32,
linear_backend="flashinfer_cutedsl",
)
with patch("vllm.model_executor.layers.fused_moe.MoERunner", _FakeMoERunner):
@@ -114,8 +112,8 @@ def test_flashinfer_autotune_uses_token_buckets_for_each_dummy_run(skip_attn):
assert get_buckets.call_args_list == [call(8192), call(128)]
assert autotune.call_args_list == [
call(tuning_buckets=max_buckets),
call(tuning_buckets=deferred_buckets),
call(tuning_buckets=max_buckets, round_up=True),
call(tuning_buckets=deferred_buckets, round_up=True),
]
assert runner._dummy_run.call_args_list == [
call(
@@ -135,6 +133,33 @@ def test_flashinfer_autotune_uses_token_buckets_for_each_dummy_run(skip_attn):
]
def test_flashinfer_autotune_buckets_cover_drafter_tokens():
"""The drafter's M can exceed the pass size; buckets must include it."""
runner = _make_runner([])
runner.max_num_reqs = 256
runner.max_num_tokens = 8192
runner.speculator = SimpleNamespace(num_query_per_req=6)
with (
patch(
"vllm.model_executor.warmup.kernel_warmup."
"_flashinfer_autotune_token_counts",
return_value=(8192, 128),
),
patch(
"vllm.utils.flashinfer.flashinfer_get_hybrid_num_tokens_buckets"
) as get_buckets,
patch("vllm.utils.flashinfer.autotune"),
):
_run_flashinfer_autotune_dummy_runs(runner)
assert get_buckets.call_args_list == [call(8192), call(128 * 6)]
assert [c.kwargs["num_tokens"] for c in runner._dummy_run.call_args_list] == [
8192,
128,
]
class _AutotuneGroup:
def __init__(self, run, ranks):
self.run = run
@@ -146,12 +171,20 @@ class _AutotuneGroup:
def record(self, operation):
self.run.collectives[self.ranks][self.run.rank].append(operation)
def broadcast_object(self, obj, src=0):
assert src == 0
self.record(("broadcast", src))
if self.rank_in_group == src:
self.run.broadcasts[self.ranks] = obj
return self.run.broadcasts[self.ranks]
def all_gather_object(self, out, obj):
# Ranks run sequentially: snapshot every rank's file before any saves.
self.record(("all_gather",))
if isinstance(obj, bool):
out[:] = [rank not in self.run.rejected_load_ranks for rank in self.ranks]
assert out[self.rank_in_group] == obj
return
out[:] = self.run.gathered.setdefault(
self.ranks,
[
warmup._autotune_cache_fingerprint(self.run.cache_path(rank))
for rank in self.ranks
],
)
def barrier(self):
self.record(("barrier",))
@@ -162,10 +195,18 @@ class _AutotuneTuner:
self.run = run
self.cache = {}
self.loaded = None
self._dirty = False
def load_configs(self, path):
if self.run.rank in self.run.rejected_load_ranks:
return False
self.loaded = json.loads(Path(path).read_text())
self.cache.update(self.loaded)
return True
def clear_cache(self):
self.cache.clear()
self._dirty = False
def save_configs(self, path):
self.run.saves.append((self.run.rank, Path(path), dict(self.cache)))
@@ -182,21 +223,27 @@ class _AutotuneTuner:
if group is not None:
group.record(("all_reduce", operation, tactic))
self.cache[operation] = self.run.rank // self.run.tp
self._dirty = True
class _AutotuneRun:
def __init__(self, pp, tp):
def __init__(self, pp, tp, cache_dir):
self.pp, self.tp = pp, tp
self.cache_dir = cache_dir
self.rank = 0
self.tuning_group = None
self.collectives: dict[tuple[int, ...], dict[int, list[tuple[Any, ...]]]] = (
defaultdict(lambda: defaultdict(list))
)
self.broadcasts = {}
self.gathered = {}
self.rejected_load_ranks: set[int] = set()
self.tuners = {}
self.saves = []
self.profile_groups = defaultdict(list)
def cache_path(self, rank):
return self.cache_dir / f"autotune_configs_dp0_rank{rank}.json"
def world(self):
return _AutotuneGroup(self, range(self.pp * self.tp))
@@ -235,7 +282,7 @@ def autotune_run(monkeypatch, tmp_path):
from vllm.distributed import parallel_state
def make_run(*, pp=2, tp=4):
run = _AutotuneRun(pp, tp)
run = _AutotuneRun(pp, tp, tmp_path)
autotuner = ModuleType("flashinfer.autotuner")
monkeypatch.setattr(
autotuner,
@@ -251,6 +298,10 @@ def autotune_run(monkeypatch, tmp_path):
monkeypatch.setattr(parallel_state, "get_tp_group", run.tensor_group)
monkeypatch.setattr(parallel_state, "get_pp_group", run.pipeline_group)
monkeypatch.setattr(fi_utils, "autotune", lambda **kwargs: nullcontext())
monkeypatch.setattr(
"torch.distributed.all_gather_object",
lambda out, obj, group: group.all_gather_object(out, obj),
)
monkeypatch.setattr(
warmup,
"resolve_flashinfer_autotune_file",
@@ -286,19 +337,18 @@ def test_pp_stage_cache_roundtrip_isolated_and_asymmetric_hits_safe(
legacy = tmp_path / "autotune_configs.json"
legacy.write_text('{"legacy_world_cache": 99}')
cold = autotune_run().execute()
assert [rank for rank, _, _ in cold.saves] == [0, 4]
paths = [path for _, path, _ in cold.saves]
assert len(set(paths)) == 2 and legacy not in paths
assert json.loads(legacy.read_text()) == {"legacy_world_cache": 99}
assert cold.saves[0][2] == {"shared_gemm": 0, "pp0_extra_gemm": 0}
assert cold.saves[1][2] == {"shared_gemm": 1}
cold.assert_collectives_match()
assert [rank for rank, _, _ in cold.saves] == list(range(8))
assert [path for _, path, _ in cold.saves] == [cold.cache_path(r) for r in range(8)]
assert json.loads(legacy.read_text()) == {"legacy_world_cache": 99}
stage_caches = [{"shared_gemm": 0, "pp0_extra_gemm": 0}, {"shared_gemm": 1}]
assert all(cache == stage_caches[r // 4] for r, _, cache in cold.saves)
warm = autotune_run().execute()
warm.assert_collectives_match()
assert not warm.profile_groups
assert not warm.profile_groups and not warm.saves
for rank, tuner in warm.tuners.items():
assert tuner.loaded == cold.saves[rank // 4][2]
paths[1].unlink()
assert tuner.loaded == stage_caches[rank // 4]
cold.cache_path(5).unlink()
mixed = autotune_run().execute()
mixed.assert_collectives_match()
assert set(mixed.profile_groups) == {4, 5, 6, 7}
@@ -306,10 +356,33 @@ def test_pp_stage_cache_roundtrip_isolated_and_asymmetric_hits_safe(
assert all(mixed.tuners[rank].loaded is None for rank in range(4, 8))
def test_pp1_retains_world_synchronization_and_existing_cache_name(autotune_run):
def test_pp1_tunes_world_group_and_saves_per_rank(autotune_run):
run = autotune_run(pp=1, tp=4).execute()
run.assert_collectives_match()
assert [(rank, path.name) for rank, path, _ in run.saves] == [
(0, "autotune_configs.json")
assert [(rank, path) for rank, path, _ in run.saves] == [
(rank, run.cache_path(rank)) for rank in range(4)
]
assert all(groups == [(0, 1, 2, 3)] for groups in run.profile_groups.values())
def test_mismatched_rank_caches_are_ignored_by_every_rank(autotune_run):
"""One rank loading while others tune deadlocks the per-tactic reduce."""
cold = autotune_run(pp=1, tp=4).execute()
stale = cold.cache_path(2)
stale.write_text(json.dumps({**json.loads(stale.read_text()), "old_gemm": 0}))
rerun = autotune_run(pp=1, tp=4).execute()
rerun.assert_collectives_match()
assert all(tuner.loaded is None for tuner in rerun.tuners.values())
assert set(rerun.profile_groups) == {0, 1, 2, 3}
@pytest.mark.parametrize("pp, tp", [(1, 1), (1, 4), (2, 4)])
def test_rejected_cache_retunes_every_rank_in_its_tuning_group(autotune_run, pp, tp):
"""Matching files can still be incompatible with one rank's runtime."""
autotune_run(pp=pp, tp=tp).execute()
rerun = autotune_run(pp=pp, tp=tp)
rerun.rejected_load_ranks = {tp - 1}
rerun.execute()
rerun.assert_collectives_match()
assert set(rerun.profile_groups) == set(range(tp))
assert {rank for rank, _, _ in rerun.saves} == set(range(tp))
@@ -0,0 +1,28 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.models.phi4mm_utils import MultiHeadedAttention
def test_eager_attention_does_not_retain_probability_matrix():
module = MultiHeadedAttention(
n_head=2,
n_feat=16,
dropout_rate=0.0,
use_pt_scaled_dot_product_attention=False,
)
module.eval()
batch, time, feat = 2, 8, 16
x = torch.randn(batch, time, feat)
with torch.no_grad():
out = module(x, x, x, None, None, None)
assert out.shape == (batch, time, feat)
assert getattr(module, "attn", None) is None
def test_multiheaded_attention_defaults_to_scaled_dot_product():
module = MultiHeadedAttention(n_head=2, n_feat=16, dropout_rate=0.0)
assert module.use_pt_scaled_dot_product_attention is True
+32 -14
View File
@@ -145,27 +145,45 @@ def test_mrope_positions_video_at_prompt_end():
assert positions.shape == (3, len(input_tokens))
def test_mrope_positions_audio_in_video_at_prompt_end():
@pytest.mark.parametrize("num_videos", [1, 2])
@pytest.mark.parametrize("trailing_tokens", [0, 3])
def test_mrope_positions_audio_in_video_boundaries(num_videos, trailing_tokens):
"""Interleaved BOS/EOS positions and decode delta match Qwen3-Omni HF."""
model = make_model()
# video grid (1, 2, 2) -> 4 video tokens; audio_feature_length=16 -> 2
# audio tokens, interleaved after the video tokens.
# [T, T, <|vision_start|>, <|audio_start|>, V x 4, A x 2,
# <|audio_end|>, <|vision_end|>]
input_tokens = (
[TEXT_TOKEN_ID] * 2
+ [VISION_START_TOKEN_ID, AUDIO_START_TOKEN_ID]
video_tokens = (
[VISION_START_TOKEN_ID, AUDIO_START_TOKEN_ID]
+ [VIDEO_TOKEN_ID] * 4
+ [AUDIO_TOKEN_ID] * 2
+ [AUDIO_END_TOKEN_ID, VISION_END_TOKEN_ID]
)
mm_features = [
make_video_feature(offset=3, grid_thw=(1, 2, 2), use_audio_in_video=True),
make_audio_feature(offset=3, audio_feature_length=16),
]
input_tokens = [TEXT_TOKEN_ID] * 2
mm_features = []
expected = [torch.tensor([[0, 1]]).expand(3, -1)]
video_positions = torch.tensor(
[
[2, 3, 4, 4, 4, 4, 4, 5, 6, 7],
[2, 3, 4, 4, 5, 5, 4, 5, 6, 7],
[2, 3, 4, 5, 4, 5, 4, 5, 6, 7],
]
)
for index in range(num_videos):
offset = len(input_tokens) + 1
mm_features.extend(
[
make_video_feature(offset, (1, 2, 2), use_audio_in_video=True),
make_audio_feature(offset, audio_feature_length=16),
]
)
input_tokens.extend(video_tokens)
expected.append(video_positions + index * 6)
positions, _ = model.get_mrope_input_positions(input_tokens, mm_features)
input_tokens.extend([TEXT_TOKEN_ID] * trailing_tokens)
next_position = 2 + num_videos * 6
expected.append((torch.arange(trailing_tokens) + next_position).expand(3, -1))
positions, delta = model.get_mrope_input_positions(input_tokens, mm_features)
assert positions.shape == (3, len(input_tokens))
torch.testing.assert_close(positions, torch.cat(expected, dim=1))
assert delta == -4 * num_videos
def test_mrope_positions_image_then_text():
@@ -582,10 +582,7 @@ def test_v2_model_runner_accepts_routed_experts(monkeypatch):
use_ubatching=False,
enable_elastic_ep=False,
),
compilation_config=SimpleNamespace(
mode=CompilationMode.NONE,
pass_config=SimpleNamespace(enable_sp=False),
),
compilation_config=SimpleNamespace(mode=CompilationMode.NONE),
cache_config=SimpleNamespace(
kv_sharing_fast_prefill=False,
mamba_cache_mode="none",