Merge branch 'main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -12,13 +12,16 @@ from unittest.mock import patch
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen3_5MoeTextConfig
|
||||
|
||||
import vllm.config as vllm_config_module
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.config.quantization import QuantizationConfigArgs
|
||||
from vllm.model_executor.layers.fused_moe import utils as fused_moe_utils
|
||||
from vllm.model_executor.layers.fused_moe.layer import determine_expert_counts
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.experts_int8 import ExpertsInt8Config
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.model_executor.layers.quantization.online.base import OnlineQuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
|
||||
from vllm.model_executor.layers.quantization.utils.config_utils import (
|
||||
get_quark_ocp_mx_group_size,
|
||||
@@ -32,7 +35,6 @@ from vllm.models.deepseek_v4 import quant_config as deepseek_v4_quant_config
|
||||
from vllm.models.minimax_m3.amd import model as minimax_m3_model
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.configs.minimax_m3 import MiniMaxM3TextConfig
|
||||
from vllm.transformers_utils.configs.qwen3_5_moe import Qwen3_5MoeTextConfig
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
current_platform.is_xpu(),
|
||||
@@ -141,8 +143,10 @@ def get_deepseek_v4_quark_config(exclude: list[str]) -> dict[str, Any]:
|
||||
return quantization_config
|
||||
|
||||
|
||||
def _stub_quant_config() -> QuantizationConfig:
|
||||
return cast(QuantizationConfig, object())
|
||||
class _StubGlm5NextAttention(nn.Module):
|
||||
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
||||
super().__init__()
|
||||
self.o_proj = SimpleNamespace(reduce_results=True)
|
||||
|
||||
|
||||
def get_fse_test_model_config(
|
||||
@@ -279,6 +283,33 @@ def get_fse_test_model_config(
|
||||
quantization_config=quantization_config,
|
||||
)
|
||||
return config, Glm4MoeModel
|
||||
if model_type == "glm5_next":
|
||||
from transformers import Glm5NextTextConfig
|
||||
|
||||
from vllm.models.glm5next.common.model import Glm5NextModel
|
||||
|
||||
config = Glm5NextTextConfig(
|
||||
vocab_size=256,
|
||||
hidden_size=128,
|
||||
intermediate_size=32,
|
||||
moe_intermediate_size=32,
|
||||
num_hidden_layers=1,
|
||||
num_attention_heads=1,
|
||||
num_key_value_heads=1,
|
||||
n_routed_experts=2,
|
||||
n_shared_experts=1,
|
||||
num_experts_per_tok=1,
|
||||
layer_types=["linear_attention"],
|
||||
mlp_layer_types=["sparse"],
|
||||
index_topk=1,
|
||||
index_kpool=1,
|
||||
pad_token_id=None,
|
||||
scoring_func="sigmoid",
|
||||
topk_method="noaux_tc",
|
||||
mhc=False,
|
||||
quantization_config=quantization_config,
|
||||
)
|
||||
return config, Glm5NextModel
|
||||
raise ValueError(f"Unsupported FSE test model: {model_type}")
|
||||
|
||||
|
||||
@@ -307,7 +338,7 @@ def test_resolve_layer_fused_shared_expert_skips_compatibility_when_disabled(
|
||||
)
|
||||
|
||||
assert not fused_moe_utils.resolve_layer_fused_shared_expert(
|
||||
_stub_quant_config(), "model.layers.0.mlp"
|
||||
ExpertsInt8Config(), "model.layers.0.mlp"
|
||||
)
|
||||
|
||||
|
||||
@@ -322,7 +353,7 @@ def test_resolve_layer_fused_shared_expert_normalizes_unavailable_aiter(
|
||||
|
||||
assert (
|
||||
fused_moe_utils.resolve_layer_fused_shared_expert(
|
||||
_stub_quant_config(), "model.layers.0.mlp"
|
||||
ExpertsInt8Config(), "model.layers.0.mlp"
|
||||
)
|
||||
is False
|
||||
)
|
||||
@@ -331,7 +362,7 @@ def test_resolve_layer_fused_shared_expert_normalizes_unavailable_aiter(
|
||||
def test_resolve_layer_fused_shared_expert_passes_module_prefixes(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
quant_config = _stub_quant_config()
|
||||
quant_config = ExpertsInt8Config()
|
||||
monkeypatch.setattr(
|
||||
fused_moe_utils.rocm_aiter_ops,
|
||||
"is_fusion_moe_shared_experts_enabled",
|
||||
@@ -380,7 +411,7 @@ def test_resolve_layer_fused_shared_expert_rejects_incompatible_quantization(
|
||||
)
|
||||
|
||||
assert not fused_moe_utils.resolve_layer_fused_shared_expert(
|
||||
_stub_quant_config(), "model.layers.0.mlp"
|
||||
ExpertsInt8Config(), "model.layers.0.mlp"
|
||||
)
|
||||
assert "shared experts are excluded" in caplog.text
|
||||
|
||||
@@ -390,6 +421,7 @@ def test_deepseek_v4_shared_expert_fse_uses_mtp_quantization_config_prefix(
|
||||
) -> None:
|
||||
class DeepseekV4Config:
|
||||
expert_dtype = "fp4"
|
||||
online_quantization_config = None
|
||||
|
||||
def _is_quark_mxfp4_ocp(self, hf_config: object) -> bool:
|
||||
return True
|
||||
@@ -796,7 +828,7 @@ def test_is_model_fused_shared_expert_compatible() -> None:
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_type",
|
||||
["minimax_m3", "deepseek_v4", "qwen3_5", "glm4_moe", "deepseek_v2"],
|
||||
["minimax_m3", "deepseek_v4", "qwen3_5", "glm4_moe", "deepseek_v2", "glm5_next"],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("use_fse", "exclude"),
|
||||
@@ -850,6 +882,16 @@ def test_models_fse_init(
|
||||
)
|
||||
else:
|
||||
vllm_config.quant_config = QuarkConfig(quantization_config)
|
||||
if model_type == "glm5_next":
|
||||
from vllm.models.glm5next.common import model as glm5_next_model
|
||||
|
||||
for attention in ("Glm5NextLinearAttention", "Glm5NextMLAAttention"):
|
||||
monkeypatch.setattr(glm5_next_model, attention, _StubGlm5NextAttention)
|
||||
# The gate allows fusion only at TP4/TP8 on gfx950, and this test
|
||||
# runs at TP1; the gate has its own test.
|
||||
monkeypatch.setattr(
|
||||
glm5_next_model, "_fused_shared_experts_tuned", lambda _: True
|
||||
)
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
@@ -902,6 +944,10 @@ def test_models_fse_init(
|
||||
)
|
||||
mtp = DeepSeekV4MTP(vllm_config=vllm_config)
|
||||
assert model.is_fused_shared_expert_enabled is (fse_enabled and not exclude)
|
||||
if model_type == "glm5_next":
|
||||
assert (model.layers[0].mlp.shared_experts is None) is (
|
||||
fse_enabled and not exclude
|
||||
)
|
||||
|
||||
# The dummy quant config here uses mixed mxfp4/fp8 for experts/shared_expert
|
||||
# so should just raise a warning.
|
||||
@@ -926,6 +972,74 @@ def test_models_fse_init(
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("setting", "value", "reason"),
|
||||
[
|
||||
("tensor_parallel_size", 4, None),
|
||||
("tensor_parallel_size", 8, None),
|
||||
("tensor_parallel_size", 2, "tensor_parallel_size is 2"),
|
||||
("data_parallel_size", 2, "data_parallel_size is 2"),
|
||||
("prefill_context_parallel_size", 2, "prefill_context_parallel_size is 2"),
|
||||
("enable_expert_parallel", True, "expert parallelism is enabled"),
|
||||
("on_gfx950", False, "the GPU is not gfx950"),
|
||||
],
|
||||
)
|
||||
def test_glm5_next_fuses_shared_experts_only_in_tuned_setups(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
setting: str,
|
||||
value: object,
|
||||
reason: str | None,
|
||||
) -> None:
|
||||
from vllm.models.glm5next.common import model as glm5_next_model
|
||||
|
||||
parallel_config = SimpleNamespace(
|
||||
tensor_parallel_size=4,
|
||||
data_parallel_size=1,
|
||||
prefill_context_parallel_size=1,
|
||||
enable_expert_parallel=False,
|
||||
)
|
||||
on_gfx950 = value if setting == "on_gfx950" else True
|
||||
if setting != "on_gfx950":
|
||||
setattr(parallel_config, setting, value)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: on_gfx950)
|
||||
|
||||
with patch.object(glm5_next_model.logger, "warning_once") as warning:
|
||||
tuned = glm5_next_model._fused_shared_experts_tuned(
|
||||
cast(ParallelConfig, parallel_config)
|
||||
)
|
||||
|
||||
assert tuned is (reason is None)
|
||||
if reason is None:
|
||||
warning.assert_not_called()
|
||||
else:
|
||||
warning.assert_called_once()
|
||||
assert warning.call_args.args[1] == reason
|
||||
|
||||
|
||||
def test_glm5_next_fused_shared_expert_loads_into_the_shared_slot() -> None:
|
||||
from vllm.models.glm5next.common import model as glm5_next_model
|
||||
|
||||
assert glm5_next_model._num_fused_shared_experts(1, True) == 1
|
||||
assert glm5_next_model._num_fused_shared_experts(1, False) == 0
|
||||
assert glm5_next_model._num_fused_shared_experts(None, True) == 0
|
||||
with pytest.raises(NotImplementedError, match="only 1 shared expert"):
|
||||
glm5_next_model._num_fused_shared_experts(2, True)
|
||||
|
||||
for shared, fused in [
|
||||
(
|
||||
"model.layers.3.mlp.shared_experts.gate_proj.weight_scale_inv",
|
||||
"model.layers.3.mlp.experts.288.gate_proj.weight_scale_inv",
|
||||
),
|
||||
(
|
||||
"model.layers.45.mtp_block.mlp.shared_experts.down_proj.weight",
|
||||
"model.layers.45.mtp_block.mlp.experts.288.down_proj.weight",
|
||||
),
|
||||
]:
|
||||
assert glm5_next_model._fused_shared_expert_name(shared, 288) == fused
|
||||
routed = "model.layers.3.mlp.experts.7.down_proj.weight"
|
||||
assert glm5_next_model._fused_shared_expert_name(routed, 288) == routed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("exclude", "expected"),
|
||||
[([], True), ([r"re:.*\.shared_expert\..*"], False)],
|
||||
@@ -955,6 +1069,73 @@ def test_quark_shared_expert_fse_compatibility(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("exclude_shared_expert", [False, True])
|
||||
def test_online_targets_off_shared_expert_defer_to_checkpoint(
|
||||
exclude_shared_expert: bool,
|
||||
) -> None:
|
||||
"""Unrelated online targets preserve the checkpoint's FSE decision."""
|
||||
shared_prefix = "model.layers.0.mlp.shared_expert"
|
||||
quant_config = QuarkConfig(
|
||||
{
|
||||
**_QUARK_FSE_CONFIG,
|
||||
"exclude": (
|
||||
[f"{shared_prefix}.down_proj"] if exclude_shared_expert else []
|
||||
),
|
||||
}
|
||||
)
|
||||
quant_config.online_quantization_config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(targets={"*linear_attn.out_proj": "mxfp4"})
|
||||
)
|
||||
|
||||
compatible, reason = is_shared_expert_quant_fse_compatible(
|
||||
quant_config,
|
||||
"model.layers.0.mlp.experts",
|
||||
shared_prefix,
|
||||
)
|
||||
|
||||
assert compatible is not exclude_shared_expert
|
||||
assert reason == (
|
||||
f"Quark excludes shared experts at {shared_prefix}"
|
||||
if exclude_shared_expert
|
||||
else None
|
||||
)
|
||||
|
||||
|
||||
def test_online_targeting_only_one_shared_projection_rejects_fse() -> None:
|
||||
shared_prefix = "model.layers.0.mlp.shared_expert"
|
||||
quant_config = QuarkConfig({**_QUARK_FSE_CONFIG, "exclude": []})
|
||||
quant_config.online_quantization_config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(targets={f"{shared_prefix}.down_proj": "mxfp4"})
|
||||
)
|
||||
|
||||
assert is_shared_expert_quant_fse_compatible(
|
||||
quant_config,
|
||||
"model.layers.0.mlp.experts",
|
||||
shared_prefix,
|
||||
) == (
|
||||
False,
|
||||
"online quantization targets only part of the shared expert at "
|
||||
f"{shared_prefix}",
|
||||
)
|
||||
|
||||
|
||||
def test_online_fp8_shared_expert_rejects_quark_mxfp4_fse() -> None:
|
||||
shared_prefix = "model.layers.0.mlp.shared_expert"
|
||||
quant_config = QuarkConfig({**_QUARK_FSE_CONFIG, "exclude": []})
|
||||
quant_config.online_quantization_config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(targets={f"{shared_prefix}*": "fp8_per_tensor"})
|
||||
)
|
||||
|
||||
assert is_shared_expert_quant_fse_compatible(
|
||||
quant_config,
|
||||
"model.layers.0.mlp.experts",
|
||||
shared_prefix,
|
||||
) == (
|
||||
False,
|
||||
"online shared-expert quantization keys do not match the routed expert keys",
|
||||
)
|
||||
|
||||
|
||||
def test_quark_shared_expert_fse_exclude_is_scoped_to_the_layer() -> None:
|
||||
"""Excluding one layer's shared experts must not disable FSE elsewhere.
|
||||
|
||||
@@ -1220,14 +1401,15 @@ def test_quark_packed_layer_config_must_match_global_config() -> None:
|
||||
|
||||
def test_non_quark_shared_expert_fse_is_incompatible() -> None:
|
||||
compatible, reason = is_shared_expert_quant_fse_compatible(
|
||||
_stub_quant_config(),
|
||||
ExpertsInt8Config(),
|
||||
"model.layers.0.mlp.experts",
|
||||
"model.layers.0.mlp.shared_experts",
|
||||
)
|
||||
|
||||
assert not compatible
|
||||
assert reason == (
|
||||
"shared-expert FSE quantization compatibility is not implemented for object"
|
||||
"shared-expert FSE quantization compatibility is not implemented for "
|
||||
"ExpertsInt8Config"
|
||||
)
|
||||
|
||||
|
||||
|
||||
+93
@@ -6,11 +6,13 @@ import types
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.config.load import LoadConfig
|
||||
from vllm.model_executor.model_loader import get_model_loader
|
||||
from vllm.model_executor.model_loader import runai_streamer_loader as rsl
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
|
||||
load_format = "runai_streamer"
|
||||
test_model = "openai-community/gpt2"
|
||||
@@ -122,3 +124,94 @@ def test_runai_invalid_extra_config_leaves_environ_untouched():
|
||||
with pytest.raises(ValueError, match="memory_limit must be an integer >= -1"):
|
||||
_runai_loader({"concurrency": 16, "memory_limit": -5})
|
||||
assert "RUNAI_STREAMER_CONCURRENCY" not in os.environ
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_config,expected_source",
|
||||
[
|
||||
# Object storage: model_weights holds the URI, model the pulled config dir.
|
||||
(
|
||||
types.SimpleNamespace(
|
||||
model="/tmp/pulled-config-files",
|
||||
model_weights="s3://bucket/weights",
|
||||
revision="myrev",
|
||||
),
|
||||
"s3://bucket/weights",
|
||||
),
|
||||
# HF repo or local path: model_weights is empty, model is the source.
|
||||
(
|
||||
types.SimpleNamespace(
|
||||
model="org/model", model_weights="", revision="myrev"
|
||||
),
|
||||
"org/model",
|
||||
),
|
||||
],
|
||||
ids=["object_storage", "hf_repo"],
|
||||
)
|
||||
def test_runai_get_all_weights_resolves_source(model_config, expected_source):
|
||||
fake_self = types.SimpleNamespace(
|
||||
_get_weights_iterator=lambda path, revision: iter(
|
||||
[(f"{path}@{revision}", None)]
|
||||
)
|
||||
)
|
||||
|
||||
weights = list(
|
||||
rsl.RunaiModelStreamerLoader.get_all_weights(
|
||||
fake_self, model_config, model=None
|
||||
)
|
||||
)
|
||||
|
||||
assert weights == [(f"{expected_source}@myrev", None)]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"base_model,mul_model,add_model",
|
||||
[
|
||||
(
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"inference-optimization/Qwen3-0.6B-debug-multiply",
|
||||
"inference-optimization/Qwen3-0.6B-debug-add",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_runai_deep_sleep_reload_weights(base_model, mul_model, add_model, vllm_runner):
|
||||
free, total = torch.accelerator.get_memory_info()
|
||||
used_bytes_baseline = total - free
|
||||
|
||||
def sleep_and_reload(llm, path):
|
||||
# Level 2 discards the parameter memory, so after wake_up the weights
|
||||
# can only come from the reload: a tensor the streamer fails to
|
||||
# deliver changes the output instead of keeping a stale valid value.
|
||||
llm.get_llm().sleep(level=2)
|
||||
free, total = torch.accelerator.get_memory_info()
|
||||
assert total - free - used_bytes_baseline < 3 * GiB_bytes
|
||||
llm.get_llm().wake_up(tags=["weights"])
|
||||
llm.collective_rpc("reload_weights", kwargs={"weights_path": path})
|
||||
free, total = torch.accelerator.get_memory_info()
|
||||
assert total - free - used_bytes_baseline < 4 * GiB_bytes
|
||||
llm.get_llm().wake_up(tags=["kv_cache"])
|
||||
|
||||
with vllm_runner(
|
||||
model_name=base_model,
|
||||
load_format=load_format,
|
||||
enable_sleep_mode=True,
|
||||
enable_prefix_caching=False,
|
||||
max_model_len=16,
|
||||
max_num_seqs=1,
|
||||
) as llm:
|
||||
base_output = llm.generate_greedy(["3 4 ="], max_tokens=4)
|
||||
|
||||
sleep_and_reload(llm, mul_model)
|
||||
mul_perp = llm.generate_prompt_perplexity(["3 4 = 12"], mask=["3 4 ="])[0]
|
||||
add_perp = llm.generate_prompt_perplexity(["3 4 = 7"], mask=["3 4 ="])[0]
|
||||
assert mul_perp < add_perp
|
||||
|
||||
sleep_and_reload(llm, add_model)
|
||||
mul_perp = llm.generate_prompt_perplexity(["3 4 = 12"], mask=["3 4 ="])[0]
|
||||
add_perp = llm.generate_prompt_perplexity(["3 4 = 7"], mask=["3 4 ="])[0]
|
||||
assert add_perp < mul_perp
|
||||
|
||||
# Round trip to the original weights over discarded memory must
|
||||
# reproduce the pre-sleep output.
|
||||
sleep_and_reload(llm, base_model)
|
||||
assert llm.generate_greedy(["3 4 ="], max_tokens=4) == base_output
|
||||
|
||||
+71
@@ -3,7 +3,9 @@
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
from runai_model_streamer.safetensors_streamer.streamer_mock import StreamerPatcher
|
||||
from safetensors.torch import load_file, save_file
|
||||
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
@@ -50,3 +52,72 @@ def test_runai_model_loader_download_files_s3_mocked_with_patch(
|
||||
|
||||
executor = RunaiDummyExecutor(vllm_config)
|
||||
executor.driver_worker.load_model()
|
||||
|
||||
|
||||
def test_runai_reload_weights_path_from_mocked_s3(tmp_path: Path, monkeypatch):
|
||||
# An engine started from object storage keeps the bucket URI in
|
||||
# model_config.model_weights. A reload with weights_path must stream the
|
||||
# new checkpoint, not re-stream the one the engine started from.
|
||||
patcher = StreamerPatcher(str(tmp_path))
|
||||
monkeypatch.setattr(
|
||||
"vllm.transformers_utils.runai_utils.runai_list_safetensors",
|
||||
patcher.shim_list_safetensors,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.transformers_utils.runai_utils.runai_pull_files",
|
||||
patcher.shim_pull_files,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.model_loader.weight_utils.SafetensorsStreamer",
|
||||
patcher.create_mock_streamer,
|
||||
)
|
||||
|
||||
base_dir = tmp_path / "gpt2"
|
||||
hf_api().snapshot_download(
|
||||
repo_id=test_model,
|
||||
local_dir=str(base_dir),
|
||||
allow_patterns=["*.json", "*.txt", "model.safetensors"],
|
||||
)
|
||||
# Second checkpoint under another prefix: the final layer norm scaled, so a
|
||||
# reload that silently re-streams the original bucket is distinguishable.
|
||||
tensors = load_file(base_dir / "model.safetensors")
|
||||
original = tensors["ln_f.weight"].clone()
|
||||
modified = original * 2
|
||||
tensors["ln_f.weight"] = modified
|
||||
mod_dir = tmp_path / "gpt2-mod"
|
||||
mod_dir.mkdir()
|
||||
save_file(tensors, mod_dir / "model.safetensors", metadata={"format": "pt"})
|
||||
|
||||
vllm_config = EngineArgs(
|
||||
model="s3://my-mock-bucket/gpt2/",
|
||||
load_format=load_format,
|
||||
tensor_parallel_size=1,
|
||||
).create_engine_config()
|
||||
assert vllm_config.model_config.model_weights == "s3://my-mock-bucket/gpt2/"
|
||||
|
||||
executor = RunaiDummyExecutor(vllm_config)
|
||||
executor.driver_worker.load_model()
|
||||
model = executor.driver_worker.model_runner.get_model()
|
||||
(param,) = [p for n, p in model.named_parameters() if n.endswith("ln_f.weight")]
|
||||
|
||||
def current() -> torch.Tensor:
|
||||
return param.detach().float().cpu()
|
||||
|
||||
torch.testing.assert_close(current(), original, rtol=1e-2, atol=1e-2)
|
||||
|
||||
# No weights_path: re-stream the checkpoint the engine started from, i.e.
|
||||
# the bucket kept in model_weights. Zero the live tensor first so a reload
|
||||
# that did nothing is distinguishable from one that streamed the bucket.
|
||||
param.data.zero_()
|
||||
executor.driver_worker.reload_weights()
|
||||
torch.testing.assert_close(current(), original, rtol=1e-2, atol=1e-2)
|
||||
|
||||
# weights_path: the new prefix must win over the stale model_weights.
|
||||
executor.driver_worker.reload_weights(weights_path="s3://my-mock-bucket/gpt2-mod/")
|
||||
torch.testing.assert_close(current(), modified, rtol=1e-2, atol=1e-2)
|
||||
|
||||
# After the override, a bare reload means the new checkpoint, not the
|
||||
# one the engine was started with.
|
||||
param.data.zero_()
|
||||
executor.driver_worker.reload_weights()
|
||||
torch.testing.assert_close(current(), modified, rtol=1e-2, atol=1e-2)
|
||||
|
||||
@@ -579,7 +579,7 @@ async def test_serialize_and_serve_entrypoints(tmp_path):
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(fut, 180)
|
||||
except asyncio.TimeoutError:
|
||||
except TimeoutError:
|
||||
pytest.fail("Server did not start successfully")
|
||||
finally:
|
||||
proc.terminate()
|
||||
|
||||
@@ -4,11 +4,17 @@
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from safetensors.torch import save_file
|
||||
|
||||
from vllm.config.load import LoadConfig
|
||||
from vllm.model_executor.model_loader import DefaultModelLoader
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
filter_duplicate_safetensors_files,
|
||||
filter_safetensors_files_by_weight_name,
|
||||
)
|
||||
|
||||
|
||||
@@ -77,3 +83,57 @@ def test_filter_duplicate_safetensors_files_all_exist():
|
||||
if __name__ == "__main__":
|
||||
test_filter_duplicate_safetensors_files_missing_weight()
|
||||
test_filter_duplicate_safetensors_files_all_exist()
|
||||
|
||||
|
||||
def _skip_non_mtp(name: str) -> bool:
|
||||
return not name.startswith("mtp.")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mtp_checkpoint(tmp_path):
|
||||
"""Three shards: target only, target + MTP, MTP only."""
|
||||
shards = {
|
||||
"model-00001-of-00003.safetensors": ["model.layers.0.weight"],
|
||||
"model-00002-of-00003.safetensors": ["model.layers.1.weight", "mtp.a.weight"],
|
||||
"model-00003-of-00003.safetensors": ["mtp.b.weight"],
|
||||
}
|
||||
for filename, names in shards.items():
|
||||
save_file({name: torch.zeros(1) for name in names}, tmp_path / filename)
|
||||
return tmp_path
|
||||
|
||||
|
||||
def test_filter_by_weight_name_drops_shards_without_wanted_weights(mtp_checkpoint):
|
||||
files = sorted(str(f) for f in mtp_checkpoint.glob("*.safetensors"))
|
||||
|
||||
assert filter_safetensors_files_by_weight_name(files, _skip_non_mtp) == files[1:]
|
||||
# A rule that rejects everything must not leave the loader without files.
|
||||
assert filter_safetensors_files_by_weight_name(files, lambda _: True) == files
|
||||
|
||||
|
||||
def test_loader_does_not_read_shards_the_model_skips(mtp_checkpoint, monkeypatch):
|
||||
"""Whole-file loaders such as InstantTensor only see the kept shards."""
|
||||
opened: list[list[str]] = []
|
||||
|
||||
def fake_iterator(hf_weights_files, *args, **kwargs):
|
||||
opened.append(sorted(os.path.basename(f) for f in hf_weights_files))
|
||||
return iter(())
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.model_loader.default_loader."
|
||||
"instanttensor_weights_iterator",
|
||||
fake_iterator,
|
||||
)
|
||||
|
||||
class Drafter(torch.nn.Module):
|
||||
def is_unused_checkpoint_weight(self, name: str) -> bool:
|
||||
return _skip_non_mtp(name)
|
||||
|
||||
loader = DefaultModelLoader(LoadConfig(load_format="instanttensor"))
|
||||
model_config = SimpleNamespace(model=str(mtp_checkpoint), revision=None)
|
||||
list(loader.get_all_weights(model_config, Drafter()))
|
||||
list(loader.get_all_weights(model_config, torch.nn.Module()))
|
||||
|
||||
assert opened == [
|
||||
["model-00002-of-00003.safetensors", "model-00003-of-00003.safetensors"],
|
||||
sorted(f.name for f in mtp_checkpoint.glob("*.safetensors")),
|
||||
]
|
||||
|
||||
@@ -528,6 +528,7 @@ def test_padded_moe_reload_releases_each_layer(
|
||||
layer.quant_method = method
|
||||
layer.expert_map_manager = SimpleNamespace(map_global_to_local=lambda i: i)
|
||||
layer._loaded_expert_biases = set()
|
||||
layer._fused_shared_expert_quantizer = None
|
||||
method.create_weights(
|
||||
layer,
|
||||
experts,
|
||||
@@ -1051,6 +1052,37 @@ def test_hpc_rope_norm_kernel_sees_refit_norm_weights(monkeypatch, hpc_rope_norm
|
||||
assert not hasattr(rnorm, "qnorm_weight")
|
||||
|
||||
|
||||
def _device_tensor_ptrs(worker) -> dict[str, int]:
|
||||
"""Addresses of every device tensor held by a module; graphs capture them."""
|
||||
ptrs = {}
|
||||
for name, module in worker.get_model().named_modules():
|
||||
tensors = {**vars(module), **module._parameters, **module._buffers}
|
||||
for attr, tensor in tensors.items():
|
||||
if isinstance(tensor, torch.Tensor) and tensor.device.type != "cpu":
|
||||
ptrs[f"{name}.{attr}"] = tensor.data_ptr()
|
||||
return ptrs
|
||||
|
||||
|
||||
def _moved_tensors(llm, ptrs: list[dict[str, int]]) -> list[list[str]]:
|
||||
after = llm.collective_rpc(_device_tensor_ptrs)
|
||||
return [
|
||||
sorted(n for n, p in old.items() if new.get(n) != p)
|
||||
for old, new in zip(ptrs, after)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"use_aiter",
|
||||
[
|
||||
False,
|
||||
pytest.param(
|
||||
True,
|
||||
marks=pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="AITER is ROCm-only"
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"tp_size", [pytest.param(1), pytest.param(2, marks=[pytest.mark.slow_test])]
|
||||
)
|
||||
@@ -1094,13 +1126,19 @@ def test_hpc_rope_norm_kernel_sees_refit_norm_weights(monkeypatch, hpc_rope_norm
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_reload_weights(base_model, mul_model, add_model, tp_size, vllm_runner):
|
||||
def test_reload_weights(
|
||||
base_model, mul_model, add_model, tp_size, use_aiter, vllm_runner, monkeypatch
|
||||
):
|
||||
if current_platform.device_count() < tp_size:
|
||||
pytest.skip(reason="Not enough CUDA devices")
|
||||
|
||||
if "FP8" in base_model and _fp8_reload_unsupported():
|
||||
pytest.skip(reason="Requires FP8 support")
|
||||
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
if use_aiter:
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
|
||||
with vllm_runner(
|
||||
model_name=base_model,
|
||||
tensor_parallel_size=tp_size,
|
||||
@@ -1109,12 +1147,16 @@ def test_reload_weights(base_model, mul_model, add_model, tp_size, vllm_runner):
|
||||
max_model_len=16,
|
||||
max_num_seqs=1,
|
||||
) as llm:
|
||||
# Reloading must update tensors in place: captured graphs keep pointers.
|
||||
ptrs = llm.collective_rpc(_device_tensor_ptrs)
|
||||
llm.collective_rpc("reload_weights", kwargs={"weights_path": mul_model})
|
||||
assert _moved_tensors(llm, ptrs) == [[]] * len(ptrs)
|
||||
mul_perp = llm.generate_prompt_perplexity(["3 4 = 12"], mask=["3 4 ="])[0]
|
||||
add_perp = llm.generate_prompt_perplexity(["3 4 = 7"], mask=["3 4 ="])[0]
|
||||
assert mul_perp < add_perp
|
||||
|
||||
llm.collective_rpc("reload_weights", kwargs={"weights_path": add_model})
|
||||
assert _moved_tensors(llm, ptrs) == [[]] * len(ptrs)
|
||||
mul_perp = llm.generate_prompt_perplexity(["3 4 = 12"], mask=["3 4 ="])[0]
|
||||
add_perp = llm.generate_prompt_perplexity(["3 4 = 7"], mask=["3 4 ="])[0]
|
||||
assert add_perp < mul_perp
|
||||
|
||||
@@ -356,6 +356,52 @@ def test_weight_cache_key_distinguishes_dp_ranks():
|
||||
assert key.mismatched_fields(replace(key, pp_rank=0)) == ["pp_rank"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hf_quant_config", [None, {"quant_algo": "NVFP4"}])
|
||||
def test_weight_cache_key_distinguishes_nvfp4_activation_override(
|
||||
tmp_path, hf_quant_config
|
||||
):
|
||||
"""Cache keys isolate activation modes and reject ambiguous legacy hashes."""
|
||||
import json
|
||||
from dataclasses import replace
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config.quantization import QuantizationConfigArgs
|
||||
from vllm.model_executor.model_loader.weight_cache.protocol import WeightCacheKey
|
||||
from vllm.utils.hashing import safe_hash
|
||||
|
||||
model_config = SimpleNamespace(
|
||||
model=str(tmp_path),
|
||||
dtype=torch.bfloat16,
|
||||
quantization="modelopt_fp4",
|
||||
quantization_config=None,
|
||||
revision=None,
|
||||
hf_config=SimpleNamespace(
|
||||
architectures=["NemotronHForCausalLM"],
|
||||
quantization_config=hf_quant_config,
|
||||
),
|
||||
)
|
||||
static_key = WeightCacheKey.from_model_config(model_config, tp_size=1, tp_rank=0)
|
||||
model_config.quantization_config = QuantizationConfigArgs(
|
||||
moe={"activation": "nvfp4_per_token"}
|
||||
)
|
||||
per_token_key = WeightCacheKey.from_model_config(model_config, tp_size=1, tp_rank=0)
|
||||
assert static_key.mismatched_fields(per_token_key) == ["quant_config_hash"]
|
||||
|
||||
legacy_hash = (
|
||||
""
|
||||
if hf_quant_config is None
|
||||
else safe_hash(
|
||||
json.dumps(hf_quant_config, sort_keys=True).encode(),
|
||||
usedforsecurity=False,
|
||||
).hexdigest()
|
||||
)
|
||||
assert static_key.mismatched_fields(
|
||||
replace(static_key, quant_config_hash=legacy_hash)
|
||||
) == ["quant_config_hash"]
|
||||
|
||||
|
||||
def test_ipc_loader_copy_mode_reports_no_external_weight_memory():
|
||||
"""Copy mode clones the weights into the engine, so nothing is external
|
||||
(vllm_config is never touched)."""
|
||||
|
||||
@@ -39,13 +39,13 @@ def _make_moe(*, max_deferred_tokens: int = 128, enabled: bool = True):
|
||||
return moe
|
||||
|
||||
|
||||
def _make_runner(modules, *, max_tokens: int = 8192, linear_backend: str = "auto"):
|
||||
def _make_runner(modules, *, max_tokens: int = 8192):
|
||||
"""Create a runner carrying only the state used by FlashInfer warmup."""
|
||||
return SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=max_tokens),
|
||||
vllm_config=SimpleNamespace(
|
||||
kernel_config=SimpleNamespace(linear_backend=linear_backend),
|
||||
attention_config=SimpleNamespace(hisparse_config=None),
|
||||
parallel_config=SimpleNamespace(data_parallel_rank=0),
|
||||
),
|
||||
get_model=Mock(
|
||||
return_value=SimpleNamespace(modules=Mock(return_value=modules))
|
||||
@@ -62,21 +62,19 @@ def test_flashinfer_autotune_token_counts_include_deferred_moe_limits():
|
||||
_make_moe(max_deferred_tokens=128),
|
||||
_make_moe(max_deferred_tokens=64, enabled=False),
|
||||
_make_moe(max_deferred_tokens=-1),
|
||||
],
|
||||
linear_backend="flashinfer_cutedsl",
|
||||
]
|
||||
)
|
||||
|
||||
with patch("vllm.model_executor.layers.fused_moe.MoERunner", _FakeMoERunner):
|
||||
token_counts = _flashinfer_autotune_token_counts(runner)
|
||||
|
||||
assert token_counts == (8192, 32, 128)
|
||||
assert token_counts == (8192, 128)
|
||||
|
||||
|
||||
def test_flashinfer_autotune_token_counts_are_bounded_and_deduplicated():
|
||||
runner = _make_runner(
|
||||
[_make_moe(max_deferred_tokens=4096)],
|
||||
max_tokens=32,
|
||||
linear_backend="flashinfer_cutedsl",
|
||||
)
|
||||
|
||||
with patch("vllm.model_executor.layers.fused_moe.MoERunner", _FakeMoERunner):
|
||||
@@ -114,8 +112,8 @@ def test_flashinfer_autotune_uses_token_buckets_for_each_dummy_run(skip_attn):
|
||||
|
||||
assert get_buckets.call_args_list == [call(8192), call(128)]
|
||||
assert autotune.call_args_list == [
|
||||
call(tuning_buckets=max_buckets),
|
||||
call(tuning_buckets=deferred_buckets),
|
||||
call(tuning_buckets=max_buckets, round_up=True),
|
||||
call(tuning_buckets=deferred_buckets, round_up=True),
|
||||
]
|
||||
assert runner._dummy_run.call_args_list == [
|
||||
call(
|
||||
@@ -135,6 +133,33 @@ def test_flashinfer_autotune_uses_token_buckets_for_each_dummy_run(skip_attn):
|
||||
]
|
||||
|
||||
|
||||
def test_flashinfer_autotune_buckets_cover_drafter_tokens():
|
||||
"""The drafter's M can exceed the pass size; buckets must include it."""
|
||||
runner = _make_runner([])
|
||||
runner.max_num_reqs = 256
|
||||
runner.max_num_tokens = 8192
|
||||
runner.speculator = SimpleNamespace(num_query_per_req=6)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.model_executor.warmup.kernel_warmup."
|
||||
"_flashinfer_autotune_token_counts",
|
||||
return_value=(8192, 128),
|
||||
),
|
||||
patch(
|
||||
"vllm.utils.flashinfer.flashinfer_get_hybrid_num_tokens_buckets"
|
||||
) as get_buckets,
|
||||
patch("vllm.utils.flashinfer.autotune"),
|
||||
):
|
||||
_run_flashinfer_autotune_dummy_runs(runner)
|
||||
|
||||
assert get_buckets.call_args_list == [call(8192), call(128 * 6)]
|
||||
assert [c.kwargs["num_tokens"] for c in runner._dummy_run.call_args_list] == [
|
||||
8192,
|
||||
128,
|
||||
]
|
||||
|
||||
|
||||
class _AutotuneGroup:
|
||||
def __init__(self, run, ranks):
|
||||
self.run = run
|
||||
@@ -146,12 +171,20 @@ class _AutotuneGroup:
|
||||
def record(self, operation):
|
||||
self.run.collectives[self.ranks][self.run.rank].append(operation)
|
||||
|
||||
def broadcast_object(self, obj, src=0):
|
||||
assert src == 0
|
||||
self.record(("broadcast", src))
|
||||
if self.rank_in_group == src:
|
||||
self.run.broadcasts[self.ranks] = obj
|
||||
return self.run.broadcasts[self.ranks]
|
||||
def all_gather_object(self, out, obj):
|
||||
# Ranks run sequentially: snapshot every rank's file before any saves.
|
||||
self.record(("all_gather",))
|
||||
if isinstance(obj, bool):
|
||||
out[:] = [rank not in self.run.rejected_load_ranks for rank in self.ranks]
|
||||
assert out[self.rank_in_group] == obj
|
||||
return
|
||||
out[:] = self.run.gathered.setdefault(
|
||||
self.ranks,
|
||||
[
|
||||
warmup._autotune_cache_fingerprint(self.run.cache_path(rank))
|
||||
for rank in self.ranks
|
||||
],
|
||||
)
|
||||
|
||||
def barrier(self):
|
||||
self.record(("barrier",))
|
||||
@@ -162,10 +195,18 @@ class _AutotuneTuner:
|
||||
self.run = run
|
||||
self.cache = {}
|
||||
self.loaded = None
|
||||
self._dirty = False
|
||||
|
||||
def load_configs(self, path):
|
||||
if self.run.rank in self.run.rejected_load_ranks:
|
||||
return False
|
||||
self.loaded = json.loads(Path(path).read_text())
|
||||
self.cache.update(self.loaded)
|
||||
return True
|
||||
|
||||
def clear_cache(self):
|
||||
self.cache.clear()
|
||||
self._dirty = False
|
||||
|
||||
def save_configs(self, path):
|
||||
self.run.saves.append((self.run.rank, Path(path), dict(self.cache)))
|
||||
@@ -182,21 +223,27 @@ class _AutotuneTuner:
|
||||
if group is not None:
|
||||
group.record(("all_reduce", operation, tactic))
|
||||
self.cache[operation] = self.run.rank // self.run.tp
|
||||
self._dirty = True
|
||||
|
||||
|
||||
class _AutotuneRun:
|
||||
def __init__(self, pp, tp):
|
||||
def __init__(self, pp, tp, cache_dir):
|
||||
self.pp, self.tp = pp, tp
|
||||
self.cache_dir = cache_dir
|
||||
self.rank = 0
|
||||
self.tuning_group = None
|
||||
self.collectives: dict[tuple[int, ...], dict[int, list[tuple[Any, ...]]]] = (
|
||||
defaultdict(lambda: defaultdict(list))
|
||||
)
|
||||
self.broadcasts = {}
|
||||
self.gathered = {}
|
||||
self.rejected_load_ranks: set[int] = set()
|
||||
self.tuners = {}
|
||||
self.saves = []
|
||||
self.profile_groups = defaultdict(list)
|
||||
|
||||
def cache_path(self, rank):
|
||||
return self.cache_dir / f"autotune_configs_dp0_rank{rank}.json"
|
||||
|
||||
def world(self):
|
||||
return _AutotuneGroup(self, range(self.pp * self.tp))
|
||||
|
||||
@@ -235,7 +282,7 @@ def autotune_run(monkeypatch, tmp_path):
|
||||
from vllm.distributed import parallel_state
|
||||
|
||||
def make_run(*, pp=2, tp=4):
|
||||
run = _AutotuneRun(pp, tp)
|
||||
run = _AutotuneRun(pp, tp, tmp_path)
|
||||
autotuner = ModuleType("flashinfer.autotuner")
|
||||
monkeypatch.setattr(
|
||||
autotuner,
|
||||
@@ -251,6 +298,10 @@ def autotune_run(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(parallel_state, "get_tp_group", run.tensor_group)
|
||||
monkeypatch.setattr(parallel_state, "get_pp_group", run.pipeline_group)
|
||||
monkeypatch.setattr(fi_utils, "autotune", lambda **kwargs: nullcontext())
|
||||
monkeypatch.setattr(
|
||||
"torch.distributed.all_gather_object",
|
||||
lambda out, obj, group: group.all_gather_object(out, obj),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
warmup,
|
||||
"resolve_flashinfer_autotune_file",
|
||||
@@ -286,19 +337,18 @@ def test_pp_stage_cache_roundtrip_isolated_and_asymmetric_hits_safe(
|
||||
legacy = tmp_path / "autotune_configs.json"
|
||||
legacy.write_text('{"legacy_world_cache": 99}')
|
||||
cold = autotune_run().execute()
|
||||
assert [rank for rank, _, _ in cold.saves] == [0, 4]
|
||||
paths = [path for _, path, _ in cold.saves]
|
||||
assert len(set(paths)) == 2 and legacy not in paths
|
||||
assert json.loads(legacy.read_text()) == {"legacy_world_cache": 99}
|
||||
assert cold.saves[0][2] == {"shared_gemm": 0, "pp0_extra_gemm": 0}
|
||||
assert cold.saves[1][2] == {"shared_gemm": 1}
|
||||
cold.assert_collectives_match()
|
||||
assert [rank for rank, _, _ in cold.saves] == list(range(8))
|
||||
assert [path for _, path, _ in cold.saves] == [cold.cache_path(r) for r in range(8)]
|
||||
assert json.loads(legacy.read_text()) == {"legacy_world_cache": 99}
|
||||
stage_caches = [{"shared_gemm": 0, "pp0_extra_gemm": 0}, {"shared_gemm": 1}]
|
||||
assert all(cache == stage_caches[r // 4] for r, _, cache in cold.saves)
|
||||
warm = autotune_run().execute()
|
||||
warm.assert_collectives_match()
|
||||
assert not warm.profile_groups
|
||||
assert not warm.profile_groups and not warm.saves
|
||||
for rank, tuner in warm.tuners.items():
|
||||
assert tuner.loaded == cold.saves[rank // 4][2]
|
||||
paths[1].unlink()
|
||||
assert tuner.loaded == stage_caches[rank // 4]
|
||||
cold.cache_path(5).unlink()
|
||||
mixed = autotune_run().execute()
|
||||
mixed.assert_collectives_match()
|
||||
assert set(mixed.profile_groups) == {4, 5, 6, 7}
|
||||
@@ -306,10 +356,33 @@ def test_pp_stage_cache_roundtrip_isolated_and_asymmetric_hits_safe(
|
||||
assert all(mixed.tuners[rank].loaded is None for rank in range(4, 8))
|
||||
|
||||
|
||||
def test_pp1_retains_world_synchronization_and_existing_cache_name(autotune_run):
|
||||
def test_pp1_tunes_world_group_and_saves_per_rank(autotune_run):
|
||||
run = autotune_run(pp=1, tp=4).execute()
|
||||
run.assert_collectives_match()
|
||||
assert [(rank, path.name) for rank, path, _ in run.saves] == [
|
||||
(0, "autotune_configs.json")
|
||||
assert [(rank, path) for rank, path, _ in run.saves] == [
|
||||
(rank, run.cache_path(rank)) for rank in range(4)
|
||||
]
|
||||
assert all(groups == [(0, 1, 2, 3)] for groups in run.profile_groups.values())
|
||||
|
||||
|
||||
def test_mismatched_rank_caches_are_ignored_by_every_rank(autotune_run):
|
||||
"""One rank loading while others tune deadlocks the per-tactic reduce."""
|
||||
cold = autotune_run(pp=1, tp=4).execute()
|
||||
stale = cold.cache_path(2)
|
||||
stale.write_text(json.dumps({**json.loads(stale.read_text()), "old_gemm": 0}))
|
||||
rerun = autotune_run(pp=1, tp=4).execute()
|
||||
rerun.assert_collectives_match()
|
||||
assert all(tuner.loaded is None for tuner in rerun.tuners.values())
|
||||
assert set(rerun.profile_groups) == {0, 1, 2, 3}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pp, tp", [(1, 1), (1, 4), (2, 4)])
|
||||
def test_rejected_cache_retunes_every_rank_in_its_tuning_group(autotune_run, pp, tp):
|
||||
"""Matching files can still be incompatible with one rank's runtime."""
|
||||
autotune_run(pp=pp, tp=tp).execute()
|
||||
rerun = autotune_run(pp=pp, tp=tp)
|
||||
rerun.rejected_load_ranks = {tp - 1}
|
||||
rerun.execute()
|
||||
rerun.assert_collectives_match()
|
||||
assert set(rerun.profile_groups) == set(range(tp))
|
||||
assert {rank for rank, _, _ in rerun.saves} == set(range(tp))
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.phi4mm_utils import MultiHeadedAttention
|
||||
|
||||
|
||||
def test_eager_attention_does_not_retain_probability_matrix():
|
||||
module = MultiHeadedAttention(
|
||||
n_head=2,
|
||||
n_feat=16,
|
||||
dropout_rate=0.0,
|
||||
use_pt_scaled_dot_product_attention=False,
|
||||
)
|
||||
module.eval()
|
||||
batch, time, feat = 2, 8, 16
|
||||
x = torch.randn(batch, time, feat)
|
||||
with torch.no_grad():
|
||||
out = module(x, x, x, None, None, None)
|
||||
|
||||
assert out.shape == (batch, time, feat)
|
||||
assert getattr(module, "attn", None) is None
|
||||
|
||||
|
||||
def test_multiheaded_attention_defaults_to_scaled_dot_product():
|
||||
module = MultiHeadedAttention(n_head=2, n_feat=16, dropout_rate=0.0)
|
||||
assert module.use_pt_scaled_dot_product_attention is True
|
||||
@@ -145,27 +145,45 @@ def test_mrope_positions_video_at_prompt_end():
|
||||
assert positions.shape == (3, len(input_tokens))
|
||||
|
||||
|
||||
def test_mrope_positions_audio_in_video_at_prompt_end():
|
||||
@pytest.mark.parametrize("num_videos", [1, 2])
|
||||
@pytest.mark.parametrize("trailing_tokens", [0, 3])
|
||||
def test_mrope_positions_audio_in_video_boundaries(num_videos, trailing_tokens):
|
||||
"""Interleaved BOS/EOS positions and decode delta match Qwen3-Omni HF."""
|
||||
model = make_model()
|
||||
# video grid (1, 2, 2) -> 4 video tokens; audio_feature_length=16 -> 2
|
||||
# audio tokens, interleaved after the video tokens.
|
||||
# [T, T, <|vision_start|>, <|audio_start|>, V x 4, A x 2,
|
||||
# <|audio_end|>, <|vision_end|>]
|
||||
input_tokens = (
|
||||
[TEXT_TOKEN_ID] * 2
|
||||
+ [VISION_START_TOKEN_ID, AUDIO_START_TOKEN_ID]
|
||||
video_tokens = (
|
||||
[VISION_START_TOKEN_ID, AUDIO_START_TOKEN_ID]
|
||||
+ [VIDEO_TOKEN_ID] * 4
|
||||
+ [AUDIO_TOKEN_ID] * 2
|
||||
+ [AUDIO_END_TOKEN_ID, VISION_END_TOKEN_ID]
|
||||
)
|
||||
mm_features = [
|
||||
make_video_feature(offset=3, grid_thw=(1, 2, 2), use_audio_in_video=True),
|
||||
make_audio_feature(offset=3, audio_feature_length=16),
|
||||
]
|
||||
input_tokens = [TEXT_TOKEN_ID] * 2
|
||||
mm_features = []
|
||||
expected = [torch.tensor([[0, 1]]).expand(3, -1)]
|
||||
video_positions = torch.tensor(
|
||||
[
|
||||
[2, 3, 4, 4, 4, 4, 4, 5, 6, 7],
|
||||
[2, 3, 4, 4, 5, 5, 4, 5, 6, 7],
|
||||
[2, 3, 4, 5, 4, 5, 4, 5, 6, 7],
|
||||
]
|
||||
)
|
||||
for index in range(num_videos):
|
||||
offset = len(input_tokens) + 1
|
||||
mm_features.extend(
|
||||
[
|
||||
make_video_feature(offset, (1, 2, 2), use_audio_in_video=True),
|
||||
make_audio_feature(offset, audio_feature_length=16),
|
||||
]
|
||||
)
|
||||
input_tokens.extend(video_tokens)
|
||||
expected.append(video_positions + index * 6)
|
||||
|
||||
positions, _ = model.get_mrope_input_positions(input_tokens, mm_features)
|
||||
input_tokens.extend([TEXT_TOKEN_ID] * trailing_tokens)
|
||||
next_position = 2 + num_videos * 6
|
||||
expected.append((torch.arange(trailing_tokens) + next_position).expand(3, -1))
|
||||
positions, delta = model.get_mrope_input_positions(input_tokens, mm_features)
|
||||
|
||||
assert positions.shape == (3, len(input_tokens))
|
||||
torch.testing.assert_close(positions, torch.cat(expected, dim=1))
|
||||
assert delta == -4 * num_videos
|
||||
|
||||
|
||||
def test_mrope_positions_image_then_text():
|
||||
|
||||
@@ -582,10 +582,7 @@ def test_v2_model_runner_accepts_routed_experts(monkeypatch):
|
||||
use_ubatching=False,
|
||||
enable_elastic_ep=False,
|
||||
),
|
||||
compilation_config=SimpleNamespace(
|
||||
mode=CompilationMode.NONE,
|
||||
pass_config=SimpleNamespace(enable_sp=False),
|
||||
),
|
||||
compilation_config=SimpleNamespace(mode=CompilationMode.NONE),
|
||||
cache_config=SimpleNamespace(
|
||||
kv_sharing_fast_prefill=False,
|
||||
mamba_cache_mode="none",
|
||||
|
||||
Reference in New Issue
Block a user