Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-15 21:16:39 +07:00
1693 changed files with 99919 additions and 32440 deletions
@@ -21,6 +21,7 @@ from vllm.model_executor.kernels.linear import (
B12xMxFp4LinearKernel,
B12xMxfp8LinearKernel,
B12xNvFp4LinearKernel,
B12xNvFp4W4A16LinearKernel,
B12xTensorFP8ScaledMMLinearKernel,
FP8ScaledMMLinearLayerConfig,
Mxfp8LinearLayerConfig,
@@ -782,7 +783,7 @@ def test_b12x_nvfp4_can_implement_supported_config() -> None:
assert reason is None
def test_b12x_backend_preserves_w4a16_fallback(monkeypatch) -> None:
def test_b12x_backend_selects_w4a16(monkeypatch) -> None:
import vllm.model_executor.kernels.linear as linear_mod
monkeypatch.setattr(linear_mod.current_platform, "_enum", PlatformEnum.CUDA)
@@ -792,17 +793,17 @@ def test_b12x_backend_preserves_w4a16_fallback(monkeypatch) -> None:
monkeypatch.setitem(
linear_mod._POSSIBLE_NVFP4_KERNELS,
PlatformEnum.CUDA,
[B12xNvFp4LinearKernel, MarlinNvFp4LinearKernel],
[B12xNvFp4LinearKernel, B12xNvFp4W4A16LinearKernel, MarlinNvFp4LinearKernel],
)
monkeypatch.setattr(
MarlinNvFp4LinearKernel,
B12xNvFp4W4A16LinearKernel,
"is_supported",
classmethod(lambda cls, compute_capability=None: (True, None)),
)
kernel = init_nvfp4_linear_kernel(use_a16=True)
assert isinstance(kernel, MarlinNvFp4LinearKernel)
assert isinstance(kernel, B12xNvFp4W4A16LinearKernel)
def test_b12x_nvfp4_apply_calls_native_blockscaled_gemm(monkeypatch) -> None:
@@ -859,3 +860,227 @@ def test_b12x_nvfp4_apply_calls_native_blockscaled_gemm(monkeypatch) -> None:
layer.alpha,
)
assert kwargs == {"out_dtype": torch.bfloat16}
def test_b12x_nvfp4_w4a16_uses_weight_only_scale_and_a16_dispatch(monkeypatch):
import vllm.model_executor.kernels.linear.nvfp4.b12x as b12x_mod
calls = []
packed_weight = object()
def pack_weight(weight, scale, *, recipe, global_scale):
assert weight is layer.weight
assert scale is layer.weight_scale
assert recipe == "nvfp4"
assert global_scale is layer.weight_global_scale
return packed_weight
def mm(source, weight, *, required_mode):
assert weight is packed_weight
assert required_mode == "a16"
assert source.dtype == torch.bfloat16
assert source.is_contiguous()
calls.append(source)
return torch.full((source.shape[0], 48), 3.0, dtype=source.dtype)
def reject_activation_quantization(*args, **kwargs):
pytest.fail("W4A16 must not quantize activations")
monkeypatch.setattr(b12x_mod, "scaled_fp4_quant", reject_activation_quantization)
monkeypatch.setattr(
b12x_mod,
"_import_b12x_blockscaled",
lambda: types.SimpleNamespace(pack_weight=pack_weight, mm=mm),
)
monkeypatch.setattr(
b12x_mod,
"_import_b12x_intrinsics",
lambda: types.SimpleNamespace(swizzle_block_scale=lambda scale: scale),
)
layer = torch.nn.Module()
layer.weight = torch.nn.Parameter(
torch.empty((48, 64), dtype=torch.uint8), requires_grad=False
)
layer.weight_scale = torch.nn.Parameter(
torch.empty((48, 8), dtype=torch.float8_e4m3fn), requires_grad=False
)
layer.weight_global_scale = torch.tensor([0.125])
kernel = object.__new__(B12xNvFp4W4A16LinearKernel)
kernel.process_weights_after_loading(layer)
assert layer.b12x_warmup_provider is kernel
kernel.get_b12x_warmup_unit(layer, (1, 8), torch.bfloat16).compile()
assert [source.shape[0] for source in calls] == [1, 8]
x = torch.randn((2, 3, 256), dtype=torch.bfloat16)[..., ::2]
output = kernel.apply_weights(layer, x, torch.ones(48, dtype=torch.bfloat16))
torch.testing.assert_close(calls[-1], x.reshape(6, 128))
torch.testing.assert_close(output, torch.full((2, 3, 48), 4.0, dtype=x.dtype))
with pytest.raises(ValueError, match="requires BF16 input"):
kernel.apply_weights(layer, x.float())
@pytest.mark.parametrize("m,n,k", [(2, 136, 96), (17, 256, 256)])
@torch.inference_mode()
def test_b12x_nvfp4_w4a16_preserves_bf16_activations_under_graph_replay(
monkeypatch, default_vllm_config, m, n, k
) -> None:
import vllm.model_executor.kernels.linear.nvfp4.b12x as b12x_mod
supported, reason = B12xNvFp4W4A16LinearKernel.is_supported()
if not supported:
pytest.skip(reason)
def reject_activation_quantization(*args, **kwargs):
pytest.fail("W4A16 must not quantize activations")
monkeypatch.setattr(b12x_mod, "scaled_fp4_quant", reject_activation_quantization)
torch.manual_seed(17)
codes = torch.randint(0, 16, (n, k), device="cuda")
scales = (2.0 ** torch.randint(-2, 2, (n, k // 16), device="cuda")).to(
torch.float8_e4m3fn
)
layer = torch.nn.Module()
layer.weight = torch.nn.Parameter(
(codes[:, ::2] | codes[:, 1::2] << 4).to(torch.uint8), requires_grad=False
)
layer.weight_scale = torch.nn.Parameter(scales, requires_grad=False)
layer.weight_global_scale = torch.nn.Parameter(
torch.tensor([0.125], device="cuda"), requires_grad=False
)
lut = torch.tensor(
[0, 0.5, 1, 1.5, 2, 3, 4, 6, 0, -0.5, -1, -1.5, -2, -3, -4, -6],
device="cuda",
)
decoded = lut[codes] * scales.float().repeat_interleave(16, dim=1) * 0.125
x_storage = torch.randn(1, m, k * 2, device="cuda", dtype=torch.bfloat16)
x = x_storage[..., ::2]
bias = torch.randn(n, device="cuda", dtype=torch.bfloat16)
kernel = B12xNvFp4W4A16LinearKernel(None)
kernel.process_weights_after_loading(layer)
kernel.get_b12x_warmup_unit(layer, (1, 2, 4, 32), x.dtype).compile()
run = torch.compile(
lambda inputs: kernel.apply_weights(layer, inputs, bias),
fullgraph=True,
dynamic=True,
)
run(x)
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
actual = run(x)
for _ in range(2):
x_storage.normal_()
graph.replay()
expected = (x.double() @ decoded.double().T).bfloat16() + bias
torch.testing.assert_close(actual, expected, atol=0.005, rtol=0.01)
def _block_weights(n, k, codec):
block_size, block_bytes = (
(32, 34) if codec == "q8_0" else (256, 66 if codec == "iq2_xxs" else 74)
)
raw = torch.randint(0, 256, (n, k // block_size, block_bytes), dtype=torch.uint8)
raw[..., :2] = torch.full(
(n, k // block_size, 1), 1 / 128, dtype=torch.float16
).view(torch.uint8)
return raw
@pytest.mark.parametrize("codec", ["iq2_xs", "iq2_xxs", "q8_0"])
@pytest.mark.parametrize("profile_first", [False, True])
@torch.inference_mode()
def test_b12x_block_quant_dense_compiled_graph(
default_vllm_config, codec, profile_first
):
supported, reason = B12xNvFp4W4A16LinearKernel.is_supported()
if not supported:
pytest.skip(reason)
from b12x.testing.iq2_xs_reference import dequantize_blocks
from vllm.model_executor.layers.quantization.modelopt_block_quant import (
ModelOptBlockQuantLinearMethod,
)
default_vllm_config.scheduler_config.max_num_batched_tokens = 32
default_vllm_config.scheduler_config.max_num_scheduled_tokens = 32
default_vllm_config.compilation_config.cudagraph_capture_sizes = [1, 4]
method = ModelOptBlockQuantLinearMethod(codec)
raw = _block_weights(128, 256, codec)
decoded = dequantize_blocks(raw).bfloat16().cuda()
layer = torch.nn.Module()
layer.weight = torch.nn.Parameter(raw.cuda(), requires_grad=False)
method.process_weights_after_loading(layer)
assert layer.b12x_block_plan.prepared is None
if profile_first:
method.apply(layer, torch.zeros(32, 256, device="cuda", dtype=torch.bfloat16))
layer.b12x_warmup_provider.get_b12x_warmup_unit(
layer, (1, 4, 32), torch.bfloat16
).compile()
assert layer.b12x_block_plan.prepared is not None
run = torch.compile(lambda x: method.apply(layer, x), fullgraph=True, dynamic=True)
for rows in (1, 4, 17):
x = torch.randn(rows, 256, device="cuda", dtype=torch.bfloat16)
run(x)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
output = run(x)
for _ in range(2):
x.normal_()
allocated = torch.accelerator.memory_allocated()
graph.replay()
torch.accelerator.synchronize()
assert torch.accelerator.memory_allocated() == allocated
expected = (x.float() @ decoded.float().T).bfloat16()
torch.testing.assert_close(output, expected, rtol=0.015, atol=0.03125)
graph.reset()
@pytest.mark.parametrize("id_dtype", [torch.int32, torch.int64])
@pytest.mark.parametrize("profile_first", [False, True])
@torch.inference_mode()
def test_b12x_q8_embedding_graph(default_vllm_config, id_dtype, profile_first):
supported, reason = B12xNvFp4W4A16LinearKernel.is_supported()
if not supported:
pytest.skip(reason)
from b12x.testing.q8_0_reference import dequantize_blocks
from vllm.model_executor.layers.quantization.modelopt_block_quant import (
ModelOptBlockQuantLinearMethod,
)
default_vllm_config.scheduler_config.max_num_batched_tokens = 32
default_vllm_config.scheduler_config.max_num_scheduled_tokens = 32
default_vllm_config.compilation_config.cudagraph_capture_sizes = [1, 4]
method = ModelOptBlockQuantLinearMethod("q8_0")
method.is_embedding = True
raw = _block_weights(128, 256, "q8_0")
decoded = dequantize_blocks(raw).bfloat16().cuda()
layer = torch.nn.Module()
layer.weight = torch.nn.Parameter(raw.cuda(), requires_grad=False)
method.process_weights_after_loading(layer)
assert all(plan.prepared is None for plan in layer.b12x_embedding_plans.values())
if profile_first:
method.embedding(layer, torch.zeros(32, device="cuda", dtype=id_dtype))
layer.b12x_warmup_provider.get_b12x_warmup_unit(
layer, (1, 4, 32), torch.bfloat16
).compile()
assert all(
plan.prepared is not None for plan in layer.b12x_embedding_plans.values()
)
run = torch.compile(
lambda x: method.embedding(layer, x), fullgraph=True, dynamic=True
)
for count in (7, 9, 70):
ids = torch.arange(count, device="cuda", dtype=id_dtype)
torch._dynamo.mark_dynamic(ids, 0)
torch.testing.assert_close(run(ids), decoded[ids.long()], rtol=0, atol=0)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
output = run(ids)
ids.add_(20)
allocated = torch.accelerator.memory_allocated()
graph.replay()
torch.accelerator.synchronize()
assert torch.accelerator.memory_allocated() == allocated
torch.testing.assert_close(output, decoded[ids.long()], rtol=0, atol=0)
graph.reset()
@@ -182,7 +182,8 @@ def get_fse_test_model_config(
head_dim=128,
max_position_embeddings=128,
sliding_window=None,
compress_ratios=[1],
layer_types=["sliding_attention"],
compress_rates={},
rope_theta=10000.0,
compress_rope_theta=10000.0,
rope_parameters={"rope_type": "default"},
@@ -771,6 +771,8 @@ def test_sparse_mqa_logits_prefill_matches_dense_masked(capture: bool):
candidates = _make_candidates(lens, num_candidates, cbk).cuda()
# Reference: dense bf16 logits masked to the candidates, then torch.topk.
# The logits are compressed: row r starts at K position ks[r].
max_len = int(lens.max())
dense = deep_gemm.fp8_fp4_mqa_logits(
(
q_fp.view(torch.int8).view(rows, num_heads, head_dim // 2),
@@ -780,19 +782,13 @@ def test_sparse_mqa_logits_prefill_matches_dense_masked(capture: bool):
weights,
ks,
ke,
clean_logits=False,
logits_dtype=torch.bfloat16,
max_len,
)
cols = torch.arange(total, device="cuda")
in_range = (cols - ks.unsqueeze(1) >= 0) & (
cols - ks.unsqueeze(1) < lens.unsqueeze(1)
)
masked = dense.masked_fill(
~(in_range & _candidate_token_mask(candidates, cbk, total, ks)),
float("-inf"),
).float()
ref_abs = _reference_topk(masked, topk) # packed-workspace positions
ref = torch.where(ref_abs >= 0, ref_abs - ks.unsqueeze(1), -1)
in_range = torch.arange(max_len, device="cuda") < lens.unsqueeze(1)
# A row's last candidate block may extend past max_len.
in_cand = _candidate_token_mask(candidates, cbk, max_len + cbk)[:, :max_len]
masked = dense.masked_fill(~(in_range & in_cand), float("-inf")).float()
ref = _reference_topk(masked, topk)
out = torch.full((rows, topk), -1, dtype=torch.int32, device="cuda")
scratch = _sparse_scratch(rows, num_candidates, topk)
@@ -827,11 +823,11 @@ def test_sparse_mqa_logits_prefill_matches_dense_masked(capture: bool):
graph.replay()
else:
kernel_metadata = run_sparse()
_assert_same_selection(dense, out, ref, row_starts=ks)
_assert_same_selection(dense, out, ref)
out.fill_(-1)
run_sparse(kernel_metadata)
_assert_same_selection(dense, out, ref, row_starts=ks)
_assert_same_selection(dense, out, ref)
def _build_fp4_paged_cache(req_lens: torch.Tensor, page_kv: int, head_dim: int):
@@ -902,21 +898,22 @@ def test_sparse_mqa_logits_paged_matches_dense_masked(next_n: int, capture: bool
candidates = _make_candidates(row_lens.flatten(), num_candidates, cbk).cuda()
def reference_topk():
context_lens_2d = row_lens.view(batch, next_n)
# SM100 paged MQA logits take varlen rows: one Q token per row.
context_lens_2d = row_lens.view(rows, 1)
indices = torch.arange(batch, dtype=torch.int32, device="cuda")
indices = indices.repeat_interleave(next_n)
sched = deep_gemm.get_paged_mqa_logits_metadata(
context_lens_2d, page_kv, deep_gemm.get_num_sms(), None
context_lens_2d, page_kv, deep_gemm.get_num_sms(), indices
)
logits = deep_gemm.fp8_fp4_paged_mqa_logits(
(q, q_scale),
(q.view(rows, 1, num_heads, -1), q_scale.view(rows, 1, num_heads)),
kv_cache,
weights,
context_lens_2d,
block_table,
block_table.repeat_interleave(next_n, dim=0),
sched,
int(req_lens.max()),
False,
torch.bfloat16,
None,
indices,
)
lens_flat = row_lens.flatten()
cols = torch.arange(logits.shape[1], device="cuda")
@@ -308,6 +308,7 @@ class TestTokenEmbeddingPoolerHead:
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_embed")
out = head.forward_chunk(x, param)
assert out is x
assert torch.equal(out, x)
def test_none_chunked_prefill(self):
@@ -13,19 +13,22 @@ from transformers import PreTrainedConfig
from vllm.config import PoolerConfig, set_current_vllm_config
from vllm.model_executor.layers.pooler import PoolingParamsUpdate
from vllm.model_executor.layers.pooler.activations import PoolerNormalize
from vllm.model_executor.layers.pooler.seqwise.methods import (
CLSPool,
LastPool,
MeanPool,
get_seq_pooling_method,
)
from vllm.model_executor.layers.pooler.special import DispatchPooler
from vllm.model_executor.layers.pooler.special import BOSEOSFilter, DispatchPooler
from vllm.model_executor.layers.pooler.tokwise.heads import TokenEmbeddingPoolerHead
from vllm.model_executor.layers.pooler.tokwise.methods import (
AllPool,
StepPool,
get_tok_pooling_method,
)
from vllm.pooling_params import PoolingParams
from vllm.model_executor.layers.pooler.tokwise.poolers import TokenPooler
from vllm.pooling_params import LateChunkingParams, PoolingParams
from vllm.tasks import PoolingTask
from vllm.v1.pool.metadata import PoolingCursor, PoolingMetadata, PoolingStates
@@ -606,3 +609,139 @@ class TestGetTokPoolingMethod:
def test_unknown_raises(self):
with pytest.raises(NotImplementedError, match="UNKNOWN"):
get_tok_pooling_method("UNKNOWN")
@pytest.mark.parametrize(
("chunk_size", "dtype", "head_dtype"),
[
(1, torch.float32, None),
(2, torch.float16, None),
(2, torch.bfloat16, torch.float32),
(7, torch.float32, None),
(10, torch.float32, None),
(2**63 - 1, torch.float32, None),
],
)
def test_late_chunk_pool_matches_fp32_means_before_projection_and_normalization(
chunk_size, dtype, head_dtype
):
# Unequal norms expose the incorrect normalize-then-average implementation.
hidden = (torch.arange(28).reshape(7, 4) - 10).to(dtype)
projector = torch.nn.Linear(4, 3, bias=True).to(head_dtype or dtype)
head = TokenEmbeddingPoolerHead(head_dtype, projector, PoolerNormalize())
pooler = TokenPooler(TestAllPool._make_all_pool(), head)
params = PoolingParams(
task="token_embed",
late_chunking_params=LateChunkingParams(chunk_size=chunk_size),
use_activation=True,
)
metadata = _make_metadata([7], pooling_params=[params])
actual = pooler(hidden, metadata)[0]
means = torch.stack(
[hidden[a : a + chunk_size].float().mean(0) for a in range(0, 7, chunk_size)]
).to(head_dtype or dtype)
expected = torch.nn.functional.normalize(projector(means), p=2, dim=-1)
torch.testing.assert_close(actual, expected)
assert actual.dtype == (head_dtype or dtype)
def test_late_chunk_pool_reduces_before_nonlinear_projector():
hidden = torch.tensor([[2.0, 0.0], [0.0, 1.0]])
projector = torch.nn.Tanh()
head = TokenEmbeddingPoolerHead(projector=projector, activation=PoolerNormalize())
params = PoolingParams(
task="token_embed",
late_chunking_params=LateChunkingParams(chunk_size=2),
use_activation=True,
)
actual = head.forward_chunk(hidden, params)
expected = torch.nn.functional.normalize(projector(hidden.mean(0, keepdim=True)))
wrong_order = torch.nn.functional.normalize(projector(hidden).mean(0, keepdim=True))
torch.testing.assert_close(actual, expected)
assert not torch.allclose(actual, wrong_order)
def test_late_chunk_pool_isolates_mixed_requests_and_owns_finished_storage():
hidden = torch.arange(36, dtype=torch.float32).reshape(9, 4)
params = [
PoolingParams(
task="token_embed", late_chunking_params=LateChunkingParams(chunk_size=2)
),
PoolingParams(task="token_embed"),
PoolingParams(
task="token_embed", late_chunking_params=LateChunkingParams(chunk_size=1)
),
]
pooler = TokenPooler(
TestAllPool._make_all_pool(async_scheduling=True), TokenEmbeddingPoolerHead()
)
outputs = pooler(hidden, _make_metadata([3, 4, 2], pooling_params=params))
expected = [
torch.stack([hidden[:2].mean(0), hidden[2]]),
hidden[3:7].clone(),
hidden[7:].clone(),
]
hidden.fill_(-999)
for actual, reference in zip(outputs, expected):
torch.testing.assert_close(actual, reference)
@pytest.mark.parametrize("chunk_size", [1, 2, 8])
@pytest.mark.parametrize("embedding_size", [1, 4])
def test_bos_eos_filter_preserves_chunk_vectors_in_mixed_batch(
chunk_size, embedding_size
):
hidden = torch.arange(10 * embedding_size, dtype=torch.float32).reshape(
10, embedding_size
)
pooler = BOSEOSFilter(
TokenPooler(TestAllPool._make_all_pool(), TokenEmbeddingPoolerHead()),
bos_token_id=101,
eos_token_id=102,
)
metadata = _make_metadata(
[5, 5],
token_ids=[[101, 1, 2, 3, 102]] * 2,
pooling_params=[
PoolingParams(
task="token_embed", late_chunking_params=LateChunkingParams(chunk_size)
),
PoolingParams(task="token_embed"),
],
)
chunked, ordinary = pooler(hidden, metadata)
expected = torch.stack([part.mean(0) for part in hidden[:5].split(chunk_size)])
torch.testing.assert_close(chunked, expected)
torch.testing.assert_close(ordinary, hidden[6:9].squeeze(-1))
def test_bos_eos_filter_waits_for_complete_chunk_vectors():
pooler = BOSEOSFilter(
TokenPooler(
TestAllPool._make_all_pool(chunked=True), TokenEmbeddingPoolerHead()
),
bos_token_id=101,
)
params = [
PoolingParams(task="token_embed", late_chunking_params=LateChunkingParams(3))
]
first = _make_metadata(
[4],
token_ids=[[101, 1, 2, 102]],
pooling_params=params,
num_scheduled_tokens=[2],
seq_lens=[2],
)
hidden = torch.arange(8, dtype=torch.float32).reshape(4, 2)
assert pooler(hidden[:2], first) == [None]
last = _make_metadata(
[4],
token_ids=[[101, 1, 2, 102]],
pooling_params=params,
num_scheduled_tokens=[2],
seq_lens=[4],
)
last.pooling_states = first.pooling_states
actual = pooler(hidden[2:], last)[0]
torch.testing.assert_close(actual, torch.stack([hidden[:3].mean(0), hidden[3]]))
assert not last.pooling_states[0].hidden_states_cache
@@ -2,9 +2,13 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import glob
import sys
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
import torch
from safetensors.torch import save_file
from vllm.model_executor.model_loader.weight_utils import (
download_weights_from_hf,
@@ -14,6 +18,60 @@ from vllm.model_executor.model_loader.weight_utils import (
from vllm.platforms import current_platform
@pytest.mark.parametrize(
("weight_name", "local_ids", "collective"),
[
("mlp.experts.gate_proj.weight", {0}, True),
("mlp.experts.0.gate_proj.weight", {0}, False),
("mlp.experts.0.gate_proj.weight", set(), False),
],
)
def test_instanttensor_ep_collective_loading(
monkeypatch, tmp_path, weight_name, local_ids, collective
):
"""Keep fused checkpoints collective and expert decisions rank-independent."""
from vllm.model_executor.model_loader import weight_utils
path = tmp_path / "model.safetensors"
save_file({weight_name: torch.ones(1)}, str(path))
native_open = MagicMock()
reader = native_open.return_value.__enter__.return_value
reader.total_tensor_size = 0
reader.tensors.return_value = ()
monkeypatch.setitem(
sys.modules, "instanttensor", SimpleNamespace(safe_open=native_open)
)
group = SimpleNamespace(world_size=2, device_group=object())
monkeypatch.setattr(weight_utils, "get_world_group", lambda: group)
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
monkeypatch.setattr(current_platform, "current_device", lambda: 0)
list(instanttensor_weights_iterator([str(path)], False, local_expert_ids=local_ids))
kwargs = native_open.call_args.kwargs
assert kwargs["process_group"] is (group.device_group if collective else None)
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="InstantTensor requires NVIDIA GPUs",
)
def test_instanttensor_ep_filter_cuda(tmp_path):
weights = {
"mlp.experts.0.gate_proj.weight": torch.ones(2),
"mlp.experts.1.gate_proj.weight": torch.ones(2),
"mlp.experts.1.gate_proj.weight_scale": torch.ones(1),
"model.layers[0].attn.weight": torch.ones(2),
}
path = tmp_path / "model.safetensors"
save_file(weights, str(path))
loaded = dict(
instanttensor_weights_iterator([str(path)], False, local_expert_ids={0})
)
del weights["mlp.experts.1.gate_proj.weight"]
assert loaded.keys() == weights.keys()
for name, tensor in loaded.items():
torch.testing.assert_close(tensor.cpu(), weights[name], rtol=0, atol=0)
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="InstantTensor requires NVIDIA GPUs",
@@ -1,13 +1,18 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import os
import types
from datetime import timedelta
from pathlib import Path
from unittest.mock import patch
import pytest
import torch
from safetensors.torch import load_file, save_file
from tests.utils import create_new_process_for_each_test
from vllm import SamplingParams
from vllm.config.load import LoadConfig
from vllm.model_executor.model_loader import get_model_loader
@@ -39,12 +44,130 @@ def test_get_model_loader_with_runai_flag():
assert model_loader.__class__.__name__ == "RunaiModelStreamerLoader"
def test_checkpoint_selection_hook_is_optional_for_model_detection():
from vllm.model_executor.models.interfaces import SupportsMultiModal
from vllm.model_executor.models.interfaces_base import (
VllmModel,
is_text_generation_model,
)
class ModelWithoutHook(torch.nn.Module):
def __init__(self, vllm_config, prefix=""):
super().__init__()
def embed_input_ids(self, input_ids):
return input_ids
def forward(self, input_ids, positions):
return input_ids
def compute_logits(self, hidden_states):
return hidden_states
assert is_text_generation_model(ModelWithoutHook)
assert is_text_generation_model(ModelWithoutHook(None))
class ModelWithDefaultRule(ModelWithoutHook, VllmModel):
pass
assert not ModelWithDefaultRule(None).is_unused_checkpoint_weight("any.weight")
# Structural models do not inherit the protocol's optional default hook.
for names in ([], ["language_model"]):
model = torch.nn.Module()
model.language_model = ModelWithoutHook(None)
model._language_model_names = names
assert SupportsMultiModal.get_language_model(model) is model.language_model
def test_runai_model_loader_download_files(vllm_runner):
with vllm_runner(test_model, load_format=load_format) as llm:
deserialized_outputs = llm.generate(prompts, sampling_params)
assert deserialized_outputs
# In-process MTP compilation must not retain GPU resources in the pytest worker.
@create_new_process_for_each_test("spawn")
def test_runai_mtp_real_checkpoint(vllm_runner, tmp_path, monkeypatch):
"""Skip main-only shards without changing a real MTP model's generation."""
from vllm.model_executor.models.qwen3_5_mtp import Qwen3_5MTP
from vllm.platforms import current_platform
from vllm.transformers_utils.repo_utils import hf_api
from vllm.v1.metrics.reader import Counter
if not current_platform.is_cuda_alike():
pytest.skip("MTP serving requires CUDA or ROCm")
checkpoint = tmp_path / "checkpoint"
hf_api().snapshot_download(
repo_id="Qwen/Qwen3.5-0.8B-Base", local_dir=str(checkpoint)
)
original_files = list(checkpoint.glob("*.safetensors"))
tensors = {}
for path in original_files:
tensors.update(load_file(path))
# Reshard real, unmodified weights so selection cannot pass vacuously on
# the published single mixed file. This predicate needs no model state.
selector = Qwen3_5MTP.__new__(Qwen3_5MTP)
main: dict[str, torch.Tensor] = {}
draft: dict[str, torch.Tensor] = {}
for name, tensor in tensors.items():
(main if selector.is_unused_checkpoint_weight(name) else draft)[name] = tensor
assert main and draft
files = [checkpoint / "main.safetensors", checkpoint / "draft.safetensors"]
for path, contents in zip(files, (main, draft)):
save_file(contents, path, metadata={"format": "pt"})
for path in original_files:
path.unlink()
(checkpoint / "model.safetensors.index.json").write_text(
json.dumps(
{
"weight_map": {
name: path.name
for path, data in zip(files, (main, draft))
for name in data
}
}
)
)
del tensors, main, draft
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
opened = []
real_stream = rsl.runai_safetensors_weights_iterator
def stream(paths, *args):
opened.append(list(paths))
yield from real_stream(paths, *args)
monkeypatch.setattr(rsl, "runai_safetensors_weights_iterator", stream)
results = []
for load_format in ("auto", "runai_streamer"):
with vllm_runner(
str(checkpoint),
load_format=load_format,
block_size=None,
enable_chunked_prefill=True,
disable_log_stats=False,
limit_mm_per_prompt={"image": 0, "video": 0},
speculative_config={"method": "mtp", "num_speculative_tokens": 1},
) as runner:
outputs = runner.generate_greedy(prompts, max_tokens=32)
results.append(outputs)
accepted = next(
m.value
for m in runner.llm.get_metrics()
if isinstance(m, Counter)
and m.name == "vllm:spec_decode_num_accepted_tokens"
)
assert accepted > 0
assert len(opened) == 2
assert set(opened[0]) == {str(path) for path in files}
assert opened[1] == [str(files[1])]
matches = sum(a == b for a, b in zip(*results))
assert matches >= 3, "RunAI MTP generation diverged from the Default loader"
@pytest.mark.skip(
reason="Temporarily disabled due to GCS access issues. "
"TODO: Re-enable this test once the underlying issue is resolved."
@@ -150,7 +273,7 @@ def test_runai_invalid_extra_config_leaves_environ_untouched():
)
def test_runai_get_all_weights_resolves_source(model_config, expected_source):
fake_self = types.SimpleNamespace(
_get_weights_iterator=lambda path, revision: iter(
_get_weights_iterator=lambda path, revision, is_unused_weight=None: iter(
[(f"{path}@{revision}", None)]
)
)
@@ -164,6 +287,387 @@ def test_runai_get_all_weights_resolves_source(model_config, expected_source):
assert weights == [(f"{expected_source}@myrev", None)]
@pytest.mark.parametrize("selection", ["draft", "unadapted", "reject_all"])
def test_runai_selects_shards_before_streaming(tmp_path, monkeypatch, selection):
"""Unused shards must not reach RunAI; mixed shards retain their tensors."""
contents = [
{"main.weight": torch.tensor([1.0])},
{"main.bias": torch.tensor([2.0]), "draft.weight": torch.tensor([3.0])},
{"draft.bias": torch.tensor([4.0])},
]
files = []
for i, tensors in enumerate(contents):
path = str(tmp_path / f"part-{i}.safetensors")
save_file(tensors, path)
files.append(path)
monkeypatch.setattr(rsl, "list_safetensors", lambda path: files)
opened = []
real_stream = rsl.runai_safetensors_weights_iterator
def stream(paths, *args):
opened.extend(paths)
yield from real_stream(paths, *args)
monkeypatch.setattr(rsl, "runai_safetensors_weights_iterator", stream)
model = torch.nn.Module()
if selection == "draft":
model.is_unused_checkpoint_weight = lambda name: not name.startswith("draft.")
elif selection == "reject_all":
model.is_unused_checkpoint_weight = lambda name: True
config = types.SimpleNamespace(model=str(tmp_path), model_weights="", revision=None)
loaded = dict(_runai_loader({}).get_all_weights(config, model))
expected = files[1:] if selection == "draft" else files
assert opened == expected
assert set(loaded) == {
name
for tensors in (contents[1:] if selection == "draft" else contents)
for name in tensors
}
for name, tensor in {**contents[1], **contents[2]}.items():
torch.testing.assert_close(loaded[name], tensor)
def _check_distributed_selection(rank, rendezvous, scenario):
torch.distributed.init_process_group(
"gloo",
init_method=rendezvous,
rank=rank,
world_size=2,
timeout=timedelta(seconds=30),
)
root = "s3://bucket/weights"
if scenario == "different_roots":
root += str(rank)
files = [root + "/main.safetensors", root + "/draft.safetensors"]
def pull_files(source, destination, **kwargs):
if scenario == "metadata_failure" and rank == 1:
raise OSError("Index temporarily unavailable")
(Path(destination) / "model.safetensors.index.json").write_text(
json.dumps(
{
"weight_map": {
"main.weight": "main.safetensors",
"draft.weight": "draft.safetensors",
}
}
)
)
try:
with pytest.MonkeyPatch.context() as monkeypatch:
monkeypatch.setattr(rsl, "list_safetensors", lambda path: files)
monkeypatch.setattr(rsl, "runai_pull_files", pull_files)
monkeypatch.setattr(
rsl,
"_get_remote_checkpoint_keys",
lambda paths: [
["main.weight"]
if path.endswith("main.safetensors")
else ["draft.weight"]
for path in paths
],
)
monkeypatch.setattr(
rsl,
"get_world_group",
lambda: types.SimpleNamespace(
world_size=2, cpu_group=torch.distributed.group.WORLD
),
raising=False,
)
monkeypatch.setattr(
rsl,
"get_pp_group",
lambda: types.SimpleNamespace(
world_size=2 if scenario == "pipeline_draft" else 1
),
raising=False,
)
if scenario == "pipeline_draft" and rank == 0:
# Earlier PP stages never invoke the draft loader.
torch.distributed.barrier()
return
# Exercise real selection and Gloo; observe the distributed stream input.
monkeypatch.setattr(
rsl,
"runai_safetensors_weights_iterator",
lambda paths, *args: iter((path, None) for path in paths),
)
model = torch.nn.Module()
if not (scenario == "missing_hook" and rank == 1):
needed = (
"main." if scenario == "different_roles" and rank == 1 else "draft."
)
model.is_unused_checkpoint_weight = lambda name: (
not name.startswith(needed)
)
config = types.SimpleNamespace(model=root, model_weights="", revision=None)
selected = [
path
for path, _ in _runai_loader({"distributed": True}).get_all_weights(
config, model
)
]
expected = files[1:] if scenario == "healthy" else files
assert selected == expected
if scenario == "pipeline_draft":
torch.distributed.barrier()
finally:
torch.distributed.destroy_process_group()
@pytest.mark.parametrize(
"scenario",
[
"healthy",
"metadata_failure",
"different_roles",
"missing_hook",
"different_roots",
"pipeline_draft",
],
)
def test_distributed_runai_preserves_every_ranks_required_shards(tmp_path, scenario):
"""All ranks must stream the same union even if one cannot filter safely."""
torch.multiprocessing.spawn(
_check_distributed_selection,
args=((tmp_path / "rendezvous").as_uri(), scenario),
nprocs=2,
)
@pytest.mark.parametrize("scheme", ["s3", "gs", "az"])
@pytest.mark.parametrize(
"index_kind",
[
"valid",
"missing",
"malformed",
"unavailable",
"unadapted",
"missing_shard",
"stale",
"header_unavailable",
],
)
def test_runai_remote_selection_uses_actual_weight_source(
tmp_path, monkeypatch, scheme, index_kind
):
root = f"{scheme}://bucket/weights"
files = [
root + "/main/part.safetensors",
root + "/draft/part.safetensors",
root + "/mixed.safetensors",
root + "/extra.safetensors",
]
monkeypatch.setattr(rsl, "list_safetensors", lambda path: files)
pulled = []
def pull_files(source, destination, allow_pattern, ignore_pattern=None):
pulled.append((source.rstrip("/"), allow_pattern))
if index_kind == "unavailable":
# Storage SDKs do not necessarily derive errors from OSError.
raise Exception("Object metadata unavailable")
index = Path(destination) / "model.safetensors.index.json"
if index_kind in ("valid", "missing_shard", "stale", "header_unavailable"):
index.write_text(
json.dumps(
{
"weight_map": {
"main.weight": "main/part.safetensors",
"draft.weight": (
"absent.safetensors"
if index_kind == "missing_shard"
else "draft/part.safetensors"
),
"main.bias": "mixed.safetensors",
"draft.bias": "mixed.safetensors",
}
}
)
)
elif index_kind == "malformed":
index.write_text("not JSON")
monkeypatch.setattr(rsl, "runai_pull_files", pull_files, raising=False)
headers_checked = []
def checkpoint_keys(paths):
headers_checked.extend(paths)
if index_kind == "header_unavailable":
raise OSError("Header temporarily unavailable")
return [
["main.weight", "draft.unindexed_weight"]
if index_kind == "stale"
else ["main.weight"]
for _ in paths
]
monkeypatch.setattr(
rsl, "_get_remote_checkpoint_keys", checkpoint_keys, raising=False
)
opened = []
def stream(paths, *args):
opened.extend(paths)
return iter(())
monkeypatch.setattr(rsl, "runai_safetensors_weights_iterator", stream)
model = torch.nn.Module()
if index_kind != "unadapted":
model.is_unused_checkpoint_weight = lambda name: not name.startswith("draft.")
# The config directory is not the checkpoint: using its index could drop
# tensors from an independent model_weights source.
(tmp_path / "model.safetensors.index.json").write_text("not the weights index")
config = types.SimpleNamespace(
model=str(tmp_path), model_weights=root, revision=None
)
list(_runai_loader({}).get_all_weights(config, model))
assert opened == (files[1:] if index_kind == "valid" else files)
assert headers_checked == (
files[:1] if index_kind in ("valid", "stale", "header_unavailable") else []
)
assert pulled == (
[]
if index_kind == "unadapted"
else [
(
root,
[
"weights/model.safetensors.index.json",
"model.safetensors.index.json",
],
)
]
)
@pytest.mark.parametrize(
"weight_map",
[None, {}, [], {"draft.weight": None}, {"draft.weight": "../part.safetensors"}],
)
def test_runai_invalid_weight_map_keeps_original_files(monkeypatch, weight_map):
files = ["s3://bucket/model/part.safetensors"]
monkeypatch.setattr(rsl, "list_safetensors", lambda path: files)
def pull_files(source, directory, **kwargs):
(Path(directory) / "model.safetensors.index.json").write_text(
json.dumps({"weight_map": weight_map})
)
monkeypatch.setattr(rsl, "runai_pull_files", pull_files)
monkeypatch.setattr(
rsl, "runai_safetensors_weights_iterator", lambda paths, *args: iter(paths)
)
model = torch.nn.Module()
model.is_unused_checkpoint_weight = lambda name: True
config = types.SimpleNamespace(
model="s3://bucket/model", model_weights="", revision=None
)
assert list(_runai_loader({}).get_all_weights(config, model)) == files
def test_runai_model_rule_error_is_not_hidden_by_metadata_fallback(monkeypatch):
files = ["s3://bucket/model/part.safetensors"]
monkeypatch.setattr(rsl, "list_safetensors", lambda path: files)
def pull_files(source, directory, **kwargs):
(Path(directory) / "model.safetensors.index.json").write_text(
json.dumps({"weight_map": {"draft.weight": "part.safetensors"}})
)
monkeypatch.setattr(rsl, "runai_pull_files", pull_files)
model = torch.nn.Module()
def invalid_rule(name):
raise ValueError("Invalid model rule")
model.is_unused_checkpoint_weight = invalid_rule
config = types.SimpleNamespace(
model="s3://bucket/model", model_weights="", revision=None
)
with pytest.raises(ValueError, match="Invalid model rule"):
list(_runai_loader({}).get_all_weights(config, model))
def test_runai_header_reader_uses_actual_tensor_names(tmp_path):
"""The RunAI header reader sees names even when an index omits them."""
path = tmp_path / "mixed.safetensors"
save_file(
{
"main.weight": torch.tensor([1.0]),
"draft.unindexed_weight": torch.tensor([2.0]),
},
path,
metadata={"format": "pt"},
)
(names,) = rsl._get_remote_checkpoint_keys([str(path)])
assert set(names) == {"main.weight", "draft.unindexed_weight"}
@pytest.mark.parametrize("stale", [False, True])
def test_runai_stale_index_keeps_actual_draft_weights(tmp_path, monkeypatch, stale):
"""Verify excluded headers before dropping a shard with real draft weights."""
root = "s3://bucket/checkpoint"
main_path = tmp_path / "main.safetensors"
draft_path = tmp_path / "draft.safetensors"
main_tensors = {"main.weight": torch.tensor([1.0])}
if stale:
main_tensors["draft.unindexed_weight"] = torch.tensor([2.0])
save_file(main_tensors, main_path)
save_file({"draft.weight": torch.tensor([3.0])}, draft_path)
paths = {
root + "/main.safetensors": str(main_path),
root + "/draft.safetensors": str(draft_path),
}
monkeypatch.setattr(rsl, "list_safetensors", lambda path: list(paths))
def pull_files(source, directory, **kwargs):
(Path(directory) / "model.safetensors.index.json").write_text(
json.dumps(
{
"weight_map": {
"main.weight": "main.safetensors",
"draft.weight": "draft.safetensors",
}
}
)
)
monkeypatch.setattr(rsl, "runai_pull_files", pull_files)
read_headers = rsl._get_remote_checkpoint_keys
monkeypatch.setattr(
rsl,
"_get_remote_checkpoint_keys",
lambda files: read_headers([paths[path] for path in files]),
)
stream = rsl.runai_safetensors_weights_iterator
def stream_local(files, *args):
for name, tensor in stream([paths[path] for path in files], *args):
yield name, tensor.clone()
monkeypatch.setattr(rsl, "runai_safetensors_weights_iterator", stream_local)
model = torch.nn.Module()
model.is_unused_checkpoint_weight = lambda name: not name.startswith("draft.")
config = types.SimpleNamespace(model=root, model_weights="", revision=None)
loaded = dict(_runai_loader({}).get_all_weights(config, model))
assert set(loaded) == (
{"main.weight", "draft.unindexed_weight", "draft.weight"}
if stale
else {"draft.weight"}
)
torch.testing.assert_close(loaded["draft.weight"], torch.tensor([3.0]))
if stale:
torch.testing.assert_close(
loaded["draft.unindexed_weight"], torch.tensor([2.0])
)
@pytest.mark.parametrize(
"base_model,mul_model,add_model",
[
@@ -0,0 +1,337 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
import torch
from safetensors.torch import save_file
from transformers import DeepseekV2Config, DeepseekV2ForCausalLM
from vllm.config import (
AttentionConfig,
DeviceConfig,
KernelConfig,
LoadConfig,
ModelConfig,
VllmConfig,
)
from vllm.model_executor.layers.fused_moe.expert_substitution import (
ConstantExpertSubstitution,
)
from vllm.model_executor.model_loader import get_model_loader
from vllm.model_executor.models.expert_substitution import as_expert_substitution_model
from vllm.model_executor.models.utils import AutoWeightsLoader, PPMissingLayer
from vllm.platforms import current_platform
from vllm.v1.attention.backends.registry import AttentionBackendEnum
MODEL_CONFIG = SimpleNamespace(
hf_config=SimpleNamespace(approximate_experts={"0": [1, 2], "1": [1, 2]})
)
def _compression_config(value_names: dict[int, dict[int, str]]) -> dict:
"""Explicitly named constants given as ``{layer: {expert: tensor name}}``."""
targets = {
f"model.layers.{layer}.mlp.experts": {
"num_logical_experts": 3,
"weight_layout": "compact_retained_experts",
"replacements": {
str(expert): {"format": "constant-v1", "tensors": {"value": name}}
for expert, name in names.items()
},
}
for layer, names in value_names.items()
}
router_semantics = {
"preserve_logical_expert_ids": True,
"preserve_router_weights": True,
"renormalize_after_substitution": False,
}
return {
"transform_config": {
"expert_substitution": {
"version": 1,
"router_semantics": router_semantics,
"targets": targets,
}
}
}
# Expert 1 of both layers shares one explicitly named constant.
EXPLICIT_MODEL_CONFIG = SimpleNamespace(
hf_config=SimpleNamespace(
compression_config=_compression_config(
{
layer: {1: "constants.shared", 2: f"constants.layer_{layer}"}
for layer in (0, 1)
}
)
)
)
def _approx_value(layer: int, expert: int) -> str:
return f"model.layers.{layer}.mlp.experts.{expert}.approx_value"
def _cpu_model(local_layers=(0, 1), tracked=True, model_config=MODEL_CONFIG):
"""Two decoder layers; the others are placeholders as on another PP stage."""
class Model(torch.nn.Module):
def __init__(self, prefix=""):
super().__init__()
self.weight = torch.nn.Parameter(torch.zeros(2), requires_grad=False)
self.model = torch.nn.Module()
self.model.layers = torch.nn.ModuleList()
for index in range(2):
if index not in local_layers:
self.model.layers.append(PPMissingLayer())
continue
layer = torch.nn.Module()
layer.mlp = torch.nn.Module()
layer.mlp.experts = ConstantExpertSubstitution(
index, 3, (1, 2), 2, torch.float32
)
self.model.layers.append(layer)
def load_weights(self, weights):
loaded = AutoWeightsLoader(self).load_weights(weights)
return loaded if tracked else None
return as_expert_substitution_model(Model, model_config)()
@pytest.mark.parametrize("local_layers", [(0, 1), (0,), (1,), ()])
def test_loads_local_approx_values_and_passes_through_others(local_layers):
"""Other PP stages and MTP drafts own none or some of the configured layers."""
model = _cpu_model(local_layers)
loaded = model.load_weights(
[
("weight", torch.tensor([1.0, 2.0])),
*(
(_approx_value(layer, expert), torch.full((2,), 10.0 * layer + expert))
for layer in range(2)
for expert in (1, 2)
),
]
)
model.process_weights_after_loading()
assert loaded == {"weight"} | {
f"model.layers.{layer}.mlp.experts.values" for layer in local_layers
}
for layer in local_layers:
torch.testing.assert_close(
model.model.layers[layer].mlp.experts.values,
torch.tensor([[1.0, 1.0], [2.0, 2.0]]) + 10.0 * layer,
)
@pytest.mark.parametrize("local_layers", [(0, 1), (1,), ()])
def test_loads_explicitly_named_values(local_layers):
"""Named values are consumed even when their layer is built elsewhere."""
model = _cpu_model(local_layers, model_config=EXPLICIT_MODEL_CONFIG)
loaded = model.load_weights(
[
("weight", torch.tensor([1.0, 2.0])),
("constants.shared", torch.full((2,), 1.0)),
("constants.layer_0", torch.full((2,), 2.0)),
("constants.layer_1", torch.full((2,), 3.0)),
]
)
model.process_weights_after_loading()
assert loaded == {"weight"} | {
f"model.layers.{layer}.mlp.experts.values" for layer in local_layers
}
for layer in local_layers:
torch.testing.assert_close(
model.model.layers[layer].mlp.experts.values,
torch.tensor([[1.0, 1.0], [2.0 + layer, 2.0 + layer]]),
)
def test_direct_reload_updates_values_incrementally():
model = _cpu_model(local_layers=(0,), tracked=False)
values = model.model.layers[0].mlp.experts.values
model.load_weights([(_approx_value(0, 2), torch.ones(2))])
with pytest.raises(ValueError, match="missing approx_value .* \\[1\\]"):
model.process_weights_after_loading()
assert model.load_weights([(_approx_value(0, 1), torch.zeros(2))]) is None
model.process_weights_after_loading()
model.load_weights([(_approx_value(0, 2), torch.full((2,), 3.0))])
torch.testing.assert_close(values, torch.tensor([[0.0, 0.0], [3.0, 3.0]]))
with pytest.raises(ValueError, match="not listed"):
model.load_weights([(_approx_value(0, 0), torch.ones(2))])
def test_layerwise_reload_is_rejected_and_keeps_weights():
from vllm.model_executor.model_loader.reload import (
initialize_layerwise_reload,
record_metadata_for_reloading,
)
model = _cpu_model(local_layers=(0,))
record_metadata_for_reloading(model)
loaded = [
("weight", torch.tensor([3.0, 4.0])),
(_approx_value(0, 1), torch.zeros(2)),
(_approx_value(0, 2), torch.ones(2)),
]
model.load_weights(loaded)
model.process_weights_after_loading()
substitution = model.model.layers[0].mlp.experts
values = substitution.values
initialize_layerwise_reload(model)
with pytest.raises(NotImplementedError, match="layerwise weight reload"):
model.load_weights([(name, weight + 1) for name, weight in loaded])
assert substitution.values is values
torch.testing.assert_close(values, torch.tensor([[0.0, 0.0], [1.0, 1.0]]))
torch.testing.assert_close(model.weight, torch.tensor([3.0, 4.0]))
assert substitution.expert_substitution_routes.tolist() == [0, -1, -2]
@pytest.mark.parametrize("missing_value", [False, True])
def test_tensorizer_finalizes_local_substitution_values(monkeypatch, missing_value):
"""Raw Tensorizer loading must reject an unloaded local constant row."""
from vllm.model_executor.model_loader import tensorizer_loader
model = _cpu_model(local_layers=(0,))
weights = [(_approx_value(0, 1), torch.zeros(2))]
if not missing_value:
weights.append((_approx_value(0, 2), torch.ones(2)))
finalize = Mock(wraps=model.process_weights_after_loading)
monkeypatch.setattr(model, "process_weights_after_loading", finalize)
monkeypatch.setattr(tensorizer_loader, "initialize_model", lambda **kwargs: model)
loader = object.__new__(tensorizer_loader.TensorizerLoader)
monkeypatch.setattr(loader, "_get_weights_iterator", lambda: iter(weights))
config = SimpleNamespace(
model_config=SimpleNamespace(dtype=torch.float32),
device_config=SimpleNamespace(device="cpu"),
)
if missing_value:
with pytest.raises(ValueError, match=r"missing approx_value .* \[2\]"):
loader._load_model_serialized_cpu(config)
else:
assert loader._load_model_serialized_cpu(config) is model
torch.testing.assert_close(
model.model.layers[0].mlp.experts.values,
torch.tensor([[0.0, 0.0], [1.0, 1.0]]),
)
finalize.assert_called_once_with()
def _write_tiny_deepseek_mone_checkpoint(
model_dir: Path, explicit_names: bool
) -> tuple[dict[str, torch.Tensor], str]:
config = DeepseekV2Config(
architectures=["DeepseekV2ForCausalLM"],
vocab_size=32,
hidden_size=16,
intermediate_size=32,
moe_intermediate_size=8,
num_hidden_layers=1,
num_attention_heads=2,
num_key_value_heads=2,
max_position_embeddings=32,
first_k_dense_replace=0,
n_routed_experts=3,
n_shared_experts=1,
num_experts_per_tok=1,
n_group=1,
topk_group=1,
norm_topk_prob=False,
q_lora_rank=None,
kv_lora_rank=4,
qk_nope_head_dim=128,
qk_rope_head_dim=64,
v_head_dim=128,
tie_word_embeddings=False,
dtype="float16",
)
experts = "model.layers.0.mlp.experts"
if explicit_names:
value_name = "model.layers.0.mlp.expert_replacements.1.value"
config.compression_config = _compression_config({0: {1: value_name}})
else:
value_name = f"{experts}.1.approx_value"
config.approximate_experts = {"0": [1]}
hf_weights = DeepseekV2ForCausalLM(config).to(torch.float16).state_dict()
gate_up = hf_weights.pop(f"{experts}.gate_up_proj")
down = hf_weights.pop(f"{experts}.down_proj")
weights = {name: value.contiguous() for name, value in hf_weights.items()}
for expert in (0, 2):
gate, up = gate_up[expert].chunk(2, dim=0)
weights[f"{experts}.{expert}.gate_proj.weight"] = gate.contiguous()
weights[f"{experts}.{expert}.up_proj.weight"] = up.contiguous()
weights[f"{experts}.{expert}.down_proj.weight"] = down[expert].contiguous()
weights[value_name] = torch.arange(16, dtype=torch.float16)
config.save_pretrained(model_dir)
save_file(weights, model_dir / "model.safetensors")
return weights, value_name
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like device"
)
@pytest.mark.usefixtures("dist_init", "workspace_init")
@pytest.mark.parametrize(
"explicit_names", [False, True], ids=["approximate_experts", "expert_substitution"]
)
def test_loads_mone_checkpoint_into_compact_experts(
tmp_path: Path, explicit_names: bool
):
weights, value_name = _write_tiny_deepseek_mone_checkpoint(tmp_path, explicit_names)
model_config = ModelConfig(
model=str(tmp_path),
tokenizer=str(tmp_path),
skip_tokenizer_init=True,
dtype="float16",
max_model_len=32,
enforce_eager=True,
)
load_config = LoadConfig(load_format="safetensors", use_tqdm_on_load=False)
vllm_config = VllmConfig(
model_config=model_config,
device_config=DeviceConfig(device="cuda"),
load_config=load_config,
attention_config=AttentionConfig(backend=AttentionBackendEnum.TRITON_MLA),
kernel_config=KernelConfig(moe_backend="triton"),
)
model = get_model_loader(load_config).load_model(vllm_config, model_config)
experts = "model.layers.0.mlp.experts"
routed_experts = model.model.layers[0].mlp.experts.routed_experts
# Logical expert 2 is stored in physical row 1.
torch.testing.assert_close(
routed_experts.w13_weight[1].cpu(),
torch.cat(
[
weights[f"{experts}.2.gate_proj.weight"],
weights[f"{experts}.2.up_proj.weight"],
]
),
)
torch.testing.assert_close(
routed_experts.w2_weight[1].cpu(), weights[f"{experts}.2.down_proj.weight"]
)
substitution = routed_experts.expert_substitution
torch.testing.assert_close(
substitution.values.cpu(), weights[value_name].unsqueeze(0)
)
updated = weights[value_name] + 1
assert model.load_weights([(value_name, updated)]) == {
f"{experts}.routed_experts.expert_substitution.values"
}
torch.testing.assert_close(substitution.values.cpu(), updated.unsqueeze(0))
@@ -137,3 +137,32 @@ def test_loader_does_not_read_shards_the_model_skips(mtp_checkpoint, monkeypatch
["model-00002-of-00003.safetensors", "model-00003-of-00003.safetensors"],
sorted(f.name for f in mtp_checkpoint.glob("*.safetensors")),
]
def test_secondary_source_with_explicit_patterns_skips_backbone_index(tmp_path):
"""A source with explicit allow patterns (e.g. a standalone head file in
the repo root) must not be validated against the backbone's index."""
save_file(
{"backbone.weight": torch.zeros(1)},
tmp_path / "model-00001-of-00001.safetensors",
)
save_file({"head.weight": torch.zeros(1)}, tmp_path / "joint_head.safetensors")
(tmp_path / "model.safetensors.index.json").write_text(
json.dumps(
{"weight_map": {"backbone.weight": "model-00001-of-00001.safetensors"}}
)
)
loader = DefaultModelLoader(LoadConfig())
secondary = DefaultModelLoader.Source(
model_or_path=str(tmp_path),
revision=None,
prefix="pooler.head.",
allow_patterns_overrides=["joint_head.safetensors"],
)
assert [n for n, _ in loader._get_weights_iterator(secondary)] == [
"pooler.head.head.weight"
]
# The primary source still filters files the backbone index doesn't list.
primary = DefaultModelLoader.Source(model_or_path=str(tmp_path), revision=None)
assert [n for n, _ in loader._get_weights_iterator(primary)] == ["backbone.weight"]
@@ -2,13 +2,18 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from torch import nn
from vllm.config import ModelConfig
from vllm.config.load import LoadConfig
from vllm.model_executor.layers.fused_moe.runner.moe_runner import MoERunner
from vllm.model_executor.layers.linear import UnquantizedLinearMethod
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
from vllm.model_executor.model_loader import get_model_loader, register_model_loader
from vllm.model_executor.model_loader.base_loader import BaseModelLoader
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
from vllm.model_executor.models.utils import AutoWeightsLoader
@register_model_loader("custom_load_format")
@@ -88,3 +93,80 @@ def test_default_loader_hf_still_falls_back_to_pt(tmp_path):
)
assert use_safetensors is False
assert any(f.endswith("model.pt") for f in files)
class _QuantMethod:
uses_meta_device = False
def process_weights_after_loading(self, layer: nn.Module) -> None:
pass
def _layer(quant_method=None, **shapes) -> nn.Module:
layer = nn.Module()
for name, shape in shapes.items():
param = nn.Parameter(torch.zeros(shape), requires_grad=False)
layer.register_parameter(name, param)
layer.quant_method = quant_method
return layer
def _track(model: nn.Module, loaded: set[str], quantized: bool) -> str:
loader = DefaultModelLoader(LoadConfig())
try:
loader.track_weights_loading(model, set(loaded), quantized=quantized)
except ValueError as e:
return str(e)
return ""
def test_weights_track_reports_missing_quantized_weights():
# Quantized checkpoints may only omit KV-cache quantization params, empty
# placeholders, fbgemm_fp8's input_scale_ub and online-quant params.
online = _QuantMethod()
online.uses_meta_device = True
model = nn.Module()
model.proj = _layer(_QuantMethod(), weight=(4, 4), weight_scale_inv=(1, 1))
model.unquantized = _layer(UnquantizedLinearMethod(), weight=(4, 4), bias=(4,))
model.gptq = _layer(_QuantMethod(), qweight=(4, 4), qzeros=(0,))
model.fbgemm = _layer(_QuantMethod(), weight=(4, 4), input_scale_ub=())
model.attn = _layer(BaseKVCacheMethod(None), k_scale=(), k_zero_point=())
model.online = _layer(online, weight=(4, 4))
loaded = {"proj.weight", "unquantized.weight", "gptq.qweight", "fbgemm.weight"}
error = _track(model, loaded, quantized=True)
assert "proj.weight_scale_inv" in error and "unquantized.bias" in error
assert not any(n in error for n in ("gptq", "fbgemm", "attn", "online"))
def test_weights_track_keeps_unquantized_models_unchanged():
# The check is on by default here, so unquantized layers stay exempt.
model = nn.Module()
model.score = _layer(_QuantMethod(), weight=(2, 4))
model.norm = _layer(weight=(4,))
assert _track(model, {"norm.weight"}, quantized=False) == ""
assert "norm.weight" in _track(model, set(), quantized=False)
def test_moe_runner_reports_registered_expert_names():
# AutoWeightsLoader prefixes these names, so they must include routed_experts.
class Experts(nn.Module):
def __init__(self):
super().__init__()
self.w13_weight = nn.Parameter(torch.zeros(2, 8, 4), requires_grad=False)
def load_weights(self, weights):
for _ in weights:
yield "w13_weight"
class Runner(nn.Module):
load_weights = MoERunner.load_weights
def __init__(self):
super().__init__()
self.routed_experts = Experts()
model = nn.Module()
model.experts = Runner()
weights = [("experts.w13_weight", torch.zeros(2, 8, 4))]
loaded = AutoWeightsLoader(model).load_weights(weights)
assert loaded == {"experts.routed_experts.w13_weight"}
@@ -5,6 +5,7 @@ import importlib.machinery
import inspect
import sys
import types
from importlib import import_module
from types import SimpleNamespace
from unittest.mock import Mock
from weakref import WeakKeyDictionary, ref
@@ -39,23 +40,227 @@ from vllm.model_executor.model_loader.reload.utils import get_layer_tensors
from vllm.model_executor.model_loader.weight_utils import (
composed_weight_loader,
default_weight_loader,
load_attention_sink,
)
from vllm.platforms import current_platform
@pytest.mark.parametrize("tp_rank", [0, 1])
@pytest.mark.parametrize(
"module_name,class_name,sink_name",
[
("mimo_v2_mtp", "MiMoV2MTP", "attention_sink_bias"),
("mimo_v2_mtp", "MiMoV2OmniMTP", "attention_sink_bias"),
("laguna", "LagunaModel", "sink"),
("qwen3_dflash", "DFlashQwen3Model", "attention_sink_bias"),
],
)
def test_attention_sink_model_loading_shards_once(
monkeypatch, tp_rank, module_name, class_name, sink_name
):
"""Model loading must pass global sinks to the loader on cold load and reload."""
module = import_module(f"vllm.model_executor.models.{module_name}")
from vllm.model_executor.model_loader import weight_utils
from vllm.model_executor.models.utils import WeightsMapper
for target in (module, weight_utils):
monkeypatch.setattr(
target, "get_tensor_model_parallel_rank", lambda: tp_rank, raising=False
)
monkeypatch.setattr(target, "get_tensor_model_parallel_world_size", lambda: 2)
cls = getattr(module, class_name)
model = cls.__new__(cls)
torch.nn.Module.__init__(model)
model.hf_to_vllm_mapper = WeightsMapper()
model.model = torch.nn.Module()
model.model.mtp = torch.nn.Module()
sink = torch.nn.Parameter(torch.zeros(4), requires_grad=False)
def sink_loader(param, loaded_weight):
assert loaded_weight.shape == (8,), "Sink was sharded before weight_loader"
load_attention_sink(param, loaded_weight)
sink.weight_loader = sink_loader
model.model.mtp.register_parameter(sink_name, sink)
name = f"model.mtp.{sink_name}"
checkpoint = torch.arange(8, dtype=torch.float32)
assert model.load_weights([(name, checkpoint)]) == {name}
assert torch.equal(sink, checkpoint.chunk(2)[tp_rank])
record_metadata_for_reloading(model)
initialize_layerwise_reload(model)
model.load_weights([(name, checkpoint + 10)])
finalize_layerwise_reload(model, SimpleNamespace(dtype=torch.float32))
assert torch.equal(sink, (checkpoint + 10).chunk(2)[tp_rank])
assert getattr(model.model.mtp, sink_name).data_ptr() == sink.data_ptr()
@pytest.mark.parametrize("tp_rank", [0, 1])
@pytest.mark.parametrize(
"module_name,class_name,attention_name,sink_name",
[
("deepseek_v4", "DeepseekV4Model", "attn", "attn_sink"),
("deepseek_v41", "DeepseekV4Model", "attn", "attn_sink"),
("hy_v4", "HYV4Model", "self_attn", "learnable_sink_param"),
],
)
def test_attention_sink_deepseek_hy_loading_uses_parameter_loader(
monkeypatch, tp_rank, module_name, class_name, attention_name, sink_name
):
"""DeepSeek and HY-V4 pass global sinks to the parameter's current loader."""
module = import_module(f"vllm.models.{module_name}.nvidia.model")
from vllm.model_executor.model_loader import weight_utils
monkeypatch.setattr(weight_utils, "get_tensor_model_parallel_world_size", lambda: 2)
monkeypatch.setattr(weight_utils, "get_tensor_model_parallel_rank", lambda: tp_rank)
cls = getattr(module, class_name)
model = cls.__new__(cls)
torch.nn.Module.__init__(model)
model.config = SimpleNamespace(tie_word_embeddings=False)
model.quant_config = None
model.get_expert_mapping = lambda: []
attention = torch.nn.Module()
layer = torch.nn.Module()
setattr(layer, attention_name, attention)
model.layers = torch.nn.ModuleList([layer])
sink = torch.nn.Parameter(torch.zeros(6), requires_grad=False)
attention.register_parameter(sink_name, sink)
checkpoint = torch.arange(8, dtype=torch.float32)
name = f"layers.0.{attention_name}.{sink_name}"
for offset in (0, 10):
weight = checkpoint + offset
sink.weight_loader = Mock(side_effect=load_attention_sink)
assert model.load_weights([(name, weight)]) == {name}
sink.weight_loader.assert_called_once_with(sink, weight)
assert torch.equal(sink[:4], weight.chunk(2)[tp_rank])
assert torch.isneginf(sink[4:]).all()
def _gpt_oss_container(monkeypatch, quant_method=None):
"""A minimal GptOssModel whose loading paths can be driven directly."""
from vllm.model_executor.models import gpt_oss
group = SimpleNamespace(world_size=1, rank_in_group=0)
for name in ("get_dp_group", "get_ep_group", "get_pcp_group"):
monkeypatch.setattr(gpt_oss, name, lambda: group)
monkeypatch.setattr(gpt_oss, "get_tensor_model_parallel_world_size", lambda: 1)
monkeypatch.setattr(gpt_oss, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(gpt_oss, "is_pp_missing_parameter", lambda *args: False)
model = gpt_oss.GptOssModel.__new__(gpt_oss.GptOssModel)
torch.nn.Module.__init__(model)
model.config = SimpleNamespace(num_local_experts=1, intermediate_size=32)
if quant_method is not None:
model.config.quantization_config = {"quant_method": quant_method}
model.parallel_config = SimpleNamespace(enable_expert_parallel=True)
model.layers = [
SimpleNamespace(
mlp=SimpleNamespace(
experts=SimpleNamespace(
_quant_method=SimpleNamespace(weight_dtype=None)
)
)
)
]
model.get_expert_mapping = lambda: []
return model
@pytest.mark.parametrize("quant_method", ["mxfp4", "quark", None])
def test_gpt_oss_attention_sink_uses_parameter_loader(monkeypatch, quant_method):
"""All quantization dispatch paths honor the sink's current weight loader."""
model = _gpt_oss_container(monkeypatch, quant_method)
model.sinks = torch.nn.Parameter(torch.zeros(4), requires_grad=False)
# A replacement loader must see the unsliced checkpoint even after reload
# wraps the parameter; bare model-specific copies would bypass this spy.
model.sinks.weight_loader = Mock(side_effect=default_weight_loader)
checkpoint = torch.arange(4, dtype=torch.float32)
assert model.load_weights([("sinks", checkpoint)]) == {"sinks"}
model.sinks.weight_loader.assert_called_once_with(model.sinks, checkpoint)
assert torch.equal(model.sinks, checkpoint)
def test_gpt_oss_attention_sink_skips_missing_parameter(monkeypatch):
"""A sink owned by another pipeline stage is skipped, not a KeyError."""
from vllm.model_executor.models import gpt_oss
model = _gpt_oss_container(monkeypatch)
monkeypatch.setattr(gpt_oss, "is_pp_missing_parameter", lambda *args: True)
# The parameter does not exist on this rank, so loading must not touch it.
sinks = [("sinks", torch.arange(4, dtype=torch.float32))]
assert model.load_weights(sinks) == set()
@pytest.mark.parametrize("tp_rank", [0, 1])
@pytest.mark.parametrize("padding", [0, 4])
def test_attention_sink_layerwise_reload(monkeypatch, tp_rank, padding):
"""Reload must update the sink's kernel storage and restore its padding."""
monkeypatch.setattr(
"vllm.model_executor.model_loader.weight_utils.get_tensor_model_parallel_rank",
lambda: tp_rank,
)
monkeypatch.setattr(
"vllm.model_executor.model_loader.weight_utils."
"get_tensor_model_parallel_world_size",
lambda: 2,
)
class SinkModel(torch.nn.Module):
def __init__(self):
super().__init__()
self.sink = torch.nn.Parameter(
torch.empty(4 + padding), requires_grad=False
)
self.sink.weight_loader = load_attention_sink
self.proj = torch.nn.Parameter(torch.zeros(16), requires_grad=False)
self.proj.weight_loader = default_weight_loader
def load_weights(self, weights):
params = dict(self.named_parameters())
for name, weight in weights:
param = params[name]
param.weight_loader(param, weight)
model = SinkModel()
checkpoint_sink = torch.arange(8, dtype=torch.float32)
model.load_weights([("sink", checkpoint_sink), ("proj", torch.ones(16))])
kernel_sink = model.sink
expected = torch.cat(
(
checkpoint_sink[4 * tp_rank : 4 * (tp_rank + 1)],
torch.full((padding,), -torch.inf),
)
)
assert torch.equal(kernel_sink, expected)
record_metadata_for_reloading(model)
for offset in (10, 20):
kernel_sink.data.fill_(0)
initialize_layerwise_reload(model)
assert model.sink.is_meta
model.load_weights(
[("sink", checkpoint_sink + offset), ("proj", torch.full((16,), offset))]
)
finalize_layerwise_reload(model, SimpleNamespace(dtype=torch.float32))
expected[:4] = checkpoint_sink[4 * tp_rank : 4 * (tp_rank + 1)] + offset
assert torch.equal(kernel_sink, expected)
assert model.sink.data_ptr() == kernel_sink.data_ptr()
assert torch.equal(model.proj, torch.full((16,), offset))
def _fp8_reload_unsupported() -> bool:
"""Whether the FP8 reload/online-quantize tests should be skipped.
``supports_fp8()`` returns True on MI250 (gfx90a) because the general
quantization paths upcast FP8 weights, but gfx90a has no native FP8 and
cannot run these reload models, so treat it as unsupported here.
``supports_fp8()`` returns True on MI250 (gfx90a) and RDNA3 (gfx11)
because the general quantization paths upcast FP8 weights, but neither has
native FP8 and cannot run these reload models, so treat them as
unsupported here.
"""
if not current_platform.supports_fp8():
return True
if current_platform.is_rocm():
from vllm.platforms.rocm import on_gfx90a
from vllm.platforms.rocm import on_gfx11, on_gfx90a
return on_gfx90a()
return on_gfx90a() or on_gfx11()
return False
@@ -6,6 +6,7 @@ import multiprocessing as mp
import os
import shutil
from tempfile import TemporaryDirectory
from types import SimpleNamespace
import pytest
import torch
@@ -49,6 +50,318 @@ def test_filter_subtensors():
assert tensor is state_dict[key]
def _mock_pp_group(
monkeypatch: pytest.MonkeyPatch,
world_size: int,
*,
pp_rank: int = 0,
tp_rank: int = 0,
tp_world_size: int = 1,
) -> None:
"""Stub the parallel-state accessors used by the sharded state loader."""
monkeypatch.setattr(
"vllm.distributed.get_pp_group",
lambda: SimpleNamespace(world_size=world_size, rank_in_group=pp_rank),
)
monkeypatch.setattr(
"vllm.distributed.get_tp_group",
lambda: SimpleNamespace(world_size=tp_world_size, rank_in_group=tp_rank),
)
monkeypatch.setattr(
"vllm.distributed.get_tensor_model_parallel_rank",
lambda: tp_rank,
)
monkeypatch.setattr(
"vllm.distributed.get_pipeline_model_parallel_rank",
lambda: pp_rank,
)
@pytest.mark.parametrize(
"world_size,expected",
[
(1, ShardedStateLoader.TP_ONLY_PATTERN),
(2, ShardedStateLoader.PP_AND_TP_PATTERN),
],
)
def test_get_default_pattern(monkeypatch, world_size, expected):
"""The default pattern is chosen from the pipeline parallel group size."""
_mock_pp_group(monkeypatch, world_size)
assert ShardedStateLoader._get_default_pattern() == expected
def test_render_default_pattern_default_tp(monkeypatch):
"""Without a custom pattern and PP=1, legacy TP-only names are rendered."""
_mock_pp_group(monkeypatch, 1)
assert (
ShardedStateLoader._render_default_pattern(part=0)
== "model-rank-0-part-0.safetensors"
)
def test_render_default_pattern_default_pp(monkeypatch):
"""Without a custom pattern and PP>1, pp/tp-aware names are rendered."""
_mock_pp_group(monkeypatch, 2)
assert (
ShardedStateLoader._render_default_pattern(part=0)
== "model-pp-0-tp-0-part-0.safetensors"
)
def test_render_default_pattern_custom_tp_only(monkeypatch):
"""A custom TP-only pattern is honored when pipeline parallelism is off."""
_mock_pp_group(monkeypatch, 1)
assert (
ShardedStateLoader._render_default_pattern(
ShardedStateLoader.TP_ONLY_PATTERN, part=0
)
== "model-rank-0-part-0.safetensors"
)
def test_render_default_pattern_tp_only_with_pp_fails(monkeypatch):
"""TP-only filenames would collide across pipeline stages; fail fast."""
_mock_pp_group(monkeypatch, 2)
with pytest.raises(ValueError, match="does not embed the pipeline parallel"):
ShardedStateLoader._render_default_pattern(
ShardedStateLoader.TP_ONLY_PATTERN, part=0
)
def test_render_default_pattern_pp_pattern_without_pp_fails(monkeypatch):
"""A pp/tp-aware pattern cannot be rendered when pipeline parallelism is off."""
_mock_pp_group(monkeypatch, 1)
with pytest.raises(ValueError, match="requires pipeline parallelism"):
ShardedStateLoader._render_default_pattern(
ShardedStateLoader.PP_AND_TP_PATTERN, part=0
)
def test_render_default_pattern_custom_pp(monkeypatch):
"""A custom pp/tp-aware pattern is honored when pipeline parallelism is on."""
_mock_pp_group(monkeypatch, 2)
assert (
ShardedStateLoader._render_default_pattern(
ShardedStateLoader.PP_AND_TP_PATTERN, part=0
)
== "model-pp-0-tp-0-part-0.safetensors"
)
@pytest.mark.parametrize(
"pattern,match",
[
# tp field present but pp field missing: pipeline stages with the
# same tp rank would write the same file.
(
"model-tp-{tp_rank}-part-{part}.safetensors",
"does not embed the pipeline parallel",
),
# pp field present but tp field missing: TP workers would overwrite
# each other's shards.
(
"model-pp-{pp_rank}-part-{part}.safetensors",
"does not embed the tensor parallel",
),
],
)
def test_render_default_pattern_incomplete_custom_patterns_fail(
monkeypatch, pattern, match
):
"""A custom pattern must distinguish every active rank (pp and tp)."""
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_rank=0, tp_world_size=2)
with pytest.raises(ValueError, match=match):
ShardedStateLoader._render_default_pattern(pattern, part=0)
def test_render_default_pattern_no_rank_field_with_tp_fails(monkeypatch):
"""Without any rank field, TP workers would collide even when PP is off."""
_mock_pp_group(monkeypatch, 1, tp_world_size=2)
with pytest.raises(ValueError, match="does not embed the tensor parallel"):
ShardedStateLoader._render_default_pattern(
"model-part-{part}.safetensors", part=0
)
def test_render_default_pattern_custom_pp_tp1_allows_pp_only(monkeypatch):
"""With TP=1 a pp-only custom pattern is safe and must be accepted."""
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_world_size=1)
assert (
ShardedStateLoader._render_default_pattern(
"model-pp-{pp_rank}-part-{part}.safetensors", part=0
)
== "model-pp-1-part-0.safetensors"
)
@pytest.mark.parametrize(
"pattern,match",
[
# {{pp_rank}} renders as literal text: both stages would write the
# same file, so it must not count as a pp field.
(
"model-{{pp_rank}}-tp-{tp_rank}-part-{part}.safetensors",
"does not embed the pipeline parallel",
),
# Same for an escaped tp field: TP workers would overwrite each
# other's shards.
(
"model-pp-{pp_rank}-tp-{{tp_rank}}-part-{part}.safetensors",
"does not embed the tensor parallel",
),
],
)
def test_render_default_pattern_escaped_field_not_rank_field(
monkeypatch, pattern, match
):
"""Escaped braces are literal text and cannot pose as rank fields."""
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_rank=0, tp_world_size=2)
with pytest.raises(ValueError, match=match):
ShardedStateLoader._render_default_pattern(pattern, part=0)
@pytest.mark.parametrize(
"pattern,world_size,tp_world_size,pp_rank,tp_rank,expected",
[
# Mixed tp alias works when pipeline parallelism is on.
(
"model-pp-{pp_rank}-rank-{rank}-part-{part}.safetensors",
2,
2,
1,
0,
"model-pp-1-rank-0-part-0.safetensors",
),
# tp_rank alias works for TP-only checkpoints too.
(
"model-tp-{tp_rank}-part-{part}.safetensors",
1,
2,
0,
3,
"model-tp-3-part-0.safetensors",
),
],
)
def test_render_default_pattern_tp_aliases_accepted(
monkeypatch, pattern, world_size, tp_world_size, pp_rank, tp_rank, expected
):
"""Both tp aliases are supplied at render time, so either works."""
_mock_pp_group(
monkeypatch,
world_size,
pp_rank=pp_rank,
tp_rank=tp_rank,
tp_world_size=tp_world_size,
)
assert ShardedStateLoader._render_default_pattern(pattern, part=0) == expected
def test_render_default_pattern_unsupported_field_fails(monkeypatch):
"""Unknown placeholders are rejected instead of KeyError-ing later."""
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_world_size=2)
with pytest.raises(ValueError, match="unsupported placeholder"):
ShardedStateLoader._render_default_pattern(
"model-{pp_rank}-{tp_rank}-{epoch}-part-{part}.safetensors", part=0
)
@pytest.mark.parametrize(
"pp_rank,tp_rank,expected",
[
(0, 0, "model-pp-0-tp-0-part-0.safetensors"),
(1, 2, "model-pp-1-tp-2-part-0.safetensors"),
(3, 5, "model-pp-3-tp-5-part-0.safetensors"),
],
)
def test_render_default_pattern_non_zero_ranks(monkeypatch, pp_rank, tp_rank, expected):
"""The rendered filename must embed the actual pp/tp ranks, not zeros."""
_mock_pp_group(monkeypatch, 2, pp_rank=pp_rank, tp_rank=tp_rank)
assert (
ShardedStateLoader._render_default_pattern(
ShardedStateLoader.PP_AND_TP_PATTERN, part=0
)
== expected
)
def test_render_default_pattern_part_wildcard(monkeypatch):
"""The glob wildcard '*' passes through as the part token for file lookup."""
_mock_pp_group(monkeypatch, 1, tp_rank=4)
assert (
ShardedStateLoader._render_default_pattern(part="*")
== "model-rank-4-part-*.safetensors"
)
def test_save_model_pp_aware_filenames(monkeypatch, tmp_path):
"""save_model writes pp/tp-aware filenames for pipeline-parallel models."""
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_rank=2)
saved: list[str] = []
def _capture(tensors, path, **kwargs):
saved.append(os.path.basename(path))
monkeypatch.setattr("safetensors.torch.save_file", _capture)
model = SimpleNamespace(
state_dict=lambda: {"w": torch.zeros(4, dtype=torch.float32)}
)
ShardedStateLoader.save_model(model, str(tmp_path))
assert saved == ["model-pp-1-tp-2-part-0.safetensors"]
def test_save_model_part_split(monkeypatch, tmp_path):
"""save_model rolls over to a new part when max_size is exceeded."""
_mock_pp_group(monkeypatch, 1, tp_rank=0)
saved: list[str] = []
def _capture(tensors, path, **kwargs):
saved.append(os.path.basename(path))
monkeypatch.setattr("safetensors.torch.save_file", _capture)
# Two float32[4] tensors (16 B each); max_size=20 fits one but not two.
model = SimpleNamespace(
state_dict=lambda: {
"a": torch.zeros(4, dtype=torch.float32),
"b": torch.zeros(4, dtype=torch.float32),
}
)
ShardedStateLoader.save_model(model, str(tmp_path), max_size=20)
assert saved == [
"model-rank-0-part-0.safetensors",
"model-rank-0-part-1.safetensors",
]
def test_save_model_rejects_bad_pattern_before_overwrite(monkeypatch, tmp_path):
"""An unsafe pattern is rejected before any shard file is written."""
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_rank=0, tp_world_size=2)
# With the escaped pp field this pattern renders both stages to the
# same literal name; the validation must fire before any write.
colliding = tmp_path / "model-{pp_rank}-tp-0-part-0.safetensors"
colliding.write_bytes(b"existing shard")
before = sorted(p.name for p in tmp_path.iterdir())
model = SimpleNamespace(
state_dict=lambda: {"w": torch.zeros(4, dtype=torch.float32)}
)
with pytest.raises(ValueError, match="does not embed the pipeline parallel"):
ShardedStateLoader.save_model(
model,
str(tmp_path),
pattern="model-{{pp_rank}}-tp-{tp_rank}-part-{part}.safetensors",
)
# The pre-existing shard is untouched and nothing new was written.
assert colliding.read_bytes() == b"existing shard"
assert sorted(p.name for p in tmp_path.iterdir()) == before
@pytest.fixture(scope="module")
def llama_3p2_1b_files():
input_dir = hf_api().snapshot_download(
@@ -163,3 +476,128 @@ def test_sharded_state_loader(
queue.join_thread()
assert out_before == out_after
def _run_sharded_state_round_trip(
input_dir: str,
*,
tp_size: int,
pp_size: int,
num_gpus_available: int,
) -> None:
"""Save a TP/PP sharded checkpoint, reload it, and compare outputs.
Args:
input_dir: directory containing the original (un-sharded) model.
tp_size: tensor parallel size used for both save and load.
pp_size: pipeline parallel size used for both save and load.
num_gpus_available: number of GPUs on the current machine.
"""
if num_gpus_available < tp_size * pp_size:
pytest.skip(f"Not enough GPUs for TP={tp_size}, PP={pp_size}")
weights_patterns = ("*.safetensors",)
gpu_memory_utilization = 0.8
ctx = mp.get_context("spawn")
# Keep batching deterministic: this test compares exact greedy outputs
# across separate engine processes, whose schedulers may otherwise form
# different batches depending on prompt-processing timing.
platform_args = {"max_num_seqs": 1}
parallel_args = dict(
tensor_parallel_size=tp_size,
pipeline_parallel_size=pp_size,
)
with TemporaryDirectory() as output_dir:
p = ctx.Process(
target=_run_writer,
args=(input_dir, output_dir, weights_patterns),
kwargs=dict(
**parallel_args,
gpu_memory_utilization=gpu_memory_utilization,
enforce_eager=True,
**platform_args,
),
)
p.start()
p.join()
assert p.exitcode == 0, f"writer process failed with exit code {p.exitcode}"
if pp_size > 1:
# PP checkpoints must use the pp/tp-aware naming scheme so that
# different pipeline stages write distinct files.
sharded_files = [
f for f in os.listdir(output_dir) if f.endswith(".safetensors")
]
assert sharded_files, "no sharded checkpoint files were written"
assert any(
fnmatch.fnmatch(f, "model-pp-*-tp-*-part-*.safetensors")
for f in sharded_files
), f"expected PP-aware filenames, got {sharded_files}"
queue = ctx.Queue()
p = ctx.Process(
target=_run_generate,
args=(input_dir, queue),
kwargs=dict(
enable_lora=False,
gpu_memory_utilization=gpu_memory_utilization,
**parallel_args,
**platform_args,
),
)
p.start()
# Call queue.get() before p.join() to prevent deadlock:
# If p.join() is called before queue.get() and the queue is full,
# the child process may block while writing to the queue and never
# terminate, causing the parent to wait indefinitely on p.join().
# See: https://github.com/vllm-project/vllm/pull/22371#discussion_r2257773814
out_before = queue.get()
p.join()
assert p.exitcode == 0, f"generate process failed with exit code {p.exitcode}"
queue.close()
queue.join_thread()
queue = ctx.Queue()
p = ctx.Process(
target=_run_generate,
args=(output_dir, queue),
kwargs=dict(
enable_lora=False,
gpu_memory_utilization=gpu_memory_utilization,
load_format="sharded_state",
**parallel_args,
**platform_args,
),
)
p.start()
out_after = queue.get()
p.join()
assert p.exitcode == 0, f"generate process failed with exit code {p.exitcode}"
queue.close()
queue.join_thread()
assert out_before == out_after
def test_sharded_state_loader_pp(num_gpus_available, llama_3p2_1b_files):
"""Round-trip sharded state save/load with pipeline parallelism (PP=2)."""
_run_sharded_state_round_trip(
llama_3p2_1b_files,
tp_size=1,
pp_size=2,
num_gpus_available=num_gpus_available,
)
def test_sharded_state_loader_pp_tp(num_gpus_available, llama_3p2_1b_files):
"""Round-trip sharded state save/load with pipeline + tensor parallelism
(PP=2, TP=2)."""
_run_sharded_state_round_trip(
llama_3p2_1b_files,
tp_size=2,
pp_size=2,
num_gpus_available=num_gpus_available,
)
@@ -12,12 +12,14 @@ import subprocess
import sys
import tempfile
import threading
import time
import urllib.error
import urllib.request
from dataclasses import dataclass, field
from typing import Any
import pytest
import regex as re
from vllm import SamplingParams
from vllm.assets.image import ImageAsset
@@ -81,6 +83,21 @@ class WeightCacheDaemon:
f"Weight cache daemon is not ready: HTTP {error.code}"
) from error
def wait_until_ready(self, timeout_s: float = 900) -> None:
"""Poll until READY (which covers the autotune tuners) or the preload
process dies."""
deadline = time.monotonic() + timeout_s
while True:
try:
self.check_health()
return
except (AssertionError, urllib.error.URLError):
if time.monotonic() > deadline:
raise
if self._proc is not None and self._proc.poll() is not None:
raise AssertionError("vllm preload exited before ready") from None
time.sleep(2)
def _stop(self) -> None:
assert self._proc is not None
self._proc.terminate()
@@ -99,6 +116,10 @@ class ModelCase:
images: list | None = None
llm_kwargs: dict[str, Any] = field(default_factory=dict)
daemon_args: list[str] = field(default_factory=list)
# Additionally assert the daemons tune in place before serving and write
# the on-disk FlashInfer autotune cache, and the restart engine reuses it
# (needs SM90+/FlashInfer).
check_flashinfer_cache: bool = False
def generate(
@@ -157,6 +178,9 @@ K3_CASE = ModelCase(
max_model_len=4096,
),
daemon_args=["--trust-remote-code"],
# Kimi K3 has tunable FlashInfer ops (a dense model like Qwen2.5 tunes
# nothing and upstream never writes the cache file for it).
check_flashinfer_cache=True,
)
# Qwen3.5-0.8B ships one MTP layer in the target checkpoint, so method="mtp"
@@ -186,18 +210,38 @@ QWEN_MTP_CASE = ModelCase(
[QWEN_CASE, K3_CASE, QWEN_MTP_CASE],
ids=["qwen3.5", "kimi-k3", "qwen3.5-mtp"],
)
def test_ipc_cache_cold_start_and_warm_restart(vllm_runner, case: ModelCase):
def test_ipc_cache_cold_start_and_warm_restart(
vllm_runner, case: ModelCase, tmp_path, monkeypatch, capfd
):
"""Cold start falls back to disk; warm restarts load weights via CUDA IPC.
All runs must produce outputs identical to a default-loader baseline. The
warm runs disable the disk fallback, so they only pass if the weights
really came from the daemon — for the MTP case, both the target's and the
draft's daemon groups.
Cases with ``check_flashinfer_cache`` additionally cover in-daemon
autotune (default-on with `--enable-flashinfer-autotune`): each daemon
tunes before binding its socket and must write the on-disk FlashInfer
autotune cache, and the restart engine must load tactics from it instead
of re-profiling. The daemon's cache file is keyed with the
OPENAI_API_SERVER batch defaults while these engines run under LLM_CLASS,
so they never hit the daemon's file by design; reuse is proven against
the file the earlier in-process engines wrote.
"""
if not current_platform.is_cuda_alike():
pytest.skip("Weight cache IPC sharing requires CUDA or ROCm")
if case is K3_CASE and not current_platform.is_device_capability_family(100):
pytest.skip("Kimi K3 IPC weight cache requires an SM100 MXFP4 backend")
if case.check_flashinfer_cache:
from vllm.utils.flashinfer import has_flashinfer
if not (has_flashinfer() and current_platform.has_device_capability(90)):
pytest.skip("FlashInfer autotune requires FlashInfer and SM90+")
# Isolate the on-disk autotune cache. The engines are in-process
# (envs read lazily); the daemon subprocesses spawn later and
# inherit it.
monkeypatch.setenv("VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR", str(tmp_path))
# Baseline: plain disk loading with the default loader.
baseline_outputs = generate(
@@ -212,15 +256,40 @@ def test_ipc_cache_cold_start_and_warm_restart(vllm_runner, case: ModelCase):
with tempfile.TemporaryDirectory(prefix="vllm_ipc_empty_") as empty_socket_dir:
cold_outputs = generate(vllm_runner, case, empty_socket_dir, fallback=True)
if case.check_flashinfer_cache:
# The baseline engine already wrote its own (LLM_CLASS-keyed) table.
engine_cache_files = set(tmp_path.rglob("autotune_configs*.json"))
with WeightCacheDaemon(
case.model,
tp_size=1,
extra_args=case.daemon_args,
) as d:
warm_outputs = generate(vllm_runner, case, d.socket_dir, fallback=False)
d.check_health()
d.wait_until_ready()
if case.check_flashinfer_cache:
# Each daemon tunes before its ready report; exactly one new file
# (their OPENAI_API_SERVER-keyed table) must appear.
tuner_files = set(tmp_path.rglob("autotune_configs*.json"))
tuner_files -= engine_cache_files
assert len(tuner_files) == 1, (
f"expected the tuner's tuned table, got {tuner_files}"
)
# Warm restart: a second engine lifetime against the same daemon.
if case.check_flashinfer_cache:
capfd.readouterr() # drain: the assertion reads the increment
restart_outputs = generate(vllm_runner, case, d.socket_dir, fallback=False)
if case.check_flashinfer_cache:
# The restart's EngineCore subprocess inherits pytest's fds; the
# flashinfer autotuner logs "[Autotuner]: Loaded N configs from
# <path>" (INFO, not once-deduplicated) on a disk cache hit.
out = capfd.readouterr()
m = re.search(
r"\[Autotuner\]: Loaded (\d+) configs from", out.out + out.err
)
assert m and int(m.group(1)) > 0, (
"restart engine did not load the on-disk autotune cache"
)
assert cold_outputs == baseline_outputs
assert warm_outputs == baseline_outputs
@@ -0,0 +1,71 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
from vllm.model_executor.warmup import attention_warmup
from vllm.model_executor.warmup.attention_warmup import mixed_batch_attention_warmup
def _worker(*, use_v2: bool, max_num_batched_tokens: int, max_model_len: int = 4096):
calls = []
worker = SimpleNamespace(
model_runner=SimpleNamespace(_dummy_run=lambda **kw: calls.append(kw)),
use_v2_model_runner=use_v2,
scheduler_config=SimpleNamespace(max_num_batched_tokens=max_num_batched_tokens),
model_config=SimpleNamespace(max_model_len=max_model_len),
execute_model=object(),
sample_tokens=object(),
)
return worker, calls
@pytest.mark.parametrize(
("max_num_batched_tokens", "max_model_len", "expected"),
[(8192, 4096, 16), (6, 4096, 6), (8192, 8, 8)],
)
def test_v1_runs_mixed_dummy_batch_within_limits(
max_num_batched_tokens, max_model_len, expected
):
worker, calls = _worker(
use_v2=False,
max_num_batched_tokens=max_num_batched_tokens,
max_model_len=max_model_len,
)
mixed_batch_attention_warmup(worker)
assert calls == [
dict(
num_tokens=expected,
skip_eplb=True,
is_profile=True,
force_attention=True,
create_mixed_batch=True,
)
]
@pytest.mark.parametrize(("max_num_batched_tokens", "expected"), [(8192, 16), (6, 6)])
@pytest.mark.parametrize("ran", [True, False])
def test_v2_runs_mixed_prefill_decode_step(
monkeypatch, max_num_batched_tokens, expected, ran
):
worker, dummy_calls = _worker(
use_v2=True, max_num_batched_tokens=max_num_batched_tokens
)
mixed_calls = []
def fake_mixed_warmup(runner, execute_model, sample_tokens, num_tokens):
mixed_calls.append((runner, execute_model, sample_tokens, num_tokens))
return ran
monkeypatch.setattr(
attention_warmup, "run_mixed_prefill_decode_warmup", fake_mixed_warmup
)
mixed_batch_attention_warmup(worker)
assert mixed_calls == [
(worker.model_runner, worker.execute_model, worker.sample_tokens, expected)
]
# No dummy-run fallback when the mixed step is skipped.
assert dummy_calls == []
+54
View File
@@ -25,6 +25,60 @@ def test_b12x_warmup_token_counts_cover_serving_regimes() -> None:
) == (1, 2, 8, 128, 2048)
@pytest.mark.parametrize("is_embedding", [False, True])
def test_b12x_block_quant_warmup_covers_each_layers_plans(monkeypatch, is_embedding):
from vllm.model_executor.layers.quantization.modelopt_block_quant import (
ModelOptBlockQuantLinearMethod,
)
from vllm.model_executor.warmup.b12x_warmup import _collect_warmup_units
method = object.__new__(ModelOptBlockQuantLinearMethod)
method.codec = "q8_0"
method.is_embedding = is_embedding
model = torch.nn.ModuleList([torch.nn.Module(), torch.nn.Module()])
calls = []
monkeypatch.setattr(
method, "apply", lambda layer, x: calls.append((layer, x.shape, x.dtype))
)
monkeypatch.setattr(
method,
"embedding",
lambda layer, ids: calls.append((layer, ids.shape, ids.dtype)),
)
for index, layer in enumerate(model):
layer.b12x_warmup_provider = method
if is_embedding:
layer.weight = torch.nn.Parameter(
torch.empty((8, 8, 34), dtype=torch.uint8), requires_grad=False
)
layer.b12x_embedding_plans = {
dtype: SimpleNamespace(
handle=2 * index + offset, query=SimpleNamespace(max_rows=32)
)
for offset, dtype in enumerate((torch.int32, torch.int64))
}
else:
layer.b12x_block_weight = SimpleNamespace(
in_features=256, values=torch.empty(1)
)
layer.b12x_block_plan = SimpleNamespace(
handle=index, token_counts=(1, 4, 32)
)
units = list(_collect_warmup_units(model, (1, 17, 32), torch.bfloat16))
assert len(units) == 2
for unit in units:
unit.compile()
expected = (
[((32,), torch.int32), ((32,), torch.int64)]
if is_embedding
else [((rows, 256), torch.bfloat16) for rows in (1, 4, 17, 32)]
)
assert calls == [
(layer, shape, dtype) for layer in model for shape, dtype in expected
]
@pytest.mark.parametrize(
("kernel_cls", "module_name", "call_name", "layer", "name"),
[
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import sys
from types import SimpleNamespace
import pytest
@@ -88,3 +89,53 @@ def test_kernel_registers_itself_as_warmup_provider(is_bmm) -> None:
provider = getattr(layer, "deep_gemm_warmup_provider", None)
assert provider is (None if is_bmm else kernel)
def test_mega_moe_predicate_matches_only_native_mega_moe_layers(monkeypatch) -> None:
"""Without the MegaMoE model modules loaded, nothing matches and nothing is
imported; with them loaded, only DeepSeek-V4 and Kimi K3 MegaMoE layers match."""
name = "vllm.models.deepseek_v4.nvidia.model"
kimi_name = "vllm.models.kimi_k3.nvidia.model"
monkeypatch.delitem(sys.modules, name, raising=False)
monkeypatch.delitem(sys.modules, kimi_name, raising=False)
other = SimpleNamespace(use_native_mega_moe=True, use_mega_moe=True)
assert not deep_gemm_warmup._mega_moe_may_use_deep_gemm(other)
assert name not in sys.modules and kimi_name not in sys.modules
moe_cls = type("DeepseekV4MoE", (), {})
monkeypatch.setitem(sys.modules, name, SimpleNamespace(DeepseekV4MoE=moe_cls))
native, fused = moe_cls(), moe_cls()
native.use_native_mega_moe, fused.use_native_mega_moe = True, False
assert deep_gemm_warmup._mega_moe_may_use_deep_gemm(native)
assert not deep_gemm_warmup._mega_moe_may_use_deep_gemm(fused)
assert not deep_gemm_warmup._mega_moe_may_use_deep_gemm(other)
kimi_cls = type("KimiMoE", (), {})
monkeypatch.setitem(sys.modules, kimi_name, SimpleNamespace(KimiMoE=kimi_cls))
kimi_mega, kimi_fused = kimi_cls(), kimi_cls()
kimi_mega.use_mega_moe, kimi_fused.use_mega_moe = True, False
assert deep_gemm_warmup._mega_moe_may_use_deep_gemm(kimi_mega)
assert not deep_gemm_warmup._mega_moe_may_use_deep_gemm(kimi_fused)
def test_mega_moe_m_values_reach_mega_gate(monkeypatch) -> None:
"""Each bf16_mega_gate config is warmed with an M above the GateLinear
threshold (1 for 128 experts), so the forward actually reaches the kernel."""
dg = SimpleNamespace(
get_bf16_mega_gate_config=lambda m, *_: {"num_sms": -(-m // 16)},
get_block_m_for_mega_moe=lambda *_: 16,
)
monkeypatch.setattr(deep_gemm_warmup, "_import_deep_gemm", lambda: dg)
monkeypatch.setattr(deep_gemm_warmup, "_mega_moe_uses_mega_gate", lambda _: True)
monkeypatch.setattr(
deep_gemm_warmup, "get_ep_group", lambda: SimpleNamespace(world_size=8)
)
buffer = SimpleNamespace(num_max_tokens_per_rank=64)
module = SimpleNamespace(
gate=SimpleNamespace(input_size=4096, output_size=128),
experts=SimpleNamespace(
top_k=6, num_experts=128, get_symm_buffer=lambda: buffer
),
use_sequence_parallel=False,
)
assert deep_gemm_warmup._get_fp8_fp4_mega_moe_m_values(module, 40) == [16, 32, 40]
@@ -386,3 +386,34 @@ def test_rejected_cache_retunes_every_rank_in_its_tuning_group(autotune_run, pp,
rerun.assert_collectives_match()
assert set(rerun.profile_groups) == set(range(tp))
assert {rank for rank, _, _ in rerun.saves} == set(range(tp))
def test_autotune_cache_key_normalizes_kv_init_fields():
"""mamba_block_size/kv_cache_layout only resolve at engine KV-init time;
they must not feed the autotune cache key, or nothing can compute the key
before startup (the daemon tunes before any KV init). block_size must
stay in the key: the daemon replays the engine's block-size resolution
before tuning, and different block sizes may select different ops."""
from vllm.config.cache import CacheConfig
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
_normalize_cache_config_for_hash,
)
fresh = CacheConfig()
# Simulate the engine's KV-init write-backs (hybrid model values).
mutated = CacheConfig()
mutated.mamba_block_size = 512
mutated.kv_cache_layout = "LBHNC"
# Negative control: the write-backs are hashed factors.
assert mutated.compute_hash() != fresh.compute_hash()
normalized = _normalize_cache_config_for_hash(mutated)
assert normalized.compute_hash() == fresh.compute_hash()
# Factory state is returned untouched.
assert _normalize_cache_config_for_hash(fresh) is fresh
# block_size stays significant (the daemon keys on the resolved value).
resolved = CacheConfig(block_size=64)
assert (
_normalize_cache_config_for_hash(resolved).compute_hash()
!= fresh.compute_hash()
)
@@ -553,12 +553,10 @@ def test_compute_slot_mapping_uses_named_launcher_inputs(monkeypatch) -> None:
owner = ComputeSlotMappingKernel()
compile_key = owner.CompileKey(
kv_cache_block_size=16,
blocks_per_kv_block=1,
total_cp_world_size=2,
total_cp_rank=1,
cp_kv_cache_interleave_size=1,
block_table_stride=128,
block_size=16,
)
launches: list[tuple[Any, ...]] = []
@@ -574,7 +572,6 @@ def test_compute_slot_mapping_uses_named_launcher_inputs(monkeypatch) -> None:
assert inputs["block_table_stride"] == 128
assert kwargs == {
"KV_CACHE_BLOCK_SIZE": 16,
"BLOCKS_PER_KV_BLOCK": 1,
"TOTAL_CP_WORLD_SIZE": 2,
"TOTAL_CP_RANK": 1,
"CP_KV_CACHE_INTERLEAVE_SIZE": 1,
+114
View File
@@ -6,6 +6,9 @@ import torch
from vllm.model_executor.models.minicpmv import MiniCPMV2_6
from vllm.model_executor.models.minicpmv4_6 import (
MiniCPMV4_6ForConditionalGeneration,
_expand_flags,
_select_vision_items,
_vision_downsample_mode,
)
pytestmark = pytest.mark.skip_global_cleanup
@@ -54,3 +57,114 @@ def test_image_and_video_embeds_stay_in_their_own_modality():
assert torch.equal(modalities["images"]["image_embeds"], image_embeds)
assert torch.equal(modalities["videos"]["image_embeds"], video_embeds)
class TestPerItemMergerFlags:
"""The merger flag is per request but items from several requests can be
batched together, so it has to be applied per item."""
def test_expand_flags_broadcasts_a_single_value(self):
assert _expand_flags(None, 3) == [None, None, None]
assert _expand_flags(torch.tensor([True]), 3) == [True, True, True]
assert _expand_flags(True, 2) == [True, True]
def test_expand_flags_keeps_per_item_values(self):
flags = [torch.tensor([True]), torch.tensor([False])]
assert _expand_flags(flags, 2) == [True, False]
def test_expand_flags_falls_back_on_a_shape_mismatch(self):
# Too many values to map onto the items: keep one decision for the
# batch rather than failing the request.
assert _expand_flags([True, False, True], 2) == [True, True]
def test_vision_downsample_mode(self):
assert _vision_downsample_mode(None) is None
assert _vision_downsample_mode(True) == "16x"
assert _vision_downsample_mode(False) == "4x"
def test_select_vision_items_slices_by_item(self):
# 3 items holding 2, 1 and 2 slices; one channel, one patch, width 4.
pixel_values = [torch.full((1, 1, 4), float(i)) for i in range(5)]
tgt_sizes = torch.arange(10, dtype=torch.long).reshape(5, 2)
image_input = {
"pixel_values": pixel_values,
"tgt_sizes": tgt_sizes,
"num_slices": torch.tensor([2, 1, 2]),
}
selected = _select_vision_items(image_input, [2], [2, 1, 2])
assert len(selected["pixel_values"]) == 2
assert bool(selected["pixel_values"][0][0, 0, 0] == 3.0)
assert bool(selected["pixel_values"][1][0, 0, 0] == 4.0)
assert selected["num_slices"].tolist() == [2]
assert torch.equal(selected["tgt_sizes"], tgt_sizes[3:5])
def test_select_vision_items_preserves_order(self):
pixel_values = [torch.full((1, 1, 4), float(i)) for i in range(3)]
image_input = {
"pixel_values": pixel_values,
"tgt_sizes": torch.zeros(3, 2, dtype=torch.long),
"num_slices": torch.tensor([1, 1, 1]),
}
selected = _select_vision_items(image_input, [2, 0], [1, 1, 1])
assert selected["num_slices"].tolist() == [1, 1]
assert bool(selected["pixel_values"][0][0, 0, 0] == 2.0)
assert bool(selected["pixel_values"][1][0, 0, 0] == 0.0)
def test_process_vision_input_applies_flags_per_item(self):
calls: list[str | None] = []
def fake_get_vision_hidden_states(data, downsample_mode=None):
calls.append(downsample_mode)
tag = 1.0 if downsample_mode == "16x" else 0.0
return [torch.full((1,), tag) for _ in data["pixel_values"]]
model = object.__new__(MiniCPMV4_6ForConditionalGeneration)
model.get_vision_hidden_states = fake_get_vision_hidden_states
image_input = {
"type": "pixel_values",
"pixel_values": [torch.zeros(1, 1, 4) for _ in range(3)],
"tgt_sizes": torch.zeros(3, 2, dtype=torch.long),
"num_slices": torch.tensor([1, 1, 1]),
}
result = model._process_vision_input(
image_input,
use_vit_merger=[
torch.tensor([True]),
torch.tensor([False]),
torch.tensor([True]),
],
)
# Each item keeps its own mode even though they share one call.
assert [float(r[0]) for r in result] == [1.0, 0.0, 1.0]
# Mixed flags are encoded as separate groups, not one collapsed batch.
assert calls == ["16x", "4x"]
def test_process_vision_input_uses_one_group_for_uniform_flags(self):
calls: list[str | None] = []
def fake_get_vision_hidden_states(data, downsample_mode=None):
calls.append(downsample_mode)
return [torch.zeros(1) for _ in data["pixel_values"]]
model = object.__new__(MiniCPMV4_6ForConditionalGeneration)
model.get_vision_hidden_states = fake_get_vision_hidden_states
image_input = {
"type": "pixel_values",
"pixel_values": [torch.zeros(1, 1, 4) for _ in range(2)],
"tgt_sizes": torch.zeros(2, 2, dtype=torch.long),
"num_slices": torch.tensor([1, 1]),
}
model._process_vision_input(image_input, use_vit_merger=torch.tensor([True]))
# The common case still makes a single encoder call.
assert calls == ["16x"]
+310
View File
@@ -0,0 +1,310 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.model_executor.models.minicpmv4_6 import _stack_vit_merger_qkv
from vllm.model_executor.models.minicpmv4_7 import (
MiniCPMV4_7MultiModalProcessor,
_compute_canvas_single,
build_image_bounds,
canvas_rope_delta,
)
pytestmark = pytest.mark.skip_global_cleanup
def test_canvas_mrope_sizes_survive_prefix_cache_stripping():
"""A cached image must keep `tgt_sizes`, or canvas M-RoPE falls back.
`strip_covered_mm_data` drops the payload of a prefix-cache-covered item and
keeps only CPU-side metadata. Canvas M-RoPE places slices from `tgt_sizes`,
so 4.7 marks it `keep_on_cpu` while the shared 4.6 config stays as it was.
"""
from vllm.model_executor.models.minicpmv4_6 import _minicpmv4_6_field_config
shared = _minicpmv4_6_field_config({})
assert shared["tgt_sizes"].field.keep_on_cpu is False
assert shared["video_tgt_sizes"].field.keep_on_cpu is False
processor = MiniCPMV4_7MultiModalProcessor.__new__(MiniCPMV4_7MultiModalProcessor)
fields = processor._get_mm_fields_config({}, {})
assert fields["tgt_sizes"].field.keep_on_cpu is True
assert fields["video_tgt_sizes"].field.keep_on_cpu is True
# Other fields keep their previous setting.
assert fields["pixel_values"].field.keep_on_cpu is False
assert fields["video_image_sizes"].field.keep_on_cpu is True
def test_modality_mm_kwargs_resolution():
"""Flat keys reach both modalities; scopes stay scoped, and win on conflict.
`_merge_and_resolve_mm_processor_kwargs` routes a flat key into every scope
whose HF processor declares it, so reading is a plain scope lookup. The
routing itself is upstream's, tested here against the MiniCPM-V schema to
pin the precedence rule (`scoped > flat`).
"""
from vllm.model_executor.models.minicpmv4_6 import (
_flat_processor_kwargs,
_scoped_mm_kwarg,
)
from vllm.multimodal.processing.context import _resolve_mm_processor_kwargs
# What MiniCPMV4_6{Image,Video}ProcessorKwargs declare.
processor_keys = {"downsample_mode", "max_slice_nums"}
schema = {
"text_kwargs": set(),
"images_kwargs": set(processor_keys),
"videos_kwargs": set(processor_keys),
"audio_kwargs": set(),
}
def read(mm_kwargs, modality, key):
merged = _resolve_mm_processor_kwargs(dict(mm_kwargs), schema)
return _scoped_mm_kwarg(merged, modality, key)
assert read({}, "image", "downsample_mode") is None
# A flat key is routed into both scopes.
assert read({"downsample_mode": "4x"}, "image", "downsample_mode") == "4x"
assert read({"downsample_mode": "4x"}, "video", "downsample_mode") == "4x"
# A scoped key stays in its scope.
assert (
read({"images_kwargs": {"downsample_mode": "16x"}}, "image", "downsample_mode")
== "16x"
)
assert (
read({"images_kwargs": {"downsample_mode": "16x"}}, "video", "downsample_mode")
is None
)
videos_only = {"videos_kwargs": {"max_slice_nums": 1}}
assert read(videos_only, "video", "max_slice_nums") == 1
# Scoped to videos, so image must not see it.
assert read(videos_only, "image", "max_slice_nums") is None
# An explicit scope wins over the flat key it is merged with.
assert (
read(
{"downsample_mode": "16x", "images_kwargs": {"downsample_mode": "4x"}},
"image",
"downsample_mode",
)
== "4x"
)
# A non-mapping scope is ignored rather than raising.
assert read({"images_kwargs": None}, "image", "downsample_mode") is None
# `_flat_processor_kwargs` drops the scoped keys and pins the mode.
flat = _flat_processor_kwargs(
{
"downsample_mode": "4x",
"images_kwargs": {"a": 1},
"videos_kwargs": {"b": 2},
"audio_kwargs": {"c": 3},
},
"16x",
)
assert flat == {"downsample_mode": "16x"}
def test_vit_merger_qkv_is_stacked_before_load():
mapped = list(
_stack_vit_merger_qkv(
[
(
"model.vision_tower.vit_merger.self_attn.q_proj.weight",
torch.zeros(1),
),
(
"model.vision_tower.vit_merger.self_attn.k_proj.weight",
torch.zeros(1),
),
(
"model.vision_tower.vit_merger.self_attn.v_proj.weight",
torch.zeros(1),
),
("vit_merger.self_attn.qkv_proj.weight", torch.zeros(1)),
]
)
)
names = [name for name, _ in mapped]
shards = [getattr(tensor, "shard_id", None) for _, tensor in mapped]
assert names == [
"model.vision_tower.vit_merger.self_attn.qkv_proj.weight",
"model.vision_tower.vit_merger.self_attn.qkv_proj.weight",
"model.vision_tower.vit_merger.self_attn.qkv_proj.weight",
"vit_merger.self_attn.qkv_proj.weight",
]
assert shards == ["q", "k", "v", None]
# Canvas M-RoPE goldens, cross-checked against the reference implementation in
# transformers PR 48979 (`modular_minicpmv4_7.py`). Synthetic ids keep the token
# sequences readable: the canvas math only compares ids for equality.
_IM_START = 10
_IM_END = 11
_SLICE_START = 12
_SLICE_END = 13
_NEWLINE = 14
_IMAGE_PAD = 20
_VIDEO_PAD = 21
_TEXT = 100
_CANVAS_IDS = {
"im_start_id": _IM_START,
"im_end_id": _IM_END,
"slice_start_id": _SLICE_START,
"slice_end_id": _SLICE_END,
"newline_id": _NEWLINE,
}
def _canvas_positions(tokens, target_sizes):
input_ids = torch.tensor([tokens], dtype=torch.long)
targets = torch.tensor(target_sizes, dtype=torch.long)
bounds = build_image_bounds(input_ids[0], _CANVAS_IDS)
positions = _compute_canvas_single(
input_ids[0],
torch.arange(len(tokens), dtype=torch.long),
bounds,
targets,
_CANVAS_IDS,
)
return positions.tolist(), canvas_rope_delta(positions, len(tokens))
def test_canvas_mrope_trailing_newline_stays_inside_the_canvas():
"""A newline right after `</image>` belongs to the canvas, not to the text.
The chat template joins content parts with ``\\n``, so an image followed by
text always leaves one there. The transformers reference closes a frame over
every marker including newlines; leaving it out pushes every later position
one step forward and makes `rope_delta` differ by one.
"""
tokens = [_TEXT, _IM_START, _IMAGE_PAD, _IM_END, _NEWLINE, _TEXT, _TEXT]
positions, delta = _canvas_positions(tokens, [[4, 4]])
assert positions == [
[0, 1, 1, 1, 1, 3, 4],
[0, 0, 1, 2, 1, 3, 4],
[0, 0, 1, 2, 1, 3, 4],
]
assert delta == -2
def test_canvas_mrope_thumbnail_matches_reference():
# <text> <image> PAD </image> <text>; 16x turns a 4x4 patch grid into 1 token.
tokens = [_TEXT, _IM_START, _IMAGE_PAD, _IM_END, _TEXT]
positions, delta = _canvas_positions(tokens, [[4, 4]])
assert positions == [
[0, 1, 1, 1, 3],
[0, 0, 1, 2, 3],
[0, 0, 1, 2, 3],
]
assert delta == -1
def test_canvas_mrope_slice_row_matches_reference():
# Thumbnail then a 1x2 slice row; the slices tile the thumbnail's canvas.
tokens = [
_TEXT,
_IM_START,
_IMAGE_PAD,
_IM_END,
_SLICE_START,
_IMAGE_PAD,
_SLICE_END,
_SLICE_START,
_IMAGE_PAD,
_SLICE_END,
_TEXT,
]
positions, delta = _canvas_positions(tokens, [[4, 4], [4, 4], [4, 4]])
assert positions == [
[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 4],
[0, 0, 1, 2, 1, 1, 1, 1, 1, 1, 4],
[0, 0, 1, 3, 1, 1, 1, 2, 2, 2, 4],
]
assert delta == -6
def test_canvas_mrope_resamples_thumbnail_over_canvas():
# A 8x8 patch grid is 2x2 LLM tokens, so the canvas is resampled (not arange).
tokens = [_TEXT, _IM_START] + [_IMAGE_PAD] * 4 + [_IM_END, _TEXT]
positions, delta = _canvas_positions(tokens, [[8, 8]])
assert positions == [
[0, 1, 1, 1, 1, 1, 1, 4],
[0, 0, 1, 1, 2, 2, 3, 4],
[0, 0, 1, 2, 1, 2, 3, 4],
]
assert delta == -3
def test_canvas_mrope_video_frames_share_one_canvas():
tokens = [
_TEXT,
_IM_START,
_VIDEO_PAD,
_IM_END,
_IM_START,
_VIDEO_PAD,
_IM_END,
_TEXT,
]
positions, delta = _canvas_positions(tokens, [[4, 4], [4, 4]])
assert positions == [
[0, 1, 1, 1, 3, 3, 3, 5],
[0, 0, 1, 2, 2, 3, 4, 5],
[0, 0, 1, 2, 2, 3, 4, 5],
]
assert delta == -2
def test_canvas_mrope_interleaved_image_and_video():
tokens = (
[_TEXT, _IM_START]
+ [_IMAGE_PAD] * 4
+ [_IM_END, _IM_START]
+ [_VIDEO_PAD] * 4
+ [_IM_END, _TEXT]
)
positions, delta = _canvas_positions(tokens, [[8, 8], [8, 8]])
assert positions == [
[0, 1, 1, 1, 1, 1, 1, 4, 4, 4, 4, 4, 4, 7],
[0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7],
[0, 0, 1, 2, 1, 2, 3, 3, 4, 5, 4, 5, 6, 7],
]
assert delta == -6
def test_canvas_span_is_shared_by_image_and_video():
"""A video frame and a same-geometry image must place identical offsets.
Both paths go through `_assign_canvas_span`; this pins that they cannot
drift apart now that the duplicated geometry is gone.
"""
thumb = [_IM_START] + [_IMAGE_PAD] * 4 + [_IM_END]
img_pos, _ = _canvas_positions([_TEXT, *thumb, _TEXT], [[8, 8]])
vid_pos, _ = _canvas_positions([_TEXT, *thumb, *thumb, _TEXT], [[8, 8], [8, 8]])
def offsets(pos, start):
# The <im_start> halo is one below the canvas base.
base = pos[1][start] + 1
return [[c[start + k] - base for k in range(3)] for c in pos]
assert offsets(img_pos, 0) == offsets(vid_pos, 0)
def test_canvas_mrope_delta_leaves_room_for_the_first_decoded_token():
# First decoded token must land on max_prefill_position + 1, matching
# transformers and the other native M-RoPE models.
tokens = [_TEXT, _IM_START, _IMAGE_PAD, _IM_END, _TEXT]
positions, delta = _canvas_positions(tokens, [[4, 4]])
max_position = max(max(channel) for channel in positions)
assert delta == max_position + 1 - len(tokens)
@@ -3,6 +3,7 @@
from types import SimpleNamespace
import pytest
import torch
from vllm.model_executor.warmup.qwen_vl_triton_warmup import (
@@ -61,7 +62,10 @@ def test_vision_warmup_calls_only_position_and_rotary_paths() -> None:
assert calls == []
def test_mrope_warmup_launches_triton_shapes() -> None:
@pytest.mark.parametrize(
"owners_fused, skipped", [((), False), ((True,), True), ((True, False), False)]
)
def test_mrope_warmup_launches_triton_shapes(owners_fused, skipped) -> None:
from vllm.model_executor.layers.rotary_embedding.mrope import MRotaryEmbedding
launched: list[tuple[torch.Size, torch.Size]] = []
@@ -84,7 +88,13 @@ def test_mrope_warmup_launches_triton_shapes() -> None:
def __init__(self) -> None:
super().__init__()
self.rotary_emb = FakeRope()
rope = FakeRope()
if not owners_fused:
self.rotary_emb = rope
for i, fused in enumerate(owners_fused):
owner = torch.nn.Module()
owner.rotary_emb, owner.use_fused_qk_norm_rope_gate = rope, fused
self.add_module(f"attn{i}", owner)
def get_mrope_input_positions(self, input_tokens, mm_features):
raise AssertionError("warmup must not build live M-RoPE positions")
@@ -99,11 +109,8 @@ def test_mrope_warmup_launches_triton_shapes() -> None:
device=torch.device("cpu"),
)
_warm_mrope(runner, FakeMropeModel())
assert [shape for shape, _ in launched] == [
torch.Size((3, 1)),
torch.Size((3, 2)),
torch.Size((3, 16)),
]
expected = [torch.Size((3, n)) for n in (1, 2, 16)]
assert [shape for shape, _ in launched] == ([] if skipped else expected)
def test_mrope_warmup_skips_models_without_supports_mrope() -> None:
+159
View File
@@ -6,13 +6,108 @@ from unittest.mock import Mock
import huggingface_hub.constants
import pytest
import torch
from huggingface_hub.utils import LocalEntryNotFoundError
import vllm.model_executor.model_loader.weight_utils as weight_utils
from vllm.model_executor.model_loader.weight_utils import (
download_weights_from_hf,
drop_checkpoint_cache,
initialize_single_dummy_weight,
load_attention_sink,
maybe_remap_kv_scale_name,
)
from vllm.platforms import current_platform
@pytest.mark.parametrize(
"checkpoint_heads,runtime_heads,tp_size",
[(8, 4, 2), (8, 8, 1), (8, 6, 2), (16, 12, 4)],
)
def test_attention_sink_loader_shapes(
monkeypatch, checkpoint_heads, runtime_heads, tp_size
):
"""Each rank loads its checkpoint heads and disables only padded heads."""
monkeypatch.setattr(
weight_utils, "get_tensor_model_parallel_world_size", lambda: tp_size
)
checkpoint = torch.arange(checkpoint_heads, dtype=torch.float32)
for tp_rank in range(tp_size):
monkeypatch.setattr(
weight_utils, "get_tensor_model_parallel_rank", lambda rank=tp_rank: rank
)
param = torch.nn.Parameter(torch.zeros(runtime_heads), requires_grad=False)
load_attention_sink(param, checkpoint)
local_heads = checkpoint.chunk(tp_size)[tp_rank]
expected = torch.cat(
(
local_heads,
torch.full((runtime_heads - local_heads.numel(),), -torch.inf),
)
)
assert torch.equal(param, expected)
@pytest.mark.parametrize(
"checkpoint_shape,runtime_shape,error",
[
((2, 4), (4,), RuntimeError),
((8,), (2, 2), AssertionError),
((7,), (4,), ValueError),
((8,), (3,), AssertionError),
],
)
def test_attention_sink_loader_rejects_invalid_shapes(
monkeypatch, checkpoint_shape, runtime_shape, error
):
"""Invalid sink shapes fail before changing the runtime parameter."""
monkeypatch.setattr(weight_utils, "get_tensor_model_parallel_world_size", lambda: 2)
monkeypatch.setattr(weight_utils, "get_tensor_model_parallel_rank", lambda: 0)
param = torch.nn.Parameter(torch.ones(runtime_shape), requires_grad=False)
with pytest.raises(error):
load_attention_sink(param, torch.zeros(checkpoint_shape))
assert torch.equal(param, torch.ones(runtime_shape))
@pytest.mark.parametrize("param_dtype", [torch.bfloat16, torch.float32])
def test_attention_sink_loader_keeps_param_dtype(monkeypatch, param_dtype):
"""A checkpoint dtype different from the runtime parameter is cast, not copied."""
monkeypatch.setattr(weight_utils, "get_tensor_model_parallel_world_size", lambda: 1)
monkeypatch.setattr(weight_utils, "get_tensor_model_parallel_rank", lambda: 0)
param = torch.nn.Parameter(torch.zeros(4, dtype=param_dtype), requires_grad=False)
load_attention_sink(param, torch.arange(4, dtype=torch.float32))
assert param.dtype == param_dtype
assert torch.equal(param.float(), torch.arange(4, dtype=torch.float32))
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires a GPU")
def test_attention_sink_loader_accepts_cpu_checkpoint(monkeypatch):
"""Sinks are normally loaded from CPU checkpoints into device parameters."""
monkeypatch.setattr(weight_utils, "get_tensor_model_parallel_world_size", lambda: 2)
monkeypatch.setattr(weight_utils, "get_tensor_model_parallel_rank", lambda: 1)
param = torch.nn.Parameter(
torch.full((6,), 123.0, device="cuda"), requires_grad=False
)
# The test suite sets the default device, so pin the checkpoint to CPU.
checkpoint = torch.arange(8, dtype=torch.float32, device="cpu")
load_attention_sink(param, checkpoint)
assert param.device.type == "cuda"
expected = torch.arange(4, 8, dtype=torch.float32, device="cpu")
assert torch.equal(param[:4].cpu(), expected)
assert torch.isinf(param[4:]).all() and (param[4:] < 0).all()
@pytest.mark.parametrize(
"num_heads,dtype", [(4, torch.float32), (8, torch.bfloat16), (6, None)]
)
def test_make_attention_sink_registers_loader_and_padding(num_heads, dtype):
"""The factory is the single place that creates and registers a sink."""
sink = weight_utils.make_attention_sink(num_heads, dtype)
assert sink.shape == (num_heads,)
assert sink.dtype == (dtype or torch.get_default_dtype())
assert sink.requires_grad is False
assert torch.isinf(sink).all() and (sink < 0).all()
assert sink.weight_loader is load_attention_sink
def test_download_weights_from_hf():
@@ -307,5 +402,69 @@ def test_drop_checkpoint_cache_releases_local_weight_files(
fadvise.assert_called_once()
@pytest.mark.parametrize("dtype", [torch.float8_e4m3fn, torch.float8_e5m2])
def test_dummy_fp8_weight_is_not_degenerate(dtype):
"""+-1e-3 underflows in FP8, so 8-bit floats default to +-1."""
param = torch.empty(4096, dtype=dtype)
initialize_single_dummy_weight(param)
values = param.float()
assert values.isfinite().all()
assert values.abs().max() <= 1.0
assert (values != 0).float().mean() > 0.99
@pytest.mark.parametrize(
("dtype", "name"),
[(torch.float8_e8m0fnu, "weight_scale_inv"), (torch.uint8, "w13_weight_scale")],
)
def test_dummy_e8m0_scale_is_fixed_exponent(dtype, name):
"""E8M0 scales, including uint8-stored ones, get a finite 2**-7."""
param = torch.empty(64, dtype=dtype)
initialize_single_dummy_weight(param, name=name)
assert (param.view(torch.uint8) == 120).all()
@pytest.mark.parametrize("dtype", [torch.uint8, torch.int8, torch.float4_e2m1fn_x2])
def test_dummy_packed_weight_gets_random_bytes(dtype):
"""Packed weights are filled deterministically instead of left uninitialized."""
# Some backends can't fill packed FP4 directly, so view zeroed bytes.
params = [torch.zeros(4096, dtype=torch.uint8).view(dtype) for _ in range(2)]
for param in params:
initialize_single_dummy_weight(param, name="w13_weight")
raw = params[0].view(torch.uint8)
assert raw.unique().numel() > 250
assert torch.equal(raw, params[1].view(torch.uint8))
@pytest.mark.skipif(current_platform.is_rocm(), reason="ROCm zeroes integer params")
def test_dummy_wide_integer_param_is_untouched():
"""Wider integer params may be indices, so random values are unsafe."""
param = torch.arange(16, dtype=torch.int32)
initialize_single_dummy_weight(param, name="g_idx")
assert torch.equal(param, torch.arange(16, dtype=torch.int32))
def test_dummy_norm_weights_are_neutral(default_vllm_config):
"""Dummy norms stay identity-like, keeping activations at a realistic scale."""
from vllm.model_executor.layers.layernorm import (
GemmaRMSNorm,
LayerNorm,
RMSNorm,
RMSNormGated,
)
expected = {
RMSNorm(16): {"weight": 1.0},
GemmaRMSNorm(16): {"weight": 0.0},
LayerNorm(16): {"weight": 1.0, "bias": 0.0},
RMSNormGated(16): {"weight": 1.0},
}
for norm, values in expected.items():
for name, param in norm.named_parameters():
param.data.uniform_(-5, 5)
initialize_single_dummy_weight(param, name=name)
assert torch.all(param == values[name]), (type(norm).__name__, name)
if __name__ == "__main__":
test_download_weights_from_hf()