Merge branch 'main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -21,6 +21,7 @@ from vllm.model_executor.kernels.linear import (
|
||||
B12xMxFp4LinearKernel,
|
||||
B12xMxfp8LinearKernel,
|
||||
B12xNvFp4LinearKernel,
|
||||
B12xNvFp4W4A16LinearKernel,
|
||||
B12xTensorFP8ScaledMMLinearKernel,
|
||||
FP8ScaledMMLinearLayerConfig,
|
||||
Mxfp8LinearLayerConfig,
|
||||
@@ -782,7 +783,7 @@ def test_b12x_nvfp4_can_implement_supported_config() -> None:
|
||||
assert reason is None
|
||||
|
||||
|
||||
def test_b12x_backend_preserves_w4a16_fallback(monkeypatch) -> None:
|
||||
def test_b12x_backend_selects_w4a16(monkeypatch) -> None:
|
||||
import vllm.model_executor.kernels.linear as linear_mod
|
||||
|
||||
monkeypatch.setattr(linear_mod.current_platform, "_enum", PlatformEnum.CUDA)
|
||||
@@ -792,17 +793,17 @@ def test_b12x_backend_preserves_w4a16_fallback(monkeypatch) -> None:
|
||||
monkeypatch.setitem(
|
||||
linear_mod._POSSIBLE_NVFP4_KERNELS,
|
||||
PlatformEnum.CUDA,
|
||||
[B12xNvFp4LinearKernel, MarlinNvFp4LinearKernel],
|
||||
[B12xNvFp4LinearKernel, B12xNvFp4W4A16LinearKernel, MarlinNvFp4LinearKernel],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
MarlinNvFp4LinearKernel,
|
||||
B12xNvFp4W4A16LinearKernel,
|
||||
"is_supported",
|
||||
classmethod(lambda cls, compute_capability=None: (True, None)),
|
||||
)
|
||||
|
||||
kernel = init_nvfp4_linear_kernel(use_a16=True)
|
||||
|
||||
assert isinstance(kernel, MarlinNvFp4LinearKernel)
|
||||
assert isinstance(kernel, B12xNvFp4W4A16LinearKernel)
|
||||
|
||||
|
||||
def test_b12x_nvfp4_apply_calls_native_blockscaled_gemm(monkeypatch) -> None:
|
||||
@@ -859,3 +860,227 @@ def test_b12x_nvfp4_apply_calls_native_blockscaled_gemm(monkeypatch) -> None:
|
||||
layer.alpha,
|
||||
)
|
||||
assert kwargs == {"out_dtype": torch.bfloat16}
|
||||
|
||||
|
||||
def test_b12x_nvfp4_w4a16_uses_weight_only_scale_and_a16_dispatch(monkeypatch):
|
||||
import vllm.model_executor.kernels.linear.nvfp4.b12x as b12x_mod
|
||||
|
||||
calls = []
|
||||
packed_weight = object()
|
||||
|
||||
def pack_weight(weight, scale, *, recipe, global_scale):
|
||||
assert weight is layer.weight
|
||||
assert scale is layer.weight_scale
|
||||
assert recipe == "nvfp4"
|
||||
assert global_scale is layer.weight_global_scale
|
||||
return packed_weight
|
||||
|
||||
def mm(source, weight, *, required_mode):
|
||||
assert weight is packed_weight
|
||||
assert required_mode == "a16"
|
||||
assert source.dtype == torch.bfloat16
|
||||
assert source.is_contiguous()
|
||||
calls.append(source)
|
||||
return torch.full((source.shape[0], 48), 3.0, dtype=source.dtype)
|
||||
|
||||
def reject_activation_quantization(*args, **kwargs):
|
||||
pytest.fail("W4A16 must not quantize activations")
|
||||
|
||||
monkeypatch.setattr(b12x_mod, "scaled_fp4_quant", reject_activation_quantization)
|
||||
monkeypatch.setattr(
|
||||
b12x_mod,
|
||||
"_import_b12x_blockscaled",
|
||||
lambda: types.SimpleNamespace(pack_weight=pack_weight, mm=mm),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
b12x_mod,
|
||||
"_import_b12x_intrinsics",
|
||||
lambda: types.SimpleNamespace(swizzle_block_scale=lambda scale: scale),
|
||||
)
|
||||
layer = torch.nn.Module()
|
||||
layer.weight = torch.nn.Parameter(
|
||||
torch.empty((48, 64), dtype=torch.uint8), requires_grad=False
|
||||
)
|
||||
layer.weight_scale = torch.nn.Parameter(
|
||||
torch.empty((48, 8), dtype=torch.float8_e4m3fn), requires_grad=False
|
||||
)
|
||||
layer.weight_global_scale = torch.tensor([0.125])
|
||||
kernel = object.__new__(B12xNvFp4W4A16LinearKernel)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
assert layer.b12x_warmup_provider is kernel
|
||||
kernel.get_b12x_warmup_unit(layer, (1, 8), torch.bfloat16).compile()
|
||||
assert [source.shape[0] for source in calls] == [1, 8]
|
||||
|
||||
x = torch.randn((2, 3, 256), dtype=torch.bfloat16)[..., ::2]
|
||||
output = kernel.apply_weights(layer, x, torch.ones(48, dtype=torch.bfloat16))
|
||||
torch.testing.assert_close(calls[-1], x.reshape(6, 128))
|
||||
torch.testing.assert_close(output, torch.full((2, 3, 48), 4.0, dtype=x.dtype))
|
||||
with pytest.raises(ValueError, match="requires BF16 input"):
|
||||
kernel.apply_weights(layer, x.float())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,n,k", [(2, 136, 96), (17, 256, 256)])
|
||||
@torch.inference_mode()
|
||||
def test_b12x_nvfp4_w4a16_preserves_bf16_activations_under_graph_replay(
|
||||
monkeypatch, default_vllm_config, m, n, k
|
||||
) -> None:
|
||||
import vllm.model_executor.kernels.linear.nvfp4.b12x as b12x_mod
|
||||
|
||||
supported, reason = B12xNvFp4W4A16LinearKernel.is_supported()
|
||||
if not supported:
|
||||
pytest.skip(reason)
|
||||
|
||||
def reject_activation_quantization(*args, **kwargs):
|
||||
pytest.fail("W4A16 must not quantize activations")
|
||||
|
||||
monkeypatch.setattr(b12x_mod, "scaled_fp4_quant", reject_activation_quantization)
|
||||
torch.manual_seed(17)
|
||||
codes = torch.randint(0, 16, (n, k), device="cuda")
|
||||
scales = (2.0 ** torch.randint(-2, 2, (n, k // 16), device="cuda")).to(
|
||||
torch.float8_e4m3fn
|
||||
)
|
||||
layer = torch.nn.Module()
|
||||
layer.weight = torch.nn.Parameter(
|
||||
(codes[:, ::2] | codes[:, 1::2] << 4).to(torch.uint8), requires_grad=False
|
||||
)
|
||||
layer.weight_scale = torch.nn.Parameter(scales, requires_grad=False)
|
||||
layer.weight_global_scale = torch.nn.Parameter(
|
||||
torch.tensor([0.125], device="cuda"), requires_grad=False
|
||||
)
|
||||
lut = torch.tensor(
|
||||
[0, 0.5, 1, 1.5, 2, 3, 4, 6, 0, -0.5, -1, -1.5, -2, -3, -4, -6],
|
||||
device="cuda",
|
||||
)
|
||||
decoded = lut[codes] * scales.float().repeat_interleave(16, dim=1) * 0.125
|
||||
x_storage = torch.randn(1, m, k * 2, device="cuda", dtype=torch.bfloat16)
|
||||
x = x_storage[..., ::2]
|
||||
bias = torch.randn(n, device="cuda", dtype=torch.bfloat16)
|
||||
kernel = B12xNvFp4W4A16LinearKernel(None)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
kernel.get_b12x_warmup_unit(layer, (1, 2, 4, 32), x.dtype).compile()
|
||||
run = torch.compile(
|
||||
lambda inputs: kernel.apply_weights(layer, inputs, bias),
|
||||
fullgraph=True,
|
||||
dynamic=True,
|
||||
)
|
||||
run(x)
|
||||
torch.accelerator.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
actual = run(x)
|
||||
for _ in range(2):
|
||||
x_storage.normal_()
|
||||
graph.replay()
|
||||
expected = (x.double() @ decoded.double().T).bfloat16() + bias
|
||||
torch.testing.assert_close(actual, expected, atol=0.005, rtol=0.01)
|
||||
|
||||
|
||||
def _block_weights(n, k, codec):
|
||||
block_size, block_bytes = (
|
||||
(32, 34) if codec == "q8_0" else (256, 66 if codec == "iq2_xxs" else 74)
|
||||
)
|
||||
raw = torch.randint(0, 256, (n, k // block_size, block_bytes), dtype=torch.uint8)
|
||||
raw[..., :2] = torch.full(
|
||||
(n, k // block_size, 1), 1 / 128, dtype=torch.float16
|
||||
).view(torch.uint8)
|
||||
return raw
|
||||
|
||||
|
||||
@pytest.mark.parametrize("codec", ["iq2_xs", "iq2_xxs", "q8_0"])
|
||||
@pytest.mark.parametrize("profile_first", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_b12x_block_quant_dense_compiled_graph(
|
||||
default_vllm_config, codec, profile_first
|
||||
):
|
||||
supported, reason = B12xNvFp4W4A16LinearKernel.is_supported()
|
||||
if not supported:
|
||||
pytest.skip(reason)
|
||||
from b12x.testing.iq2_xs_reference import dequantize_blocks
|
||||
|
||||
from vllm.model_executor.layers.quantization.modelopt_block_quant import (
|
||||
ModelOptBlockQuantLinearMethod,
|
||||
)
|
||||
|
||||
default_vllm_config.scheduler_config.max_num_batched_tokens = 32
|
||||
default_vllm_config.scheduler_config.max_num_scheduled_tokens = 32
|
||||
default_vllm_config.compilation_config.cudagraph_capture_sizes = [1, 4]
|
||||
method = ModelOptBlockQuantLinearMethod(codec)
|
||||
raw = _block_weights(128, 256, codec)
|
||||
decoded = dequantize_blocks(raw).bfloat16().cuda()
|
||||
layer = torch.nn.Module()
|
||||
layer.weight = torch.nn.Parameter(raw.cuda(), requires_grad=False)
|
||||
method.process_weights_after_loading(layer)
|
||||
assert layer.b12x_block_plan.prepared is None
|
||||
if profile_first:
|
||||
method.apply(layer, torch.zeros(32, 256, device="cuda", dtype=torch.bfloat16))
|
||||
layer.b12x_warmup_provider.get_b12x_warmup_unit(
|
||||
layer, (1, 4, 32), torch.bfloat16
|
||||
).compile()
|
||||
assert layer.b12x_block_plan.prepared is not None
|
||||
run = torch.compile(lambda x: method.apply(layer, x), fullgraph=True, dynamic=True)
|
||||
for rows in (1, 4, 17):
|
||||
x = torch.randn(rows, 256, device="cuda", dtype=torch.bfloat16)
|
||||
run(x)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
output = run(x)
|
||||
for _ in range(2):
|
||||
x.normal_()
|
||||
allocated = torch.accelerator.memory_allocated()
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.accelerator.memory_allocated() == allocated
|
||||
expected = (x.float() @ decoded.float().T).bfloat16()
|
||||
torch.testing.assert_close(output, expected, rtol=0.015, atol=0.03125)
|
||||
graph.reset()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("id_dtype", [torch.int32, torch.int64])
|
||||
@pytest.mark.parametrize("profile_first", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_b12x_q8_embedding_graph(default_vllm_config, id_dtype, profile_first):
|
||||
supported, reason = B12xNvFp4W4A16LinearKernel.is_supported()
|
||||
if not supported:
|
||||
pytest.skip(reason)
|
||||
from b12x.testing.q8_0_reference import dequantize_blocks
|
||||
|
||||
from vllm.model_executor.layers.quantization.modelopt_block_quant import (
|
||||
ModelOptBlockQuantLinearMethod,
|
||||
)
|
||||
|
||||
default_vllm_config.scheduler_config.max_num_batched_tokens = 32
|
||||
default_vllm_config.scheduler_config.max_num_scheduled_tokens = 32
|
||||
default_vllm_config.compilation_config.cudagraph_capture_sizes = [1, 4]
|
||||
method = ModelOptBlockQuantLinearMethod("q8_0")
|
||||
method.is_embedding = True
|
||||
raw = _block_weights(128, 256, "q8_0")
|
||||
decoded = dequantize_blocks(raw).bfloat16().cuda()
|
||||
layer = torch.nn.Module()
|
||||
layer.weight = torch.nn.Parameter(raw.cuda(), requires_grad=False)
|
||||
method.process_weights_after_loading(layer)
|
||||
assert all(plan.prepared is None for plan in layer.b12x_embedding_plans.values())
|
||||
if profile_first:
|
||||
method.embedding(layer, torch.zeros(32, device="cuda", dtype=id_dtype))
|
||||
layer.b12x_warmup_provider.get_b12x_warmup_unit(
|
||||
layer, (1, 4, 32), torch.bfloat16
|
||||
).compile()
|
||||
assert all(
|
||||
plan.prepared is not None for plan in layer.b12x_embedding_plans.values()
|
||||
)
|
||||
run = torch.compile(
|
||||
lambda x: method.embedding(layer, x), fullgraph=True, dynamic=True
|
||||
)
|
||||
for count in (7, 9, 70):
|
||||
ids = torch.arange(count, device="cuda", dtype=id_dtype)
|
||||
torch._dynamo.mark_dynamic(ids, 0)
|
||||
torch.testing.assert_close(run(ids), decoded[ids.long()], rtol=0, atol=0)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
output = run(ids)
|
||||
ids.add_(20)
|
||||
allocated = torch.accelerator.memory_allocated()
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.accelerator.memory_allocated() == allocated
|
||||
torch.testing.assert_close(output, decoded[ids.long()], rtol=0, atol=0)
|
||||
graph.reset()
|
||||
|
||||
@@ -182,7 +182,8 @@ def get_fse_test_model_config(
|
||||
head_dim=128,
|
||||
max_position_embeddings=128,
|
||||
sliding_window=None,
|
||||
compress_ratios=[1],
|
||||
layer_types=["sliding_attention"],
|
||||
compress_rates={},
|
||||
rope_theta=10000.0,
|
||||
compress_rope_theta=10000.0,
|
||||
rope_parameters={"rope_type": "default"},
|
||||
|
||||
@@ -771,6 +771,8 @@ def test_sparse_mqa_logits_prefill_matches_dense_masked(capture: bool):
|
||||
candidates = _make_candidates(lens, num_candidates, cbk).cuda()
|
||||
|
||||
# Reference: dense bf16 logits masked to the candidates, then torch.topk.
|
||||
# The logits are compressed: row r starts at K position ks[r].
|
||||
max_len = int(lens.max())
|
||||
dense = deep_gemm.fp8_fp4_mqa_logits(
|
||||
(
|
||||
q_fp.view(torch.int8).view(rows, num_heads, head_dim // 2),
|
||||
@@ -780,19 +782,13 @@ def test_sparse_mqa_logits_prefill_matches_dense_masked(capture: bool):
|
||||
weights,
|
||||
ks,
|
||||
ke,
|
||||
clean_logits=False,
|
||||
logits_dtype=torch.bfloat16,
|
||||
max_len,
|
||||
)
|
||||
cols = torch.arange(total, device="cuda")
|
||||
in_range = (cols - ks.unsqueeze(1) >= 0) & (
|
||||
cols - ks.unsqueeze(1) < lens.unsqueeze(1)
|
||||
)
|
||||
masked = dense.masked_fill(
|
||||
~(in_range & _candidate_token_mask(candidates, cbk, total, ks)),
|
||||
float("-inf"),
|
||||
).float()
|
||||
ref_abs = _reference_topk(masked, topk) # packed-workspace positions
|
||||
ref = torch.where(ref_abs >= 0, ref_abs - ks.unsqueeze(1), -1)
|
||||
in_range = torch.arange(max_len, device="cuda") < lens.unsqueeze(1)
|
||||
# A row's last candidate block may extend past max_len.
|
||||
in_cand = _candidate_token_mask(candidates, cbk, max_len + cbk)[:, :max_len]
|
||||
masked = dense.masked_fill(~(in_range & in_cand), float("-inf")).float()
|
||||
ref = _reference_topk(masked, topk)
|
||||
|
||||
out = torch.full((rows, topk), -1, dtype=torch.int32, device="cuda")
|
||||
scratch = _sparse_scratch(rows, num_candidates, topk)
|
||||
@@ -827,11 +823,11 @@ def test_sparse_mqa_logits_prefill_matches_dense_masked(capture: bool):
|
||||
graph.replay()
|
||||
else:
|
||||
kernel_metadata = run_sparse()
|
||||
_assert_same_selection(dense, out, ref, row_starts=ks)
|
||||
_assert_same_selection(dense, out, ref)
|
||||
out.fill_(-1)
|
||||
run_sparse(kernel_metadata)
|
||||
|
||||
_assert_same_selection(dense, out, ref, row_starts=ks)
|
||||
_assert_same_selection(dense, out, ref)
|
||||
|
||||
|
||||
def _build_fp4_paged_cache(req_lens: torch.Tensor, page_kv: int, head_dim: int):
|
||||
@@ -902,21 +898,22 @@ def test_sparse_mqa_logits_paged_matches_dense_masked(next_n: int, capture: bool
|
||||
candidates = _make_candidates(row_lens.flatten(), num_candidates, cbk).cuda()
|
||||
|
||||
def reference_topk():
|
||||
context_lens_2d = row_lens.view(batch, next_n)
|
||||
# SM100 paged MQA logits take varlen rows: one Q token per row.
|
||||
context_lens_2d = row_lens.view(rows, 1)
|
||||
indices = torch.arange(batch, dtype=torch.int32, device="cuda")
|
||||
indices = indices.repeat_interleave(next_n)
|
||||
sched = deep_gemm.get_paged_mqa_logits_metadata(
|
||||
context_lens_2d, page_kv, deep_gemm.get_num_sms(), None
|
||||
context_lens_2d, page_kv, deep_gemm.get_num_sms(), indices
|
||||
)
|
||||
logits = deep_gemm.fp8_fp4_paged_mqa_logits(
|
||||
(q, q_scale),
|
||||
(q.view(rows, 1, num_heads, -1), q_scale.view(rows, 1, num_heads)),
|
||||
kv_cache,
|
||||
weights,
|
||||
context_lens_2d,
|
||||
block_table,
|
||||
block_table.repeat_interleave(next_n, dim=0),
|
||||
sched,
|
||||
int(req_lens.max()),
|
||||
False,
|
||||
torch.bfloat16,
|
||||
None,
|
||||
indices,
|
||||
)
|
||||
lens_flat = row_lens.flatten()
|
||||
cols = torch.arange(logits.shape[1], device="cuda")
|
||||
|
||||
@@ -308,6 +308,7 @@ class TestTokenEmbeddingPoolerHead:
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_embed")
|
||||
out = head.forward_chunk(x, param)
|
||||
assert out is x
|
||||
assert torch.equal(out, x)
|
||||
|
||||
def test_none_chunked_prefill(self):
|
||||
|
||||
@@ -13,19 +13,22 @@ from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.config import PoolerConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.pooler import PoolingParamsUpdate
|
||||
from vllm.model_executor.layers.pooler.activations import PoolerNormalize
|
||||
from vllm.model_executor.layers.pooler.seqwise.methods import (
|
||||
CLSPool,
|
||||
LastPool,
|
||||
MeanPool,
|
||||
get_seq_pooling_method,
|
||||
)
|
||||
from vllm.model_executor.layers.pooler.special import DispatchPooler
|
||||
from vllm.model_executor.layers.pooler.special import BOSEOSFilter, DispatchPooler
|
||||
from vllm.model_executor.layers.pooler.tokwise.heads import TokenEmbeddingPoolerHead
|
||||
from vllm.model_executor.layers.pooler.tokwise.methods import (
|
||||
AllPool,
|
||||
StepPool,
|
||||
get_tok_pooling_method,
|
||||
)
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.model_executor.layers.pooler.tokwise.poolers import TokenPooler
|
||||
from vllm.pooling_params import LateChunkingParams, PoolingParams
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.v1.pool.metadata import PoolingCursor, PoolingMetadata, PoolingStates
|
||||
|
||||
@@ -606,3 +609,139 @@ class TestGetTokPoolingMethod:
|
||||
def test_unknown_raises(self):
|
||||
with pytest.raises(NotImplementedError, match="UNKNOWN"):
|
||||
get_tok_pooling_method("UNKNOWN")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("chunk_size", "dtype", "head_dtype"),
|
||||
[
|
||||
(1, torch.float32, None),
|
||||
(2, torch.float16, None),
|
||||
(2, torch.bfloat16, torch.float32),
|
||||
(7, torch.float32, None),
|
||||
(10, torch.float32, None),
|
||||
(2**63 - 1, torch.float32, None),
|
||||
],
|
||||
)
|
||||
def test_late_chunk_pool_matches_fp32_means_before_projection_and_normalization(
|
||||
chunk_size, dtype, head_dtype
|
||||
):
|
||||
# Unequal norms expose the incorrect normalize-then-average implementation.
|
||||
hidden = (torch.arange(28).reshape(7, 4) - 10).to(dtype)
|
||||
projector = torch.nn.Linear(4, 3, bias=True).to(head_dtype or dtype)
|
||||
head = TokenEmbeddingPoolerHead(head_dtype, projector, PoolerNormalize())
|
||||
pooler = TokenPooler(TestAllPool._make_all_pool(), head)
|
||||
params = PoolingParams(
|
||||
task="token_embed",
|
||||
late_chunking_params=LateChunkingParams(chunk_size=chunk_size),
|
||||
use_activation=True,
|
||||
)
|
||||
metadata = _make_metadata([7], pooling_params=[params])
|
||||
actual = pooler(hidden, metadata)[0]
|
||||
means = torch.stack(
|
||||
[hidden[a : a + chunk_size].float().mean(0) for a in range(0, 7, chunk_size)]
|
||||
).to(head_dtype or dtype)
|
||||
expected = torch.nn.functional.normalize(projector(means), p=2, dim=-1)
|
||||
torch.testing.assert_close(actual, expected)
|
||||
assert actual.dtype == (head_dtype or dtype)
|
||||
|
||||
|
||||
def test_late_chunk_pool_reduces_before_nonlinear_projector():
|
||||
hidden = torch.tensor([[2.0, 0.0], [0.0, 1.0]])
|
||||
projector = torch.nn.Tanh()
|
||||
head = TokenEmbeddingPoolerHead(projector=projector, activation=PoolerNormalize())
|
||||
params = PoolingParams(
|
||||
task="token_embed",
|
||||
late_chunking_params=LateChunkingParams(chunk_size=2),
|
||||
use_activation=True,
|
||||
)
|
||||
actual = head.forward_chunk(hidden, params)
|
||||
expected = torch.nn.functional.normalize(projector(hidden.mean(0, keepdim=True)))
|
||||
wrong_order = torch.nn.functional.normalize(projector(hidden).mean(0, keepdim=True))
|
||||
torch.testing.assert_close(actual, expected)
|
||||
assert not torch.allclose(actual, wrong_order)
|
||||
|
||||
|
||||
def test_late_chunk_pool_isolates_mixed_requests_and_owns_finished_storage():
|
||||
hidden = torch.arange(36, dtype=torch.float32).reshape(9, 4)
|
||||
params = [
|
||||
PoolingParams(
|
||||
task="token_embed", late_chunking_params=LateChunkingParams(chunk_size=2)
|
||||
),
|
||||
PoolingParams(task="token_embed"),
|
||||
PoolingParams(
|
||||
task="token_embed", late_chunking_params=LateChunkingParams(chunk_size=1)
|
||||
),
|
||||
]
|
||||
pooler = TokenPooler(
|
||||
TestAllPool._make_all_pool(async_scheduling=True), TokenEmbeddingPoolerHead()
|
||||
)
|
||||
outputs = pooler(hidden, _make_metadata([3, 4, 2], pooling_params=params))
|
||||
expected = [
|
||||
torch.stack([hidden[:2].mean(0), hidden[2]]),
|
||||
hidden[3:7].clone(),
|
||||
hidden[7:].clone(),
|
||||
]
|
||||
hidden.fill_(-999)
|
||||
for actual, reference in zip(outputs, expected):
|
||||
torch.testing.assert_close(actual, reference)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("chunk_size", [1, 2, 8])
|
||||
@pytest.mark.parametrize("embedding_size", [1, 4])
|
||||
def test_bos_eos_filter_preserves_chunk_vectors_in_mixed_batch(
|
||||
chunk_size, embedding_size
|
||||
):
|
||||
hidden = torch.arange(10 * embedding_size, dtype=torch.float32).reshape(
|
||||
10, embedding_size
|
||||
)
|
||||
pooler = BOSEOSFilter(
|
||||
TokenPooler(TestAllPool._make_all_pool(), TokenEmbeddingPoolerHead()),
|
||||
bos_token_id=101,
|
||||
eos_token_id=102,
|
||||
)
|
||||
metadata = _make_metadata(
|
||||
[5, 5],
|
||||
token_ids=[[101, 1, 2, 3, 102]] * 2,
|
||||
pooling_params=[
|
||||
PoolingParams(
|
||||
task="token_embed", late_chunking_params=LateChunkingParams(chunk_size)
|
||||
),
|
||||
PoolingParams(task="token_embed"),
|
||||
],
|
||||
)
|
||||
chunked, ordinary = pooler(hidden, metadata)
|
||||
expected = torch.stack([part.mean(0) for part in hidden[:5].split(chunk_size)])
|
||||
torch.testing.assert_close(chunked, expected)
|
||||
torch.testing.assert_close(ordinary, hidden[6:9].squeeze(-1))
|
||||
|
||||
|
||||
def test_bos_eos_filter_waits_for_complete_chunk_vectors():
|
||||
pooler = BOSEOSFilter(
|
||||
TokenPooler(
|
||||
TestAllPool._make_all_pool(chunked=True), TokenEmbeddingPoolerHead()
|
||||
),
|
||||
bos_token_id=101,
|
||||
)
|
||||
params = [
|
||||
PoolingParams(task="token_embed", late_chunking_params=LateChunkingParams(3))
|
||||
]
|
||||
first = _make_metadata(
|
||||
[4],
|
||||
token_ids=[[101, 1, 2, 102]],
|
||||
pooling_params=params,
|
||||
num_scheduled_tokens=[2],
|
||||
seq_lens=[2],
|
||||
)
|
||||
hidden = torch.arange(8, dtype=torch.float32).reshape(4, 2)
|
||||
assert pooler(hidden[:2], first) == [None]
|
||||
last = _make_metadata(
|
||||
[4],
|
||||
token_ids=[[101, 1, 2, 102]],
|
||||
pooling_params=params,
|
||||
num_scheduled_tokens=[2],
|
||||
seq_lens=[4],
|
||||
)
|
||||
last.pooling_states = first.pooling_states
|
||||
actual = pooler(hidden[2:], last)[0]
|
||||
torch.testing.assert_close(actual, torch.stack([hidden[:3].mean(0), hidden[3]]))
|
||||
assert not last.pooling_states[0].hidden_states_cache
|
||||
|
||||
@@ -2,9 +2,13 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import glob
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from safetensors.torch import save_file
|
||||
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
download_weights_from_hf,
|
||||
@@ -14,6 +18,60 @@ from vllm.model_executor.model_loader.weight_utils import (
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("weight_name", "local_ids", "collective"),
|
||||
[
|
||||
("mlp.experts.gate_proj.weight", {0}, True),
|
||||
("mlp.experts.0.gate_proj.weight", {0}, False),
|
||||
("mlp.experts.0.gate_proj.weight", set(), False),
|
||||
],
|
||||
)
|
||||
def test_instanttensor_ep_collective_loading(
|
||||
monkeypatch, tmp_path, weight_name, local_ids, collective
|
||||
):
|
||||
"""Keep fused checkpoints collective and expert decisions rank-independent."""
|
||||
from vllm.model_executor.model_loader import weight_utils
|
||||
|
||||
path = tmp_path / "model.safetensors"
|
||||
save_file({weight_name: torch.ones(1)}, str(path))
|
||||
native_open = MagicMock()
|
||||
reader = native_open.return_value.__enter__.return_value
|
||||
reader.total_tensor_size = 0
|
||||
reader.tensors.return_value = ()
|
||||
monkeypatch.setitem(
|
||||
sys.modules, "instanttensor", SimpleNamespace(safe_open=native_open)
|
||||
)
|
||||
group = SimpleNamespace(world_size=2, device_group=object())
|
||||
monkeypatch.setattr(weight_utils, "get_world_group", lambda: group)
|
||||
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
|
||||
monkeypatch.setattr(current_platform, "current_device", lambda: 0)
|
||||
list(instanttensor_weights_iterator([str(path)], False, local_expert_ids=local_ids))
|
||||
kwargs = native_open.call_args.kwargs
|
||||
assert kwargs["process_group"] is (group.device_group if collective else None)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="InstantTensor requires NVIDIA GPUs",
|
||||
)
|
||||
def test_instanttensor_ep_filter_cuda(tmp_path):
|
||||
weights = {
|
||||
"mlp.experts.0.gate_proj.weight": torch.ones(2),
|
||||
"mlp.experts.1.gate_proj.weight": torch.ones(2),
|
||||
"mlp.experts.1.gate_proj.weight_scale": torch.ones(1),
|
||||
"model.layers[0].attn.weight": torch.ones(2),
|
||||
}
|
||||
path = tmp_path / "model.safetensors"
|
||||
save_file(weights, str(path))
|
||||
loaded = dict(
|
||||
instanttensor_weights_iterator([str(path)], False, local_expert_ids={0})
|
||||
)
|
||||
del weights["mlp.experts.1.gate_proj.weight"]
|
||||
assert loaded.keys() == weights.keys()
|
||||
for name, tensor in loaded.items():
|
||||
torch.testing.assert_close(tensor.cpu(), weights[name], rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="InstantTensor requires NVIDIA GPUs",
|
||||
|
||||
+505
-1
@@ -1,13 +1,18 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
import os
|
||||
import types
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from safetensors.torch import load_file, save_file
|
||||
|
||||
from tests.utils import create_new_process_for_each_test
|
||||
from vllm import SamplingParams
|
||||
from vllm.config.load import LoadConfig
|
||||
from vllm.model_executor.model_loader import get_model_loader
|
||||
@@ -39,12 +44,130 @@ def test_get_model_loader_with_runai_flag():
|
||||
assert model_loader.__class__.__name__ == "RunaiModelStreamerLoader"
|
||||
|
||||
|
||||
def test_checkpoint_selection_hook_is_optional_for_model_detection():
|
||||
from vllm.model_executor.models.interfaces import SupportsMultiModal
|
||||
from vllm.model_executor.models.interfaces_base import (
|
||||
VllmModel,
|
||||
is_text_generation_model,
|
||||
)
|
||||
|
||||
class ModelWithoutHook(torch.nn.Module):
|
||||
def __init__(self, vllm_config, prefix=""):
|
||||
super().__init__()
|
||||
|
||||
def embed_input_ids(self, input_ids):
|
||||
return input_ids
|
||||
|
||||
def forward(self, input_ids, positions):
|
||||
return input_ids
|
||||
|
||||
def compute_logits(self, hidden_states):
|
||||
return hidden_states
|
||||
|
||||
assert is_text_generation_model(ModelWithoutHook)
|
||||
assert is_text_generation_model(ModelWithoutHook(None))
|
||||
|
||||
class ModelWithDefaultRule(ModelWithoutHook, VllmModel):
|
||||
pass
|
||||
|
||||
assert not ModelWithDefaultRule(None).is_unused_checkpoint_weight("any.weight")
|
||||
|
||||
# Structural models do not inherit the protocol's optional default hook.
|
||||
for names in ([], ["language_model"]):
|
||||
model = torch.nn.Module()
|
||||
model.language_model = ModelWithoutHook(None)
|
||||
model._language_model_names = names
|
||||
assert SupportsMultiModal.get_language_model(model) is model.language_model
|
||||
|
||||
|
||||
def test_runai_model_loader_download_files(vllm_runner):
|
||||
with vllm_runner(test_model, load_format=load_format) as llm:
|
||||
deserialized_outputs = llm.generate(prompts, sampling_params)
|
||||
assert deserialized_outputs
|
||||
|
||||
|
||||
# In-process MTP compilation must not retain GPU resources in the pytest worker.
|
||||
@create_new_process_for_each_test("spawn")
|
||||
def test_runai_mtp_real_checkpoint(vllm_runner, tmp_path, monkeypatch):
|
||||
"""Skip main-only shards without changing a real MTP model's generation."""
|
||||
from vllm.model_executor.models.qwen3_5_mtp import Qwen3_5MTP
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
from vllm.v1.metrics.reader import Counter
|
||||
|
||||
if not current_platform.is_cuda_alike():
|
||||
pytest.skip("MTP serving requires CUDA or ROCm")
|
||||
|
||||
checkpoint = tmp_path / "checkpoint"
|
||||
hf_api().snapshot_download(
|
||||
repo_id="Qwen/Qwen3.5-0.8B-Base", local_dir=str(checkpoint)
|
||||
)
|
||||
original_files = list(checkpoint.glob("*.safetensors"))
|
||||
tensors = {}
|
||||
for path in original_files:
|
||||
tensors.update(load_file(path))
|
||||
# Reshard real, unmodified weights so selection cannot pass vacuously on
|
||||
# the published single mixed file. This predicate needs no model state.
|
||||
selector = Qwen3_5MTP.__new__(Qwen3_5MTP)
|
||||
main: dict[str, torch.Tensor] = {}
|
||||
draft: dict[str, torch.Tensor] = {}
|
||||
for name, tensor in tensors.items():
|
||||
(main if selector.is_unused_checkpoint_weight(name) else draft)[name] = tensor
|
||||
assert main and draft
|
||||
files = [checkpoint / "main.safetensors", checkpoint / "draft.safetensors"]
|
||||
for path, contents in zip(files, (main, draft)):
|
||||
save_file(contents, path, metadata={"format": "pt"})
|
||||
for path in original_files:
|
||||
path.unlink()
|
||||
(checkpoint / "model.safetensors.index.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"weight_map": {
|
||||
name: path.name
|
||||
for path, data in zip(files, (main, draft))
|
||||
for name in data
|
||||
}
|
||||
}
|
||||
)
|
||||
)
|
||||
del tensors, main, draft
|
||||
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
opened = []
|
||||
real_stream = rsl.runai_safetensors_weights_iterator
|
||||
|
||||
def stream(paths, *args):
|
||||
opened.append(list(paths))
|
||||
yield from real_stream(paths, *args)
|
||||
|
||||
monkeypatch.setattr(rsl, "runai_safetensors_weights_iterator", stream)
|
||||
results = []
|
||||
for load_format in ("auto", "runai_streamer"):
|
||||
with vllm_runner(
|
||||
str(checkpoint),
|
||||
load_format=load_format,
|
||||
block_size=None,
|
||||
enable_chunked_prefill=True,
|
||||
disable_log_stats=False,
|
||||
limit_mm_per_prompt={"image": 0, "video": 0},
|
||||
speculative_config={"method": "mtp", "num_speculative_tokens": 1},
|
||||
) as runner:
|
||||
outputs = runner.generate_greedy(prompts, max_tokens=32)
|
||||
results.append(outputs)
|
||||
accepted = next(
|
||||
m.value
|
||||
for m in runner.llm.get_metrics()
|
||||
if isinstance(m, Counter)
|
||||
and m.name == "vllm:spec_decode_num_accepted_tokens"
|
||||
)
|
||||
assert accepted > 0
|
||||
assert len(opened) == 2
|
||||
assert set(opened[0]) == {str(path) for path in files}
|
||||
assert opened[1] == [str(files[1])]
|
||||
matches = sum(a == b for a, b in zip(*results))
|
||||
assert matches >= 3, "RunAI MTP generation diverged from the Default loader"
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason="Temporarily disabled due to GCS access issues. "
|
||||
"TODO: Re-enable this test once the underlying issue is resolved."
|
||||
@@ -150,7 +273,7 @@ def test_runai_invalid_extra_config_leaves_environ_untouched():
|
||||
)
|
||||
def test_runai_get_all_weights_resolves_source(model_config, expected_source):
|
||||
fake_self = types.SimpleNamespace(
|
||||
_get_weights_iterator=lambda path, revision: iter(
|
||||
_get_weights_iterator=lambda path, revision, is_unused_weight=None: iter(
|
||||
[(f"{path}@{revision}", None)]
|
||||
)
|
||||
)
|
||||
@@ -164,6 +287,387 @@ def test_runai_get_all_weights_resolves_source(model_config, expected_source):
|
||||
assert weights == [(f"{expected_source}@myrev", None)]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("selection", ["draft", "unadapted", "reject_all"])
|
||||
def test_runai_selects_shards_before_streaming(tmp_path, monkeypatch, selection):
|
||||
"""Unused shards must not reach RunAI; mixed shards retain their tensors."""
|
||||
contents = [
|
||||
{"main.weight": torch.tensor([1.0])},
|
||||
{"main.bias": torch.tensor([2.0]), "draft.weight": torch.tensor([3.0])},
|
||||
{"draft.bias": torch.tensor([4.0])},
|
||||
]
|
||||
files = []
|
||||
for i, tensors in enumerate(contents):
|
||||
path = str(tmp_path / f"part-{i}.safetensors")
|
||||
save_file(tensors, path)
|
||||
files.append(path)
|
||||
monkeypatch.setattr(rsl, "list_safetensors", lambda path: files)
|
||||
|
||||
opened = []
|
||||
real_stream = rsl.runai_safetensors_weights_iterator
|
||||
|
||||
def stream(paths, *args):
|
||||
opened.extend(paths)
|
||||
yield from real_stream(paths, *args)
|
||||
|
||||
monkeypatch.setattr(rsl, "runai_safetensors_weights_iterator", stream)
|
||||
model = torch.nn.Module()
|
||||
if selection == "draft":
|
||||
model.is_unused_checkpoint_weight = lambda name: not name.startswith("draft.")
|
||||
elif selection == "reject_all":
|
||||
model.is_unused_checkpoint_weight = lambda name: True
|
||||
config = types.SimpleNamespace(model=str(tmp_path), model_weights="", revision=None)
|
||||
loaded = dict(_runai_loader({}).get_all_weights(config, model))
|
||||
|
||||
expected = files[1:] if selection == "draft" else files
|
||||
assert opened == expected
|
||||
assert set(loaded) == {
|
||||
name
|
||||
for tensors in (contents[1:] if selection == "draft" else contents)
|
||||
for name in tensors
|
||||
}
|
||||
for name, tensor in {**contents[1], **contents[2]}.items():
|
||||
torch.testing.assert_close(loaded[name], tensor)
|
||||
|
||||
|
||||
def _check_distributed_selection(rank, rendezvous, scenario):
|
||||
torch.distributed.init_process_group(
|
||||
"gloo",
|
||||
init_method=rendezvous,
|
||||
rank=rank,
|
||||
world_size=2,
|
||||
timeout=timedelta(seconds=30),
|
||||
)
|
||||
root = "s3://bucket/weights"
|
||||
if scenario == "different_roots":
|
||||
root += str(rank)
|
||||
files = [root + "/main.safetensors", root + "/draft.safetensors"]
|
||||
|
||||
def pull_files(source, destination, **kwargs):
|
||||
if scenario == "metadata_failure" and rank == 1:
|
||||
raise OSError("Index temporarily unavailable")
|
||||
(Path(destination) / "model.safetensors.index.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"weight_map": {
|
||||
"main.weight": "main.safetensors",
|
||||
"draft.weight": "draft.safetensors",
|
||||
}
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
try:
|
||||
with pytest.MonkeyPatch.context() as monkeypatch:
|
||||
monkeypatch.setattr(rsl, "list_safetensors", lambda path: files)
|
||||
monkeypatch.setattr(rsl, "runai_pull_files", pull_files)
|
||||
monkeypatch.setattr(
|
||||
rsl,
|
||||
"_get_remote_checkpoint_keys",
|
||||
lambda paths: [
|
||||
["main.weight"]
|
||||
if path.endswith("main.safetensors")
|
||||
else ["draft.weight"]
|
||||
for path in paths
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rsl,
|
||||
"get_world_group",
|
||||
lambda: types.SimpleNamespace(
|
||||
world_size=2, cpu_group=torch.distributed.group.WORLD
|
||||
),
|
||||
raising=False,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rsl,
|
||||
"get_pp_group",
|
||||
lambda: types.SimpleNamespace(
|
||||
world_size=2 if scenario == "pipeline_draft" else 1
|
||||
),
|
||||
raising=False,
|
||||
)
|
||||
if scenario == "pipeline_draft" and rank == 0:
|
||||
# Earlier PP stages never invoke the draft loader.
|
||||
torch.distributed.barrier()
|
||||
return
|
||||
# Exercise real selection and Gloo; observe the distributed stream input.
|
||||
monkeypatch.setattr(
|
||||
rsl,
|
||||
"runai_safetensors_weights_iterator",
|
||||
lambda paths, *args: iter((path, None) for path in paths),
|
||||
)
|
||||
model = torch.nn.Module()
|
||||
if not (scenario == "missing_hook" and rank == 1):
|
||||
needed = (
|
||||
"main." if scenario == "different_roles" and rank == 1 else "draft."
|
||||
)
|
||||
model.is_unused_checkpoint_weight = lambda name: (
|
||||
not name.startswith(needed)
|
||||
)
|
||||
config = types.SimpleNamespace(model=root, model_weights="", revision=None)
|
||||
selected = [
|
||||
path
|
||||
for path, _ in _runai_loader({"distributed": True}).get_all_weights(
|
||||
config, model
|
||||
)
|
||||
]
|
||||
expected = files[1:] if scenario == "healthy" else files
|
||||
assert selected == expected
|
||||
if scenario == "pipeline_draft":
|
||||
torch.distributed.barrier()
|
||||
finally:
|
||||
torch.distributed.destroy_process_group()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"scenario",
|
||||
[
|
||||
"healthy",
|
||||
"metadata_failure",
|
||||
"different_roles",
|
||||
"missing_hook",
|
||||
"different_roots",
|
||||
"pipeline_draft",
|
||||
],
|
||||
)
|
||||
def test_distributed_runai_preserves_every_ranks_required_shards(tmp_path, scenario):
|
||||
"""All ranks must stream the same union even if one cannot filter safely."""
|
||||
torch.multiprocessing.spawn(
|
||||
_check_distributed_selection,
|
||||
args=((tmp_path / "rendezvous").as_uri(), scenario),
|
||||
nprocs=2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scheme", ["s3", "gs", "az"])
|
||||
@pytest.mark.parametrize(
|
||||
"index_kind",
|
||||
[
|
||||
"valid",
|
||||
"missing",
|
||||
"malformed",
|
||||
"unavailable",
|
||||
"unadapted",
|
||||
"missing_shard",
|
||||
"stale",
|
||||
"header_unavailable",
|
||||
],
|
||||
)
|
||||
def test_runai_remote_selection_uses_actual_weight_source(
|
||||
tmp_path, monkeypatch, scheme, index_kind
|
||||
):
|
||||
root = f"{scheme}://bucket/weights"
|
||||
files = [
|
||||
root + "/main/part.safetensors",
|
||||
root + "/draft/part.safetensors",
|
||||
root + "/mixed.safetensors",
|
||||
root + "/extra.safetensors",
|
||||
]
|
||||
monkeypatch.setattr(rsl, "list_safetensors", lambda path: files)
|
||||
pulled = []
|
||||
|
||||
def pull_files(source, destination, allow_pattern, ignore_pattern=None):
|
||||
pulled.append((source.rstrip("/"), allow_pattern))
|
||||
if index_kind == "unavailable":
|
||||
# Storage SDKs do not necessarily derive errors from OSError.
|
||||
raise Exception("Object metadata unavailable")
|
||||
index = Path(destination) / "model.safetensors.index.json"
|
||||
if index_kind in ("valid", "missing_shard", "stale", "header_unavailable"):
|
||||
index.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"weight_map": {
|
||||
"main.weight": "main/part.safetensors",
|
||||
"draft.weight": (
|
||||
"absent.safetensors"
|
||||
if index_kind == "missing_shard"
|
||||
else "draft/part.safetensors"
|
||||
),
|
||||
"main.bias": "mixed.safetensors",
|
||||
"draft.bias": "mixed.safetensors",
|
||||
}
|
||||
}
|
||||
)
|
||||
)
|
||||
elif index_kind == "malformed":
|
||||
index.write_text("not JSON")
|
||||
|
||||
monkeypatch.setattr(rsl, "runai_pull_files", pull_files, raising=False)
|
||||
headers_checked = []
|
||||
|
||||
def checkpoint_keys(paths):
|
||||
headers_checked.extend(paths)
|
||||
if index_kind == "header_unavailable":
|
||||
raise OSError("Header temporarily unavailable")
|
||||
return [
|
||||
["main.weight", "draft.unindexed_weight"]
|
||||
if index_kind == "stale"
|
||||
else ["main.weight"]
|
||||
for _ in paths
|
||||
]
|
||||
|
||||
monkeypatch.setattr(
|
||||
rsl, "_get_remote_checkpoint_keys", checkpoint_keys, raising=False
|
||||
)
|
||||
opened = []
|
||||
|
||||
def stream(paths, *args):
|
||||
opened.extend(paths)
|
||||
return iter(())
|
||||
|
||||
monkeypatch.setattr(rsl, "runai_safetensors_weights_iterator", stream)
|
||||
model = torch.nn.Module()
|
||||
if index_kind != "unadapted":
|
||||
model.is_unused_checkpoint_weight = lambda name: not name.startswith("draft.")
|
||||
# The config directory is not the checkpoint: using its index could drop
|
||||
# tensors from an independent model_weights source.
|
||||
(tmp_path / "model.safetensors.index.json").write_text("not the weights index")
|
||||
config = types.SimpleNamespace(
|
||||
model=str(tmp_path), model_weights=root, revision=None
|
||||
)
|
||||
list(_runai_loader({}).get_all_weights(config, model))
|
||||
|
||||
assert opened == (files[1:] if index_kind == "valid" else files)
|
||||
assert headers_checked == (
|
||||
files[:1] if index_kind in ("valid", "stale", "header_unavailable") else []
|
||||
)
|
||||
assert pulled == (
|
||||
[]
|
||||
if index_kind == "unadapted"
|
||||
else [
|
||||
(
|
||||
root,
|
||||
[
|
||||
"weights/model.safetensors.index.json",
|
||||
"model.safetensors.index.json",
|
||||
],
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"weight_map",
|
||||
[None, {}, [], {"draft.weight": None}, {"draft.weight": "../part.safetensors"}],
|
||||
)
|
||||
def test_runai_invalid_weight_map_keeps_original_files(monkeypatch, weight_map):
|
||||
files = ["s3://bucket/model/part.safetensors"]
|
||||
monkeypatch.setattr(rsl, "list_safetensors", lambda path: files)
|
||||
|
||||
def pull_files(source, directory, **kwargs):
|
||||
(Path(directory) / "model.safetensors.index.json").write_text(
|
||||
json.dumps({"weight_map": weight_map})
|
||||
)
|
||||
|
||||
monkeypatch.setattr(rsl, "runai_pull_files", pull_files)
|
||||
monkeypatch.setattr(
|
||||
rsl, "runai_safetensors_weights_iterator", lambda paths, *args: iter(paths)
|
||||
)
|
||||
model = torch.nn.Module()
|
||||
model.is_unused_checkpoint_weight = lambda name: True
|
||||
config = types.SimpleNamespace(
|
||||
model="s3://bucket/model", model_weights="", revision=None
|
||||
)
|
||||
assert list(_runai_loader({}).get_all_weights(config, model)) == files
|
||||
|
||||
|
||||
def test_runai_model_rule_error_is_not_hidden_by_metadata_fallback(monkeypatch):
|
||||
files = ["s3://bucket/model/part.safetensors"]
|
||||
monkeypatch.setattr(rsl, "list_safetensors", lambda path: files)
|
||||
|
||||
def pull_files(source, directory, **kwargs):
|
||||
(Path(directory) / "model.safetensors.index.json").write_text(
|
||||
json.dumps({"weight_map": {"draft.weight": "part.safetensors"}})
|
||||
)
|
||||
|
||||
monkeypatch.setattr(rsl, "runai_pull_files", pull_files)
|
||||
model = torch.nn.Module()
|
||||
|
||||
def invalid_rule(name):
|
||||
raise ValueError("Invalid model rule")
|
||||
|
||||
model.is_unused_checkpoint_weight = invalid_rule
|
||||
config = types.SimpleNamespace(
|
||||
model="s3://bucket/model", model_weights="", revision=None
|
||||
)
|
||||
with pytest.raises(ValueError, match="Invalid model rule"):
|
||||
list(_runai_loader({}).get_all_weights(config, model))
|
||||
|
||||
|
||||
def test_runai_header_reader_uses_actual_tensor_names(tmp_path):
|
||||
"""The RunAI header reader sees names even when an index omits them."""
|
||||
path = tmp_path / "mixed.safetensors"
|
||||
save_file(
|
||||
{
|
||||
"main.weight": torch.tensor([1.0]),
|
||||
"draft.unindexed_weight": torch.tensor([2.0]),
|
||||
},
|
||||
path,
|
||||
metadata={"format": "pt"},
|
||||
)
|
||||
(names,) = rsl._get_remote_checkpoint_keys([str(path)])
|
||||
assert set(names) == {"main.weight", "draft.unindexed_weight"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stale", [False, True])
|
||||
def test_runai_stale_index_keeps_actual_draft_weights(tmp_path, monkeypatch, stale):
|
||||
"""Verify excluded headers before dropping a shard with real draft weights."""
|
||||
root = "s3://bucket/checkpoint"
|
||||
main_path = tmp_path / "main.safetensors"
|
||||
draft_path = tmp_path / "draft.safetensors"
|
||||
main_tensors = {"main.weight": torch.tensor([1.0])}
|
||||
if stale:
|
||||
main_tensors["draft.unindexed_weight"] = torch.tensor([2.0])
|
||||
save_file(main_tensors, main_path)
|
||||
save_file({"draft.weight": torch.tensor([3.0])}, draft_path)
|
||||
paths = {
|
||||
root + "/main.safetensors": str(main_path),
|
||||
root + "/draft.safetensors": str(draft_path),
|
||||
}
|
||||
monkeypatch.setattr(rsl, "list_safetensors", lambda path: list(paths))
|
||||
|
||||
def pull_files(source, directory, **kwargs):
|
||||
(Path(directory) / "model.safetensors.index.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"weight_map": {
|
||||
"main.weight": "main.safetensors",
|
||||
"draft.weight": "draft.safetensors",
|
||||
}
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
monkeypatch.setattr(rsl, "runai_pull_files", pull_files)
|
||||
read_headers = rsl._get_remote_checkpoint_keys
|
||||
monkeypatch.setattr(
|
||||
rsl,
|
||||
"_get_remote_checkpoint_keys",
|
||||
lambda files: read_headers([paths[path] for path in files]),
|
||||
)
|
||||
stream = rsl.runai_safetensors_weights_iterator
|
||||
|
||||
def stream_local(files, *args):
|
||||
for name, tensor in stream([paths[path] for path in files], *args):
|
||||
yield name, tensor.clone()
|
||||
|
||||
monkeypatch.setattr(rsl, "runai_safetensors_weights_iterator", stream_local)
|
||||
model = torch.nn.Module()
|
||||
model.is_unused_checkpoint_weight = lambda name: not name.startswith("draft.")
|
||||
config = types.SimpleNamespace(model=root, model_weights="", revision=None)
|
||||
loaded = dict(_runai_loader({}).get_all_weights(config, model))
|
||||
assert set(loaded) == (
|
||||
{"main.weight", "draft.unindexed_weight", "draft.weight"}
|
||||
if stale
|
||||
else {"draft.weight"}
|
||||
)
|
||||
torch.testing.assert_close(loaded["draft.weight"], torch.tensor([3.0]))
|
||||
if stale:
|
||||
torch.testing.assert_close(
|
||||
loaded["draft.unindexed_weight"], torch.tensor([2.0])
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"base_model,mul_model,add_model",
|
||||
[
|
||||
|
||||
@@ -0,0 +1,337 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from safetensors.torch import save_file
|
||||
from transformers import DeepseekV2Config, DeepseekV2ForCausalLM
|
||||
|
||||
from vllm.config import (
|
||||
AttentionConfig,
|
||||
DeviceConfig,
|
||||
KernelConfig,
|
||||
LoadConfig,
|
||||
ModelConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.expert_substitution import (
|
||||
ConstantExpertSubstitution,
|
||||
)
|
||||
from vllm.model_executor.model_loader import get_model_loader
|
||||
from vllm.model_executor.models.expert_substitution import as_expert_substitution_model
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader, PPMissingLayer
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
MODEL_CONFIG = SimpleNamespace(
|
||||
hf_config=SimpleNamespace(approximate_experts={"0": [1, 2], "1": [1, 2]})
|
||||
)
|
||||
|
||||
|
||||
def _compression_config(value_names: dict[int, dict[int, str]]) -> dict:
|
||||
"""Explicitly named constants given as ``{layer: {expert: tensor name}}``."""
|
||||
targets = {
|
||||
f"model.layers.{layer}.mlp.experts": {
|
||||
"num_logical_experts": 3,
|
||||
"weight_layout": "compact_retained_experts",
|
||||
"replacements": {
|
||||
str(expert): {"format": "constant-v1", "tensors": {"value": name}}
|
||||
for expert, name in names.items()
|
||||
},
|
||||
}
|
||||
for layer, names in value_names.items()
|
||||
}
|
||||
router_semantics = {
|
||||
"preserve_logical_expert_ids": True,
|
||||
"preserve_router_weights": True,
|
||||
"renormalize_after_substitution": False,
|
||||
}
|
||||
return {
|
||||
"transform_config": {
|
||||
"expert_substitution": {
|
||||
"version": 1,
|
||||
"router_semantics": router_semantics,
|
||||
"targets": targets,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
# Expert 1 of both layers shares one explicitly named constant.
|
||||
EXPLICIT_MODEL_CONFIG = SimpleNamespace(
|
||||
hf_config=SimpleNamespace(
|
||||
compression_config=_compression_config(
|
||||
{
|
||||
layer: {1: "constants.shared", 2: f"constants.layer_{layer}"}
|
||||
for layer in (0, 1)
|
||||
}
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _approx_value(layer: int, expert: int) -> str:
|
||||
return f"model.layers.{layer}.mlp.experts.{expert}.approx_value"
|
||||
|
||||
|
||||
def _cpu_model(local_layers=(0, 1), tracked=True, model_config=MODEL_CONFIG):
|
||||
"""Two decoder layers; the others are placeholders as on another PP stage."""
|
||||
|
||||
class Model(torch.nn.Module):
|
||||
def __init__(self, prefix=""):
|
||||
super().__init__()
|
||||
self.weight = torch.nn.Parameter(torch.zeros(2), requires_grad=False)
|
||||
self.model = torch.nn.Module()
|
||||
self.model.layers = torch.nn.ModuleList()
|
||||
for index in range(2):
|
||||
if index not in local_layers:
|
||||
self.model.layers.append(PPMissingLayer())
|
||||
continue
|
||||
layer = torch.nn.Module()
|
||||
layer.mlp = torch.nn.Module()
|
||||
layer.mlp.experts = ConstantExpertSubstitution(
|
||||
index, 3, (1, 2), 2, torch.float32
|
||||
)
|
||||
self.model.layers.append(layer)
|
||||
|
||||
def load_weights(self, weights):
|
||||
loaded = AutoWeightsLoader(self).load_weights(weights)
|
||||
return loaded if tracked else None
|
||||
|
||||
return as_expert_substitution_model(Model, model_config)()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("local_layers", [(0, 1), (0,), (1,), ()])
|
||||
def test_loads_local_approx_values_and_passes_through_others(local_layers):
|
||||
"""Other PP stages and MTP drafts own none or some of the configured layers."""
|
||||
model = _cpu_model(local_layers)
|
||||
loaded = model.load_weights(
|
||||
[
|
||||
("weight", torch.tensor([1.0, 2.0])),
|
||||
*(
|
||||
(_approx_value(layer, expert), torch.full((2,), 10.0 * layer + expert))
|
||||
for layer in range(2)
|
||||
for expert in (1, 2)
|
||||
),
|
||||
]
|
||||
)
|
||||
model.process_weights_after_loading()
|
||||
|
||||
assert loaded == {"weight"} | {
|
||||
f"model.layers.{layer}.mlp.experts.values" for layer in local_layers
|
||||
}
|
||||
for layer in local_layers:
|
||||
torch.testing.assert_close(
|
||||
model.model.layers[layer].mlp.experts.values,
|
||||
torch.tensor([[1.0, 1.0], [2.0, 2.0]]) + 10.0 * layer,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("local_layers", [(0, 1), (1,), ()])
|
||||
def test_loads_explicitly_named_values(local_layers):
|
||||
"""Named values are consumed even when their layer is built elsewhere."""
|
||||
model = _cpu_model(local_layers, model_config=EXPLICIT_MODEL_CONFIG)
|
||||
loaded = model.load_weights(
|
||||
[
|
||||
("weight", torch.tensor([1.0, 2.0])),
|
||||
("constants.shared", torch.full((2,), 1.0)),
|
||||
("constants.layer_0", torch.full((2,), 2.0)),
|
||||
("constants.layer_1", torch.full((2,), 3.0)),
|
||||
]
|
||||
)
|
||||
model.process_weights_after_loading()
|
||||
|
||||
assert loaded == {"weight"} | {
|
||||
f"model.layers.{layer}.mlp.experts.values" for layer in local_layers
|
||||
}
|
||||
for layer in local_layers:
|
||||
torch.testing.assert_close(
|
||||
model.model.layers[layer].mlp.experts.values,
|
||||
torch.tensor([[1.0, 1.0], [2.0 + layer, 2.0 + layer]]),
|
||||
)
|
||||
|
||||
|
||||
def test_direct_reload_updates_values_incrementally():
|
||||
model = _cpu_model(local_layers=(0,), tracked=False)
|
||||
values = model.model.layers[0].mlp.experts.values
|
||||
model.load_weights([(_approx_value(0, 2), torch.ones(2))])
|
||||
with pytest.raises(ValueError, match="missing approx_value .* \\[1\\]"):
|
||||
model.process_weights_after_loading()
|
||||
|
||||
assert model.load_weights([(_approx_value(0, 1), torch.zeros(2))]) is None
|
||||
model.process_weights_after_loading()
|
||||
model.load_weights([(_approx_value(0, 2), torch.full((2,), 3.0))])
|
||||
torch.testing.assert_close(values, torch.tensor([[0.0, 0.0], [3.0, 3.0]]))
|
||||
|
||||
with pytest.raises(ValueError, match="not listed"):
|
||||
model.load_weights([(_approx_value(0, 0), torch.ones(2))])
|
||||
|
||||
|
||||
def test_layerwise_reload_is_rejected_and_keeps_weights():
|
||||
from vllm.model_executor.model_loader.reload import (
|
||||
initialize_layerwise_reload,
|
||||
record_metadata_for_reloading,
|
||||
)
|
||||
|
||||
model = _cpu_model(local_layers=(0,))
|
||||
record_metadata_for_reloading(model)
|
||||
loaded = [
|
||||
("weight", torch.tensor([3.0, 4.0])),
|
||||
(_approx_value(0, 1), torch.zeros(2)),
|
||||
(_approx_value(0, 2), torch.ones(2)),
|
||||
]
|
||||
model.load_weights(loaded)
|
||||
model.process_weights_after_loading()
|
||||
substitution = model.model.layers[0].mlp.experts
|
||||
values = substitution.values
|
||||
|
||||
initialize_layerwise_reload(model)
|
||||
with pytest.raises(NotImplementedError, match="layerwise weight reload"):
|
||||
model.load_weights([(name, weight + 1) for name, weight in loaded])
|
||||
|
||||
assert substitution.values is values
|
||||
torch.testing.assert_close(values, torch.tensor([[0.0, 0.0], [1.0, 1.0]]))
|
||||
torch.testing.assert_close(model.weight, torch.tensor([3.0, 4.0]))
|
||||
assert substitution.expert_substitution_routes.tolist() == [0, -1, -2]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("missing_value", [False, True])
|
||||
def test_tensorizer_finalizes_local_substitution_values(monkeypatch, missing_value):
|
||||
"""Raw Tensorizer loading must reject an unloaded local constant row."""
|
||||
from vllm.model_executor.model_loader import tensorizer_loader
|
||||
|
||||
model = _cpu_model(local_layers=(0,))
|
||||
weights = [(_approx_value(0, 1), torch.zeros(2))]
|
||||
if not missing_value:
|
||||
weights.append((_approx_value(0, 2), torch.ones(2)))
|
||||
finalize = Mock(wraps=model.process_weights_after_loading)
|
||||
monkeypatch.setattr(model, "process_weights_after_loading", finalize)
|
||||
monkeypatch.setattr(tensorizer_loader, "initialize_model", lambda **kwargs: model)
|
||||
loader = object.__new__(tensorizer_loader.TensorizerLoader)
|
||||
monkeypatch.setattr(loader, "_get_weights_iterator", lambda: iter(weights))
|
||||
config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(dtype=torch.float32),
|
||||
device_config=SimpleNamespace(device="cpu"),
|
||||
)
|
||||
|
||||
if missing_value:
|
||||
with pytest.raises(ValueError, match=r"missing approx_value .* \[2\]"):
|
||||
loader._load_model_serialized_cpu(config)
|
||||
else:
|
||||
assert loader._load_model_serialized_cpu(config) is model
|
||||
torch.testing.assert_close(
|
||||
model.model.layers[0].mlp.experts.values,
|
||||
torch.tensor([[0.0, 0.0], [1.0, 1.0]]),
|
||||
)
|
||||
finalize.assert_called_once_with()
|
||||
|
||||
|
||||
def _write_tiny_deepseek_mone_checkpoint(
|
||||
model_dir: Path, explicit_names: bool
|
||||
) -> tuple[dict[str, torch.Tensor], str]:
|
||||
config = DeepseekV2Config(
|
||||
architectures=["DeepseekV2ForCausalLM"],
|
||||
vocab_size=32,
|
||||
hidden_size=16,
|
||||
intermediate_size=32,
|
||||
moe_intermediate_size=8,
|
||||
num_hidden_layers=1,
|
||||
num_attention_heads=2,
|
||||
num_key_value_heads=2,
|
||||
max_position_embeddings=32,
|
||||
first_k_dense_replace=0,
|
||||
n_routed_experts=3,
|
||||
n_shared_experts=1,
|
||||
num_experts_per_tok=1,
|
||||
n_group=1,
|
||||
topk_group=1,
|
||||
norm_topk_prob=False,
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=4,
|
||||
qk_nope_head_dim=128,
|
||||
qk_rope_head_dim=64,
|
||||
v_head_dim=128,
|
||||
tie_word_embeddings=False,
|
||||
dtype="float16",
|
||||
)
|
||||
experts = "model.layers.0.mlp.experts"
|
||||
if explicit_names:
|
||||
value_name = "model.layers.0.mlp.expert_replacements.1.value"
|
||||
config.compression_config = _compression_config({0: {1: value_name}})
|
||||
else:
|
||||
value_name = f"{experts}.1.approx_value"
|
||||
config.approximate_experts = {"0": [1]}
|
||||
|
||||
hf_weights = DeepseekV2ForCausalLM(config).to(torch.float16).state_dict()
|
||||
gate_up = hf_weights.pop(f"{experts}.gate_up_proj")
|
||||
down = hf_weights.pop(f"{experts}.down_proj")
|
||||
weights = {name: value.contiguous() for name, value in hf_weights.items()}
|
||||
for expert in (0, 2):
|
||||
gate, up = gate_up[expert].chunk(2, dim=0)
|
||||
weights[f"{experts}.{expert}.gate_proj.weight"] = gate.contiguous()
|
||||
weights[f"{experts}.{expert}.up_proj.weight"] = up.contiguous()
|
||||
weights[f"{experts}.{expert}.down_proj.weight"] = down[expert].contiguous()
|
||||
weights[value_name] = torch.arange(16, dtype=torch.float16)
|
||||
config.save_pretrained(model_dir)
|
||||
save_file(weights, model_dir / "model.safetensors")
|
||||
return weights, value_name
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like device"
|
||||
)
|
||||
@pytest.mark.usefixtures("dist_init", "workspace_init")
|
||||
@pytest.mark.parametrize(
|
||||
"explicit_names", [False, True], ids=["approximate_experts", "expert_substitution"]
|
||||
)
|
||||
def test_loads_mone_checkpoint_into_compact_experts(
|
||||
tmp_path: Path, explicit_names: bool
|
||||
):
|
||||
weights, value_name = _write_tiny_deepseek_mone_checkpoint(tmp_path, explicit_names)
|
||||
model_config = ModelConfig(
|
||||
model=str(tmp_path),
|
||||
tokenizer=str(tmp_path),
|
||||
skip_tokenizer_init=True,
|
||||
dtype="float16",
|
||||
max_model_len=32,
|
||||
enforce_eager=True,
|
||||
)
|
||||
load_config = LoadConfig(load_format="safetensors", use_tqdm_on_load=False)
|
||||
vllm_config = VllmConfig(
|
||||
model_config=model_config,
|
||||
device_config=DeviceConfig(device="cuda"),
|
||||
load_config=load_config,
|
||||
attention_config=AttentionConfig(backend=AttentionBackendEnum.TRITON_MLA),
|
||||
kernel_config=KernelConfig(moe_backend="triton"),
|
||||
)
|
||||
model = get_model_loader(load_config).load_model(vllm_config, model_config)
|
||||
|
||||
experts = "model.layers.0.mlp.experts"
|
||||
routed_experts = model.model.layers[0].mlp.experts.routed_experts
|
||||
# Logical expert 2 is stored in physical row 1.
|
||||
torch.testing.assert_close(
|
||||
routed_experts.w13_weight[1].cpu(),
|
||||
torch.cat(
|
||||
[
|
||||
weights[f"{experts}.2.gate_proj.weight"],
|
||||
weights[f"{experts}.2.up_proj.weight"],
|
||||
]
|
||||
),
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
routed_experts.w2_weight[1].cpu(), weights[f"{experts}.2.down_proj.weight"]
|
||||
)
|
||||
substitution = routed_experts.expert_substitution
|
||||
torch.testing.assert_close(
|
||||
substitution.values.cpu(), weights[value_name].unsqueeze(0)
|
||||
)
|
||||
|
||||
updated = weights[value_name] + 1
|
||||
assert model.load_weights([(value_name, updated)]) == {
|
||||
f"{experts}.routed_experts.expert_substitution.values"
|
||||
}
|
||||
torch.testing.assert_close(substitution.values.cpu(), updated.unsqueeze(0))
|
||||
@@ -137,3 +137,32 @@ def test_loader_does_not_read_shards_the_model_skips(mtp_checkpoint, monkeypatch
|
||||
["model-00002-of-00003.safetensors", "model-00003-of-00003.safetensors"],
|
||||
sorted(f.name for f in mtp_checkpoint.glob("*.safetensors")),
|
||||
]
|
||||
|
||||
|
||||
def test_secondary_source_with_explicit_patterns_skips_backbone_index(tmp_path):
|
||||
"""A source with explicit allow patterns (e.g. a standalone head file in
|
||||
the repo root) must not be validated against the backbone's index."""
|
||||
save_file(
|
||||
{"backbone.weight": torch.zeros(1)},
|
||||
tmp_path / "model-00001-of-00001.safetensors",
|
||||
)
|
||||
save_file({"head.weight": torch.zeros(1)}, tmp_path / "joint_head.safetensors")
|
||||
(tmp_path / "model.safetensors.index.json").write_text(
|
||||
json.dumps(
|
||||
{"weight_map": {"backbone.weight": "model-00001-of-00001.safetensors"}}
|
||||
)
|
||||
)
|
||||
|
||||
loader = DefaultModelLoader(LoadConfig())
|
||||
secondary = DefaultModelLoader.Source(
|
||||
model_or_path=str(tmp_path),
|
||||
revision=None,
|
||||
prefix="pooler.head.",
|
||||
allow_patterns_overrides=["joint_head.safetensors"],
|
||||
)
|
||||
assert [n for n, _ in loader._get_weights_iterator(secondary)] == [
|
||||
"pooler.head.head.weight"
|
||||
]
|
||||
# The primary source still filters files the backbone index doesn't list.
|
||||
primary = DefaultModelLoader.Source(model_or_path=str(tmp_path), revision=None)
|
||||
assert [n for n, _ in loader._get_weights_iterator(primary)] == ["backbone.weight"]
|
||||
|
||||
@@ -2,13 +2,18 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.config.load import LoadConfig
|
||||
from vllm.model_executor.layers.fused_moe.runner.moe_runner import MoERunner
|
||||
from vllm.model_executor.layers.linear import UnquantizedLinearMethod
|
||||
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
|
||||
from vllm.model_executor.model_loader import get_model_loader, register_model_loader
|
||||
from vllm.model_executor.model_loader.base_loader import BaseModelLoader
|
||||
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||
|
||||
|
||||
@register_model_loader("custom_load_format")
|
||||
@@ -88,3 +93,80 @@ def test_default_loader_hf_still_falls_back_to_pt(tmp_path):
|
||||
)
|
||||
assert use_safetensors is False
|
||||
assert any(f.endswith("model.pt") for f in files)
|
||||
|
||||
|
||||
class _QuantMethod:
|
||||
uses_meta_device = False
|
||||
|
||||
def process_weights_after_loading(self, layer: nn.Module) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def _layer(quant_method=None, **shapes) -> nn.Module:
|
||||
layer = nn.Module()
|
||||
for name, shape in shapes.items():
|
||||
param = nn.Parameter(torch.zeros(shape), requires_grad=False)
|
||||
layer.register_parameter(name, param)
|
||||
layer.quant_method = quant_method
|
||||
return layer
|
||||
|
||||
|
||||
def _track(model: nn.Module, loaded: set[str], quantized: bool) -> str:
|
||||
loader = DefaultModelLoader(LoadConfig())
|
||||
try:
|
||||
loader.track_weights_loading(model, set(loaded), quantized=quantized)
|
||||
except ValueError as e:
|
||||
return str(e)
|
||||
return ""
|
||||
|
||||
|
||||
def test_weights_track_reports_missing_quantized_weights():
|
||||
# Quantized checkpoints may only omit KV-cache quantization params, empty
|
||||
# placeholders, fbgemm_fp8's input_scale_ub and online-quant params.
|
||||
online = _QuantMethod()
|
||||
online.uses_meta_device = True
|
||||
model = nn.Module()
|
||||
model.proj = _layer(_QuantMethod(), weight=(4, 4), weight_scale_inv=(1, 1))
|
||||
model.unquantized = _layer(UnquantizedLinearMethod(), weight=(4, 4), bias=(4,))
|
||||
model.gptq = _layer(_QuantMethod(), qweight=(4, 4), qzeros=(0,))
|
||||
model.fbgemm = _layer(_QuantMethod(), weight=(4, 4), input_scale_ub=())
|
||||
model.attn = _layer(BaseKVCacheMethod(None), k_scale=(), k_zero_point=())
|
||||
model.online = _layer(online, weight=(4, 4))
|
||||
loaded = {"proj.weight", "unquantized.weight", "gptq.qweight", "fbgemm.weight"}
|
||||
error = _track(model, loaded, quantized=True)
|
||||
assert "proj.weight_scale_inv" in error and "unquantized.bias" in error
|
||||
assert not any(n in error for n in ("gptq", "fbgemm", "attn", "online"))
|
||||
|
||||
|
||||
def test_weights_track_keeps_unquantized_models_unchanged():
|
||||
# The check is on by default here, so unquantized layers stay exempt.
|
||||
model = nn.Module()
|
||||
model.score = _layer(_QuantMethod(), weight=(2, 4))
|
||||
model.norm = _layer(weight=(4,))
|
||||
assert _track(model, {"norm.weight"}, quantized=False) == ""
|
||||
assert "norm.weight" in _track(model, set(), quantized=False)
|
||||
|
||||
|
||||
def test_moe_runner_reports_registered_expert_names():
|
||||
# AutoWeightsLoader prefixes these names, so they must include routed_experts.
|
||||
class Experts(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.w13_weight = nn.Parameter(torch.zeros(2, 8, 4), requires_grad=False)
|
||||
|
||||
def load_weights(self, weights):
|
||||
for _ in weights:
|
||||
yield "w13_weight"
|
||||
|
||||
class Runner(nn.Module):
|
||||
load_weights = MoERunner.load_weights
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.routed_experts = Experts()
|
||||
|
||||
model = nn.Module()
|
||||
model.experts = Runner()
|
||||
weights = [("experts.w13_weight", torch.zeros(2, 8, 4))]
|
||||
loaded = AutoWeightsLoader(model).load_weights(weights)
|
||||
assert loaded == {"experts.routed_experts.w13_weight"}
|
||||
|
||||
@@ -5,6 +5,7 @@ import importlib.machinery
|
||||
import inspect
|
||||
import sys
|
||||
import types
|
||||
from importlib import import_module
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
from weakref import WeakKeyDictionary, ref
|
||||
@@ -39,23 +40,227 @@ from vllm.model_executor.model_loader.reload.utils import get_layer_tensors
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
composed_weight_loader,
|
||||
default_weight_loader,
|
||||
load_attention_sink,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_rank", [0, 1])
|
||||
@pytest.mark.parametrize(
|
||||
"module_name,class_name,sink_name",
|
||||
[
|
||||
("mimo_v2_mtp", "MiMoV2MTP", "attention_sink_bias"),
|
||||
("mimo_v2_mtp", "MiMoV2OmniMTP", "attention_sink_bias"),
|
||||
("laguna", "LagunaModel", "sink"),
|
||||
("qwen3_dflash", "DFlashQwen3Model", "attention_sink_bias"),
|
||||
],
|
||||
)
|
||||
def test_attention_sink_model_loading_shards_once(
|
||||
monkeypatch, tp_rank, module_name, class_name, sink_name
|
||||
):
|
||||
"""Model loading must pass global sinks to the loader on cold load and reload."""
|
||||
module = import_module(f"vllm.model_executor.models.{module_name}")
|
||||
from vllm.model_executor.model_loader import weight_utils
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
|
||||
for target in (module, weight_utils):
|
||||
monkeypatch.setattr(
|
||||
target, "get_tensor_model_parallel_rank", lambda: tp_rank, raising=False
|
||||
)
|
||||
monkeypatch.setattr(target, "get_tensor_model_parallel_world_size", lambda: 2)
|
||||
cls = getattr(module, class_name)
|
||||
model = cls.__new__(cls)
|
||||
torch.nn.Module.__init__(model)
|
||||
model.hf_to_vllm_mapper = WeightsMapper()
|
||||
model.model = torch.nn.Module()
|
||||
model.model.mtp = torch.nn.Module()
|
||||
sink = torch.nn.Parameter(torch.zeros(4), requires_grad=False)
|
||||
|
||||
def sink_loader(param, loaded_weight):
|
||||
assert loaded_weight.shape == (8,), "Sink was sharded before weight_loader"
|
||||
load_attention_sink(param, loaded_weight)
|
||||
|
||||
sink.weight_loader = sink_loader
|
||||
model.model.mtp.register_parameter(sink_name, sink)
|
||||
name = f"model.mtp.{sink_name}"
|
||||
checkpoint = torch.arange(8, dtype=torch.float32)
|
||||
assert model.load_weights([(name, checkpoint)]) == {name}
|
||||
assert torch.equal(sink, checkpoint.chunk(2)[tp_rank])
|
||||
record_metadata_for_reloading(model)
|
||||
initialize_layerwise_reload(model)
|
||||
model.load_weights([(name, checkpoint + 10)])
|
||||
finalize_layerwise_reload(model, SimpleNamespace(dtype=torch.float32))
|
||||
assert torch.equal(sink, (checkpoint + 10).chunk(2)[tp_rank])
|
||||
assert getattr(model.model.mtp, sink_name).data_ptr() == sink.data_ptr()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_rank", [0, 1])
|
||||
@pytest.mark.parametrize(
|
||||
"module_name,class_name,attention_name,sink_name",
|
||||
[
|
||||
("deepseek_v4", "DeepseekV4Model", "attn", "attn_sink"),
|
||||
("deepseek_v41", "DeepseekV4Model", "attn", "attn_sink"),
|
||||
("hy_v4", "HYV4Model", "self_attn", "learnable_sink_param"),
|
||||
],
|
||||
)
|
||||
def test_attention_sink_deepseek_hy_loading_uses_parameter_loader(
|
||||
monkeypatch, tp_rank, module_name, class_name, attention_name, sink_name
|
||||
):
|
||||
"""DeepSeek and HY-V4 pass global sinks to the parameter's current loader."""
|
||||
module = import_module(f"vllm.models.{module_name}.nvidia.model")
|
||||
from vllm.model_executor.model_loader import weight_utils
|
||||
|
||||
monkeypatch.setattr(weight_utils, "get_tensor_model_parallel_world_size", lambda: 2)
|
||||
monkeypatch.setattr(weight_utils, "get_tensor_model_parallel_rank", lambda: tp_rank)
|
||||
cls = getattr(module, class_name)
|
||||
model = cls.__new__(cls)
|
||||
torch.nn.Module.__init__(model)
|
||||
model.config = SimpleNamespace(tie_word_embeddings=False)
|
||||
model.quant_config = None
|
||||
model.get_expert_mapping = lambda: []
|
||||
attention = torch.nn.Module()
|
||||
layer = torch.nn.Module()
|
||||
setattr(layer, attention_name, attention)
|
||||
model.layers = torch.nn.ModuleList([layer])
|
||||
sink = torch.nn.Parameter(torch.zeros(6), requires_grad=False)
|
||||
attention.register_parameter(sink_name, sink)
|
||||
checkpoint = torch.arange(8, dtype=torch.float32)
|
||||
name = f"layers.0.{attention_name}.{sink_name}"
|
||||
|
||||
for offset in (0, 10):
|
||||
weight = checkpoint + offset
|
||||
sink.weight_loader = Mock(side_effect=load_attention_sink)
|
||||
assert model.load_weights([(name, weight)]) == {name}
|
||||
sink.weight_loader.assert_called_once_with(sink, weight)
|
||||
assert torch.equal(sink[:4], weight.chunk(2)[tp_rank])
|
||||
assert torch.isneginf(sink[4:]).all()
|
||||
|
||||
|
||||
def _gpt_oss_container(monkeypatch, quant_method=None):
|
||||
"""A minimal GptOssModel whose loading paths can be driven directly."""
|
||||
from vllm.model_executor.models import gpt_oss
|
||||
|
||||
group = SimpleNamespace(world_size=1, rank_in_group=0)
|
||||
for name in ("get_dp_group", "get_ep_group", "get_pcp_group"):
|
||||
monkeypatch.setattr(gpt_oss, name, lambda: group)
|
||||
monkeypatch.setattr(gpt_oss, "get_tensor_model_parallel_world_size", lambda: 1)
|
||||
monkeypatch.setattr(gpt_oss, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(gpt_oss, "is_pp_missing_parameter", lambda *args: False)
|
||||
model = gpt_oss.GptOssModel.__new__(gpt_oss.GptOssModel)
|
||||
torch.nn.Module.__init__(model)
|
||||
model.config = SimpleNamespace(num_local_experts=1, intermediate_size=32)
|
||||
if quant_method is not None:
|
||||
model.config.quantization_config = {"quant_method": quant_method}
|
||||
model.parallel_config = SimpleNamespace(enable_expert_parallel=True)
|
||||
model.layers = [
|
||||
SimpleNamespace(
|
||||
mlp=SimpleNamespace(
|
||||
experts=SimpleNamespace(
|
||||
_quant_method=SimpleNamespace(weight_dtype=None)
|
||||
)
|
||||
)
|
||||
)
|
||||
]
|
||||
model.get_expert_mapping = lambda: []
|
||||
return model
|
||||
|
||||
|
||||
@pytest.mark.parametrize("quant_method", ["mxfp4", "quark", None])
|
||||
def test_gpt_oss_attention_sink_uses_parameter_loader(monkeypatch, quant_method):
|
||||
"""All quantization dispatch paths honor the sink's current weight loader."""
|
||||
model = _gpt_oss_container(monkeypatch, quant_method)
|
||||
model.sinks = torch.nn.Parameter(torch.zeros(4), requires_grad=False)
|
||||
# A replacement loader must see the unsliced checkpoint even after reload
|
||||
# wraps the parameter; bare model-specific copies would bypass this spy.
|
||||
model.sinks.weight_loader = Mock(side_effect=default_weight_loader)
|
||||
checkpoint = torch.arange(4, dtype=torch.float32)
|
||||
assert model.load_weights([("sinks", checkpoint)]) == {"sinks"}
|
||||
model.sinks.weight_loader.assert_called_once_with(model.sinks, checkpoint)
|
||||
assert torch.equal(model.sinks, checkpoint)
|
||||
|
||||
|
||||
def test_gpt_oss_attention_sink_skips_missing_parameter(monkeypatch):
|
||||
"""A sink owned by another pipeline stage is skipped, not a KeyError."""
|
||||
from vllm.model_executor.models import gpt_oss
|
||||
|
||||
model = _gpt_oss_container(monkeypatch)
|
||||
monkeypatch.setattr(gpt_oss, "is_pp_missing_parameter", lambda *args: True)
|
||||
# The parameter does not exist on this rank, so loading must not touch it.
|
||||
sinks = [("sinks", torch.arange(4, dtype=torch.float32))]
|
||||
assert model.load_weights(sinks) == set()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_rank", [0, 1])
|
||||
@pytest.mark.parametrize("padding", [0, 4])
|
||||
def test_attention_sink_layerwise_reload(monkeypatch, tp_rank, padding):
|
||||
"""Reload must update the sink's kernel storage and restore its padding."""
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.model_loader.weight_utils.get_tensor_model_parallel_rank",
|
||||
lambda: tp_rank,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.model_loader.weight_utils."
|
||||
"get_tensor_model_parallel_world_size",
|
||||
lambda: 2,
|
||||
)
|
||||
|
||||
class SinkModel(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.sink = torch.nn.Parameter(
|
||||
torch.empty(4 + padding), requires_grad=False
|
||||
)
|
||||
self.sink.weight_loader = load_attention_sink
|
||||
self.proj = torch.nn.Parameter(torch.zeros(16), requires_grad=False)
|
||||
self.proj.weight_loader = default_weight_loader
|
||||
|
||||
def load_weights(self, weights):
|
||||
params = dict(self.named_parameters())
|
||||
for name, weight in weights:
|
||||
param = params[name]
|
||||
param.weight_loader(param, weight)
|
||||
|
||||
model = SinkModel()
|
||||
checkpoint_sink = torch.arange(8, dtype=torch.float32)
|
||||
model.load_weights([("sink", checkpoint_sink), ("proj", torch.ones(16))])
|
||||
kernel_sink = model.sink
|
||||
expected = torch.cat(
|
||||
(
|
||||
checkpoint_sink[4 * tp_rank : 4 * (tp_rank + 1)],
|
||||
torch.full((padding,), -torch.inf),
|
||||
)
|
||||
)
|
||||
assert torch.equal(kernel_sink, expected)
|
||||
record_metadata_for_reloading(model)
|
||||
|
||||
for offset in (10, 20):
|
||||
kernel_sink.data.fill_(0)
|
||||
initialize_layerwise_reload(model)
|
||||
assert model.sink.is_meta
|
||||
model.load_weights(
|
||||
[("sink", checkpoint_sink + offset), ("proj", torch.full((16,), offset))]
|
||||
)
|
||||
finalize_layerwise_reload(model, SimpleNamespace(dtype=torch.float32))
|
||||
expected[:4] = checkpoint_sink[4 * tp_rank : 4 * (tp_rank + 1)] + offset
|
||||
assert torch.equal(kernel_sink, expected)
|
||||
assert model.sink.data_ptr() == kernel_sink.data_ptr()
|
||||
assert torch.equal(model.proj, torch.full((16,), offset))
|
||||
|
||||
|
||||
def _fp8_reload_unsupported() -> bool:
|
||||
"""Whether the FP8 reload/online-quantize tests should be skipped.
|
||||
|
||||
``supports_fp8()`` returns True on MI250 (gfx90a) because the general
|
||||
quantization paths upcast FP8 weights, but gfx90a has no native FP8 and
|
||||
cannot run these reload models, so treat it as unsupported here.
|
||||
``supports_fp8()`` returns True on MI250 (gfx90a) and RDNA3 (gfx11)
|
||||
because the general quantization paths upcast FP8 weights, but neither has
|
||||
native FP8 and cannot run these reload models, so treat them as
|
||||
unsupported here.
|
||||
"""
|
||||
if not current_platform.supports_fp8():
|
||||
return True
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import on_gfx90a
|
||||
from vllm.platforms.rocm import on_gfx11, on_gfx90a
|
||||
|
||||
return on_gfx90a()
|
||||
return on_gfx90a() or on_gfx11()
|
||||
return False
|
||||
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import multiprocessing as mp
|
||||
import os
|
||||
import shutil
|
||||
from tempfile import TemporaryDirectory
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -49,6 +50,318 @@ def test_filter_subtensors():
|
||||
assert tensor is state_dict[key]
|
||||
|
||||
|
||||
def _mock_pp_group(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
world_size: int,
|
||||
*,
|
||||
pp_rank: int = 0,
|
||||
tp_rank: int = 0,
|
||||
tp_world_size: int = 1,
|
||||
) -> None:
|
||||
"""Stub the parallel-state accessors used by the sharded state loader."""
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.get_pp_group",
|
||||
lambda: SimpleNamespace(world_size=world_size, rank_in_group=pp_rank),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.get_tp_group",
|
||||
lambda: SimpleNamespace(world_size=tp_world_size, rank_in_group=tp_rank),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.get_tensor_model_parallel_rank",
|
||||
lambda: tp_rank,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.get_pipeline_model_parallel_rank",
|
||||
lambda: pp_rank,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"world_size,expected",
|
||||
[
|
||||
(1, ShardedStateLoader.TP_ONLY_PATTERN),
|
||||
(2, ShardedStateLoader.PP_AND_TP_PATTERN),
|
||||
],
|
||||
)
|
||||
def test_get_default_pattern(monkeypatch, world_size, expected):
|
||||
"""The default pattern is chosen from the pipeline parallel group size."""
|
||||
_mock_pp_group(monkeypatch, world_size)
|
||||
assert ShardedStateLoader._get_default_pattern() == expected
|
||||
|
||||
|
||||
def test_render_default_pattern_default_tp(monkeypatch):
|
||||
"""Without a custom pattern and PP=1, legacy TP-only names are rendered."""
|
||||
_mock_pp_group(monkeypatch, 1)
|
||||
assert (
|
||||
ShardedStateLoader._render_default_pattern(part=0)
|
||||
== "model-rank-0-part-0.safetensors"
|
||||
)
|
||||
|
||||
|
||||
def test_render_default_pattern_default_pp(monkeypatch):
|
||||
"""Without a custom pattern and PP>1, pp/tp-aware names are rendered."""
|
||||
_mock_pp_group(monkeypatch, 2)
|
||||
assert (
|
||||
ShardedStateLoader._render_default_pattern(part=0)
|
||||
== "model-pp-0-tp-0-part-0.safetensors"
|
||||
)
|
||||
|
||||
|
||||
def test_render_default_pattern_custom_tp_only(monkeypatch):
|
||||
"""A custom TP-only pattern is honored when pipeline parallelism is off."""
|
||||
_mock_pp_group(monkeypatch, 1)
|
||||
assert (
|
||||
ShardedStateLoader._render_default_pattern(
|
||||
ShardedStateLoader.TP_ONLY_PATTERN, part=0
|
||||
)
|
||||
== "model-rank-0-part-0.safetensors"
|
||||
)
|
||||
|
||||
|
||||
def test_render_default_pattern_tp_only_with_pp_fails(monkeypatch):
|
||||
"""TP-only filenames would collide across pipeline stages; fail fast."""
|
||||
_mock_pp_group(monkeypatch, 2)
|
||||
with pytest.raises(ValueError, match="does not embed the pipeline parallel"):
|
||||
ShardedStateLoader._render_default_pattern(
|
||||
ShardedStateLoader.TP_ONLY_PATTERN, part=0
|
||||
)
|
||||
|
||||
|
||||
def test_render_default_pattern_pp_pattern_without_pp_fails(monkeypatch):
|
||||
"""A pp/tp-aware pattern cannot be rendered when pipeline parallelism is off."""
|
||||
_mock_pp_group(monkeypatch, 1)
|
||||
with pytest.raises(ValueError, match="requires pipeline parallelism"):
|
||||
ShardedStateLoader._render_default_pattern(
|
||||
ShardedStateLoader.PP_AND_TP_PATTERN, part=0
|
||||
)
|
||||
|
||||
|
||||
def test_render_default_pattern_custom_pp(monkeypatch):
|
||||
"""A custom pp/tp-aware pattern is honored when pipeline parallelism is on."""
|
||||
_mock_pp_group(monkeypatch, 2)
|
||||
assert (
|
||||
ShardedStateLoader._render_default_pattern(
|
||||
ShardedStateLoader.PP_AND_TP_PATTERN, part=0
|
||||
)
|
||||
== "model-pp-0-tp-0-part-0.safetensors"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pattern,match",
|
||||
[
|
||||
# tp field present but pp field missing: pipeline stages with the
|
||||
# same tp rank would write the same file.
|
||||
(
|
||||
"model-tp-{tp_rank}-part-{part}.safetensors",
|
||||
"does not embed the pipeline parallel",
|
||||
),
|
||||
# pp field present but tp field missing: TP workers would overwrite
|
||||
# each other's shards.
|
||||
(
|
||||
"model-pp-{pp_rank}-part-{part}.safetensors",
|
||||
"does not embed the tensor parallel",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_render_default_pattern_incomplete_custom_patterns_fail(
|
||||
monkeypatch, pattern, match
|
||||
):
|
||||
"""A custom pattern must distinguish every active rank (pp and tp)."""
|
||||
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_rank=0, tp_world_size=2)
|
||||
with pytest.raises(ValueError, match=match):
|
||||
ShardedStateLoader._render_default_pattern(pattern, part=0)
|
||||
|
||||
|
||||
def test_render_default_pattern_no_rank_field_with_tp_fails(monkeypatch):
|
||||
"""Without any rank field, TP workers would collide even when PP is off."""
|
||||
_mock_pp_group(monkeypatch, 1, tp_world_size=2)
|
||||
with pytest.raises(ValueError, match="does not embed the tensor parallel"):
|
||||
ShardedStateLoader._render_default_pattern(
|
||||
"model-part-{part}.safetensors", part=0
|
||||
)
|
||||
|
||||
|
||||
def test_render_default_pattern_custom_pp_tp1_allows_pp_only(monkeypatch):
|
||||
"""With TP=1 a pp-only custom pattern is safe and must be accepted."""
|
||||
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_world_size=1)
|
||||
assert (
|
||||
ShardedStateLoader._render_default_pattern(
|
||||
"model-pp-{pp_rank}-part-{part}.safetensors", part=0
|
||||
)
|
||||
== "model-pp-1-part-0.safetensors"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pattern,match",
|
||||
[
|
||||
# {{pp_rank}} renders as literal text: both stages would write the
|
||||
# same file, so it must not count as a pp field.
|
||||
(
|
||||
"model-{{pp_rank}}-tp-{tp_rank}-part-{part}.safetensors",
|
||||
"does not embed the pipeline parallel",
|
||||
),
|
||||
# Same for an escaped tp field: TP workers would overwrite each
|
||||
# other's shards.
|
||||
(
|
||||
"model-pp-{pp_rank}-tp-{{tp_rank}}-part-{part}.safetensors",
|
||||
"does not embed the tensor parallel",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_render_default_pattern_escaped_field_not_rank_field(
|
||||
monkeypatch, pattern, match
|
||||
):
|
||||
"""Escaped braces are literal text and cannot pose as rank fields."""
|
||||
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_rank=0, tp_world_size=2)
|
||||
with pytest.raises(ValueError, match=match):
|
||||
ShardedStateLoader._render_default_pattern(pattern, part=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pattern,world_size,tp_world_size,pp_rank,tp_rank,expected",
|
||||
[
|
||||
# Mixed tp alias works when pipeline parallelism is on.
|
||||
(
|
||||
"model-pp-{pp_rank}-rank-{rank}-part-{part}.safetensors",
|
||||
2,
|
||||
2,
|
||||
1,
|
||||
0,
|
||||
"model-pp-1-rank-0-part-0.safetensors",
|
||||
),
|
||||
# tp_rank alias works for TP-only checkpoints too.
|
||||
(
|
||||
"model-tp-{tp_rank}-part-{part}.safetensors",
|
||||
1,
|
||||
2,
|
||||
0,
|
||||
3,
|
||||
"model-tp-3-part-0.safetensors",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_render_default_pattern_tp_aliases_accepted(
|
||||
monkeypatch, pattern, world_size, tp_world_size, pp_rank, tp_rank, expected
|
||||
):
|
||||
"""Both tp aliases are supplied at render time, so either works."""
|
||||
_mock_pp_group(
|
||||
monkeypatch,
|
||||
world_size,
|
||||
pp_rank=pp_rank,
|
||||
tp_rank=tp_rank,
|
||||
tp_world_size=tp_world_size,
|
||||
)
|
||||
assert ShardedStateLoader._render_default_pattern(pattern, part=0) == expected
|
||||
|
||||
|
||||
def test_render_default_pattern_unsupported_field_fails(monkeypatch):
|
||||
"""Unknown placeholders are rejected instead of KeyError-ing later."""
|
||||
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_world_size=2)
|
||||
with pytest.raises(ValueError, match="unsupported placeholder"):
|
||||
ShardedStateLoader._render_default_pattern(
|
||||
"model-{pp_rank}-{tp_rank}-{epoch}-part-{part}.safetensors", part=0
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pp_rank,tp_rank,expected",
|
||||
[
|
||||
(0, 0, "model-pp-0-tp-0-part-0.safetensors"),
|
||||
(1, 2, "model-pp-1-tp-2-part-0.safetensors"),
|
||||
(3, 5, "model-pp-3-tp-5-part-0.safetensors"),
|
||||
],
|
||||
)
|
||||
def test_render_default_pattern_non_zero_ranks(monkeypatch, pp_rank, tp_rank, expected):
|
||||
"""The rendered filename must embed the actual pp/tp ranks, not zeros."""
|
||||
_mock_pp_group(monkeypatch, 2, pp_rank=pp_rank, tp_rank=tp_rank)
|
||||
assert (
|
||||
ShardedStateLoader._render_default_pattern(
|
||||
ShardedStateLoader.PP_AND_TP_PATTERN, part=0
|
||||
)
|
||||
== expected
|
||||
)
|
||||
|
||||
|
||||
def test_render_default_pattern_part_wildcard(monkeypatch):
|
||||
"""The glob wildcard '*' passes through as the part token for file lookup."""
|
||||
_mock_pp_group(monkeypatch, 1, tp_rank=4)
|
||||
assert (
|
||||
ShardedStateLoader._render_default_pattern(part="*")
|
||||
== "model-rank-4-part-*.safetensors"
|
||||
)
|
||||
|
||||
|
||||
def test_save_model_pp_aware_filenames(monkeypatch, tmp_path):
|
||||
"""save_model writes pp/tp-aware filenames for pipeline-parallel models."""
|
||||
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_rank=2)
|
||||
|
||||
saved: list[str] = []
|
||||
|
||||
def _capture(tensors, path, **kwargs):
|
||||
saved.append(os.path.basename(path))
|
||||
|
||||
monkeypatch.setattr("safetensors.torch.save_file", _capture)
|
||||
|
||||
model = SimpleNamespace(
|
||||
state_dict=lambda: {"w": torch.zeros(4, dtype=torch.float32)}
|
||||
)
|
||||
ShardedStateLoader.save_model(model, str(tmp_path))
|
||||
assert saved == ["model-pp-1-tp-2-part-0.safetensors"]
|
||||
|
||||
|
||||
def test_save_model_part_split(monkeypatch, tmp_path):
|
||||
"""save_model rolls over to a new part when max_size is exceeded."""
|
||||
_mock_pp_group(monkeypatch, 1, tp_rank=0)
|
||||
|
||||
saved: list[str] = []
|
||||
|
||||
def _capture(tensors, path, **kwargs):
|
||||
saved.append(os.path.basename(path))
|
||||
|
||||
monkeypatch.setattr("safetensors.torch.save_file", _capture)
|
||||
|
||||
# Two float32[4] tensors (16 B each); max_size=20 fits one but not two.
|
||||
model = SimpleNamespace(
|
||||
state_dict=lambda: {
|
||||
"a": torch.zeros(4, dtype=torch.float32),
|
||||
"b": torch.zeros(4, dtype=torch.float32),
|
||||
}
|
||||
)
|
||||
ShardedStateLoader.save_model(model, str(tmp_path), max_size=20)
|
||||
assert saved == [
|
||||
"model-rank-0-part-0.safetensors",
|
||||
"model-rank-0-part-1.safetensors",
|
||||
]
|
||||
|
||||
|
||||
def test_save_model_rejects_bad_pattern_before_overwrite(monkeypatch, tmp_path):
|
||||
"""An unsafe pattern is rejected before any shard file is written."""
|
||||
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_rank=0, tp_world_size=2)
|
||||
|
||||
# With the escaped pp field this pattern renders both stages to the
|
||||
# same literal name; the validation must fire before any write.
|
||||
colliding = tmp_path / "model-{pp_rank}-tp-0-part-0.safetensors"
|
||||
colliding.write_bytes(b"existing shard")
|
||||
before = sorted(p.name for p in tmp_path.iterdir())
|
||||
|
||||
model = SimpleNamespace(
|
||||
state_dict=lambda: {"w": torch.zeros(4, dtype=torch.float32)}
|
||||
)
|
||||
with pytest.raises(ValueError, match="does not embed the pipeline parallel"):
|
||||
ShardedStateLoader.save_model(
|
||||
model,
|
||||
str(tmp_path),
|
||||
pattern="model-{{pp_rank}}-tp-{tp_rank}-part-{part}.safetensors",
|
||||
)
|
||||
|
||||
# The pre-existing shard is untouched and nothing new was written.
|
||||
assert colliding.read_bytes() == b"existing shard"
|
||||
assert sorted(p.name for p in tmp_path.iterdir()) == before
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llama_3p2_1b_files():
|
||||
input_dir = hf_api().snapshot_download(
|
||||
@@ -163,3 +476,128 @@ def test_sharded_state_loader(
|
||||
queue.join_thread()
|
||||
|
||||
assert out_before == out_after
|
||||
|
||||
|
||||
def _run_sharded_state_round_trip(
|
||||
input_dir: str,
|
||||
*,
|
||||
tp_size: int,
|
||||
pp_size: int,
|
||||
num_gpus_available: int,
|
||||
) -> None:
|
||||
"""Save a TP/PP sharded checkpoint, reload it, and compare outputs.
|
||||
|
||||
Args:
|
||||
input_dir: directory containing the original (un-sharded) model.
|
||||
tp_size: tensor parallel size used for both save and load.
|
||||
pp_size: pipeline parallel size used for both save and load.
|
||||
num_gpus_available: number of GPUs on the current machine.
|
||||
|
||||
"""
|
||||
if num_gpus_available < tp_size * pp_size:
|
||||
pytest.skip(f"Not enough GPUs for TP={tp_size}, PP={pp_size}")
|
||||
|
||||
weights_patterns = ("*.safetensors",)
|
||||
gpu_memory_utilization = 0.8
|
||||
ctx = mp.get_context("spawn")
|
||||
|
||||
# Keep batching deterministic: this test compares exact greedy outputs
|
||||
# across separate engine processes, whose schedulers may otherwise form
|
||||
# different batches depending on prompt-processing timing.
|
||||
platform_args = {"max_num_seqs": 1}
|
||||
parallel_args = dict(
|
||||
tensor_parallel_size=tp_size,
|
||||
pipeline_parallel_size=pp_size,
|
||||
)
|
||||
|
||||
with TemporaryDirectory() as output_dir:
|
||||
p = ctx.Process(
|
||||
target=_run_writer,
|
||||
args=(input_dir, output_dir, weights_patterns),
|
||||
kwargs=dict(
|
||||
**parallel_args,
|
||||
gpu_memory_utilization=gpu_memory_utilization,
|
||||
enforce_eager=True,
|
||||
**platform_args,
|
||||
),
|
||||
)
|
||||
p.start()
|
||||
p.join()
|
||||
assert p.exitcode == 0, f"writer process failed with exit code {p.exitcode}"
|
||||
|
||||
if pp_size > 1:
|
||||
# PP checkpoints must use the pp/tp-aware naming scheme so that
|
||||
# different pipeline stages write distinct files.
|
||||
sharded_files = [
|
||||
f for f in os.listdir(output_dir) if f.endswith(".safetensors")
|
||||
]
|
||||
assert sharded_files, "no sharded checkpoint files were written"
|
||||
assert any(
|
||||
fnmatch.fnmatch(f, "model-pp-*-tp-*-part-*.safetensors")
|
||||
for f in sharded_files
|
||||
), f"expected PP-aware filenames, got {sharded_files}"
|
||||
|
||||
queue = ctx.Queue()
|
||||
p = ctx.Process(
|
||||
target=_run_generate,
|
||||
args=(input_dir, queue),
|
||||
kwargs=dict(
|
||||
enable_lora=False,
|
||||
gpu_memory_utilization=gpu_memory_utilization,
|
||||
**parallel_args,
|
||||
**platform_args,
|
||||
),
|
||||
)
|
||||
p.start()
|
||||
# Call queue.get() before p.join() to prevent deadlock:
|
||||
# If p.join() is called before queue.get() and the queue is full,
|
||||
# the child process may block while writing to the queue and never
|
||||
# terminate, causing the parent to wait indefinitely on p.join().
|
||||
# See: https://github.com/vllm-project/vllm/pull/22371#discussion_r2257773814
|
||||
out_before = queue.get()
|
||||
p.join()
|
||||
assert p.exitcode == 0, f"generate process failed with exit code {p.exitcode}"
|
||||
queue.close()
|
||||
queue.join_thread()
|
||||
|
||||
queue = ctx.Queue()
|
||||
p = ctx.Process(
|
||||
target=_run_generate,
|
||||
args=(output_dir, queue),
|
||||
kwargs=dict(
|
||||
enable_lora=False,
|
||||
gpu_memory_utilization=gpu_memory_utilization,
|
||||
load_format="sharded_state",
|
||||
**parallel_args,
|
||||
**platform_args,
|
||||
),
|
||||
)
|
||||
p.start()
|
||||
out_after = queue.get()
|
||||
p.join()
|
||||
assert p.exitcode == 0, f"generate process failed with exit code {p.exitcode}"
|
||||
queue.close()
|
||||
queue.join_thread()
|
||||
|
||||
assert out_before == out_after
|
||||
|
||||
|
||||
def test_sharded_state_loader_pp(num_gpus_available, llama_3p2_1b_files):
|
||||
"""Round-trip sharded state save/load with pipeline parallelism (PP=2)."""
|
||||
_run_sharded_state_round_trip(
|
||||
llama_3p2_1b_files,
|
||||
tp_size=1,
|
||||
pp_size=2,
|
||||
num_gpus_available=num_gpus_available,
|
||||
)
|
||||
|
||||
|
||||
def test_sharded_state_loader_pp_tp(num_gpus_available, llama_3p2_1b_files):
|
||||
"""Round-trip sharded state save/load with pipeline + tensor parallelism
|
||||
(PP=2, TP=2)."""
|
||||
_run_sharded_state_round_trip(
|
||||
llama_3p2_1b_files,
|
||||
tp_size=2,
|
||||
pp_size=2,
|
||||
num_gpus_available=num_gpus_available,
|
||||
)
|
||||
|
||||
@@ -12,12 +12,14 @@ import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.assets.image import ImageAsset
|
||||
@@ -81,6 +83,21 @@ class WeightCacheDaemon:
|
||||
f"Weight cache daemon is not ready: HTTP {error.code}"
|
||||
) from error
|
||||
|
||||
def wait_until_ready(self, timeout_s: float = 900) -> None:
|
||||
"""Poll until READY (which covers the autotune tuners) or the preload
|
||||
process dies."""
|
||||
deadline = time.monotonic() + timeout_s
|
||||
while True:
|
||||
try:
|
||||
self.check_health()
|
||||
return
|
||||
except (AssertionError, urllib.error.URLError):
|
||||
if time.monotonic() > deadline:
|
||||
raise
|
||||
if self._proc is not None and self._proc.poll() is not None:
|
||||
raise AssertionError("vllm preload exited before ready") from None
|
||||
time.sleep(2)
|
||||
|
||||
def _stop(self) -> None:
|
||||
assert self._proc is not None
|
||||
self._proc.terminate()
|
||||
@@ -99,6 +116,10 @@ class ModelCase:
|
||||
images: list | None = None
|
||||
llm_kwargs: dict[str, Any] = field(default_factory=dict)
|
||||
daemon_args: list[str] = field(default_factory=list)
|
||||
# Additionally assert the daemons tune in place before serving and write
|
||||
# the on-disk FlashInfer autotune cache, and the restart engine reuses it
|
||||
# (needs SM90+/FlashInfer).
|
||||
check_flashinfer_cache: bool = False
|
||||
|
||||
|
||||
def generate(
|
||||
@@ -157,6 +178,9 @@ K3_CASE = ModelCase(
|
||||
max_model_len=4096,
|
||||
),
|
||||
daemon_args=["--trust-remote-code"],
|
||||
# Kimi K3 has tunable FlashInfer ops (a dense model like Qwen2.5 tunes
|
||||
# nothing and upstream never writes the cache file for it).
|
||||
check_flashinfer_cache=True,
|
||||
)
|
||||
|
||||
# Qwen3.5-0.8B ships one MTP layer in the target checkpoint, so method="mtp"
|
||||
@@ -186,18 +210,38 @@ QWEN_MTP_CASE = ModelCase(
|
||||
[QWEN_CASE, K3_CASE, QWEN_MTP_CASE],
|
||||
ids=["qwen3.5", "kimi-k3", "qwen3.5-mtp"],
|
||||
)
|
||||
def test_ipc_cache_cold_start_and_warm_restart(vllm_runner, case: ModelCase):
|
||||
def test_ipc_cache_cold_start_and_warm_restart(
|
||||
vllm_runner, case: ModelCase, tmp_path, monkeypatch, capfd
|
||||
):
|
||||
"""Cold start falls back to disk; warm restarts load weights via CUDA IPC.
|
||||
|
||||
All runs must produce outputs identical to a default-loader baseline. The
|
||||
warm runs disable the disk fallback, so they only pass if the weights
|
||||
really came from the daemon — for the MTP case, both the target's and the
|
||||
draft's daemon groups.
|
||||
|
||||
Cases with ``check_flashinfer_cache`` additionally cover in-daemon
|
||||
autotune (default-on with `--enable-flashinfer-autotune`): each daemon
|
||||
tunes before binding its socket and must write the on-disk FlashInfer
|
||||
autotune cache, and the restart engine must load tactics from it instead
|
||||
of re-profiling. The daemon's cache file is keyed with the
|
||||
OPENAI_API_SERVER batch defaults while these engines run under LLM_CLASS,
|
||||
so they never hit the daemon's file by design; reuse is proven against
|
||||
the file the earlier in-process engines wrote.
|
||||
"""
|
||||
if not current_platform.is_cuda_alike():
|
||||
pytest.skip("Weight cache IPC sharing requires CUDA or ROCm")
|
||||
if case is K3_CASE and not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("Kimi K3 IPC weight cache requires an SM100 MXFP4 backend")
|
||||
if case.check_flashinfer_cache:
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
|
||||
if not (has_flashinfer() and current_platform.has_device_capability(90)):
|
||||
pytest.skip("FlashInfer autotune requires FlashInfer and SM90+")
|
||||
# Isolate the on-disk autotune cache. The engines are in-process
|
||||
# (envs read lazily); the daemon subprocesses spawn later and
|
||||
# inherit it.
|
||||
monkeypatch.setenv("VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR", str(tmp_path))
|
||||
|
||||
# Baseline: plain disk loading with the default loader.
|
||||
baseline_outputs = generate(
|
||||
@@ -212,15 +256,40 @@ def test_ipc_cache_cold_start_and_warm_restart(vllm_runner, case: ModelCase):
|
||||
with tempfile.TemporaryDirectory(prefix="vllm_ipc_empty_") as empty_socket_dir:
|
||||
cold_outputs = generate(vllm_runner, case, empty_socket_dir, fallback=True)
|
||||
|
||||
if case.check_flashinfer_cache:
|
||||
# The baseline engine already wrote its own (LLM_CLASS-keyed) table.
|
||||
engine_cache_files = set(tmp_path.rglob("autotune_configs*.json"))
|
||||
|
||||
with WeightCacheDaemon(
|
||||
case.model,
|
||||
tp_size=1,
|
||||
extra_args=case.daemon_args,
|
||||
) as d:
|
||||
warm_outputs = generate(vllm_runner, case, d.socket_dir, fallback=False)
|
||||
d.check_health()
|
||||
d.wait_until_ready()
|
||||
if case.check_flashinfer_cache:
|
||||
# Each daemon tunes before its ready report; exactly one new file
|
||||
# (their OPENAI_API_SERVER-keyed table) must appear.
|
||||
tuner_files = set(tmp_path.rglob("autotune_configs*.json"))
|
||||
tuner_files -= engine_cache_files
|
||||
assert len(tuner_files) == 1, (
|
||||
f"expected the tuner's tuned table, got {tuner_files}"
|
||||
)
|
||||
# Warm restart: a second engine lifetime against the same daemon.
|
||||
if case.check_flashinfer_cache:
|
||||
capfd.readouterr() # drain: the assertion reads the increment
|
||||
restart_outputs = generate(vllm_runner, case, d.socket_dir, fallback=False)
|
||||
if case.check_flashinfer_cache:
|
||||
# The restart's EngineCore subprocess inherits pytest's fds; the
|
||||
# flashinfer autotuner logs "[Autotuner]: Loaded N configs from
|
||||
# <path>" (INFO, not once-deduplicated) on a disk cache hit.
|
||||
out = capfd.readouterr()
|
||||
m = re.search(
|
||||
r"\[Autotuner\]: Loaded (\d+) configs from", out.out + out.err
|
||||
)
|
||||
assert m and int(m.group(1)) > 0, (
|
||||
"restart engine did not load the on-disk autotune cache"
|
||||
)
|
||||
|
||||
assert cold_outputs == baseline_outputs
|
||||
assert warm_outputs == baseline_outputs
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.model_executor.warmup import attention_warmup
|
||||
from vllm.model_executor.warmup.attention_warmup import mixed_batch_attention_warmup
|
||||
|
||||
|
||||
def _worker(*, use_v2: bool, max_num_batched_tokens: int, max_model_len: int = 4096):
|
||||
calls = []
|
||||
worker = SimpleNamespace(
|
||||
model_runner=SimpleNamespace(_dummy_run=lambda **kw: calls.append(kw)),
|
||||
use_v2_model_runner=use_v2,
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=max_num_batched_tokens),
|
||||
model_config=SimpleNamespace(max_model_len=max_model_len),
|
||||
execute_model=object(),
|
||||
sample_tokens=object(),
|
||||
)
|
||||
return worker, calls
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("max_num_batched_tokens", "max_model_len", "expected"),
|
||||
[(8192, 4096, 16), (6, 4096, 6), (8192, 8, 8)],
|
||||
)
|
||||
def test_v1_runs_mixed_dummy_batch_within_limits(
|
||||
max_num_batched_tokens, max_model_len, expected
|
||||
):
|
||||
worker, calls = _worker(
|
||||
use_v2=False,
|
||||
max_num_batched_tokens=max_num_batched_tokens,
|
||||
max_model_len=max_model_len,
|
||||
)
|
||||
mixed_batch_attention_warmup(worker)
|
||||
assert calls == [
|
||||
dict(
|
||||
num_tokens=expected,
|
||||
skip_eplb=True,
|
||||
is_profile=True,
|
||||
force_attention=True,
|
||||
create_mixed_batch=True,
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("max_num_batched_tokens", "expected"), [(8192, 16), (6, 6)])
|
||||
@pytest.mark.parametrize("ran", [True, False])
|
||||
def test_v2_runs_mixed_prefill_decode_step(
|
||||
monkeypatch, max_num_batched_tokens, expected, ran
|
||||
):
|
||||
worker, dummy_calls = _worker(
|
||||
use_v2=True, max_num_batched_tokens=max_num_batched_tokens
|
||||
)
|
||||
mixed_calls = []
|
||||
|
||||
def fake_mixed_warmup(runner, execute_model, sample_tokens, num_tokens):
|
||||
mixed_calls.append((runner, execute_model, sample_tokens, num_tokens))
|
||||
return ran
|
||||
|
||||
monkeypatch.setattr(
|
||||
attention_warmup, "run_mixed_prefill_decode_warmup", fake_mixed_warmup
|
||||
)
|
||||
mixed_batch_attention_warmup(worker)
|
||||
|
||||
assert mixed_calls == [
|
||||
(worker.model_runner, worker.execute_model, worker.sample_tokens, expected)
|
||||
]
|
||||
# No dummy-run fallback when the mixed step is skipped.
|
||||
assert dummy_calls == []
|
||||
@@ -25,6 +25,60 @@ def test_b12x_warmup_token_counts_cover_serving_regimes() -> None:
|
||||
) == (1, 2, 8, 128, 2048)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_embedding", [False, True])
|
||||
def test_b12x_block_quant_warmup_covers_each_layers_plans(monkeypatch, is_embedding):
|
||||
from vllm.model_executor.layers.quantization.modelopt_block_quant import (
|
||||
ModelOptBlockQuantLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.warmup.b12x_warmup import _collect_warmup_units
|
||||
|
||||
method = object.__new__(ModelOptBlockQuantLinearMethod)
|
||||
method.codec = "q8_0"
|
||||
method.is_embedding = is_embedding
|
||||
model = torch.nn.ModuleList([torch.nn.Module(), torch.nn.Module()])
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
method, "apply", lambda layer, x: calls.append((layer, x.shape, x.dtype))
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
method,
|
||||
"embedding",
|
||||
lambda layer, ids: calls.append((layer, ids.shape, ids.dtype)),
|
||||
)
|
||||
for index, layer in enumerate(model):
|
||||
layer.b12x_warmup_provider = method
|
||||
if is_embedding:
|
||||
layer.weight = torch.nn.Parameter(
|
||||
torch.empty((8, 8, 34), dtype=torch.uint8), requires_grad=False
|
||||
)
|
||||
layer.b12x_embedding_plans = {
|
||||
dtype: SimpleNamespace(
|
||||
handle=2 * index + offset, query=SimpleNamespace(max_rows=32)
|
||||
)
|
||||
for offset, dtype in enumerate((torch.int32, torch.int64))
|
||||
}
|
||||
else:
|
||||
layer.b12x_block_weight = SimpleNamespace(
|
||||
in_features=256, values=torch.empty(1)
|
||||
)
|
||||
layer.b12x_block_plan = SimpleNamespace(
|
||||
handle=index, token_counts=(1, 4, 32)
|
||||
)
|
||||
|
||||
units = list(_collect_warmup_units(model, (1, 17, 32), torch.bfloat16))
|
||||
assert len(units) == 2
|
||||
for unit in units:
|
||||
unit.compile()
|
||||
expected = (
|
||||
[((32,), torch.int32), ((32,), torch.int64)]
|
||||
if is_embedding
|
||||
else [((rows, 256), torch.bfloat16) for rows in (1, 4, 17, 32)]
|
||||
)
|
||||
assert calls == [
|
||||
(layer, shape, dtype) for layer in model for shape, dtype in expected
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kernel_cls", "module_name", "call_name", "layer", "name"),
|
||||
[
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
@@ -88,3 +89,53 @@ def test_kernel_registers_itself_as_warmup_provider(is_bmm) -> None:
|
||||
|
||||
provider = getattr(layer, "deep_gemm_warmup_provider", None)
|
||||
assert provider is (None if is_bmm else kernel)
|
||||
|
||||
|
||||
def test_mega_moe_predicate_matches_only_native_mega_moe_layers(monkeypatch) -> None:
|
||||
"""Without the MegaMoE model modules loaded, nothing matches and nothing is
|
||||
imported; with them loaded, only DeepSeek-V4 and Kimi K3 MegaMoE layers match."""
|
||||
name = "vllm.models.deepseek_v4.nvidia.model"
|
||||
kimi_name = "vllm.models.kimi_k3.nvidia.model"
|
||||
monkeypatch.delitem(sys.modules, name, raising=False)
|
||||
monkeypatch.delitem(sys.modules, kimi_name, raising=False)
|
||||
other = SimpleNamespace(use_native_mega_moe=True, use_mega_moe=True)
|
||||
assert not deep_gemm_warmup._mega_moe_may_use_deep_gemm(other)
|
||||
assert name not in sys.modules and kimi_name not in sys.modules
|
||||
|
||||
moe_cls = type("DeepseekV4MoE", (), {})
|
||||
monkeypatch.setitem(sys.modules, name, SimpleNamespace(DeepseekV4MoE=moe_cls))
|
||||
native, fused = moe_cls(), moe_cls()
|
||||
native.use_native_mega_moe, fused.use_native_mega_moe = True, False
|
||||
assert deep_gemm_warmup._mega_moe_may_use_deep_gemm(native)
|
||||
assert not deep_gemm_warmup._mega_moe_may_use_deep_gemm(fused)
|
||||
assert not deep_gemm_warmup._mega_moe_may_use_deep_gemm(other)
|
||||
|
||||
kimi_cls = type("KimiMoE", (), {})
|
||||
monkeypatch.setitem(sys.modules, kimi_name, SimpleNamespace(KimiMoE=kimi_cls))
|
||||
kimi_mega, kimi_fused = kimi_cls(), kimi_cls()
|
||||
kimi_mega.use_mega_moe, kimi_fused.use_mega_moe = True, False
|
||||
assert deep_gemm_warmup._mega_moe_may_use_deep_gemm(kimi_mega)
|
||||
assert not deep_gemm_warmup._mega_moe_may_use_deep_gemm(kimi_fused)
|
||||
|
||||
|
||||
def test_mega_moe_m_values_reach_mega_gate(monkeypatch) -> None:
|
||||
"""Each bf16_mega_gate config is warmed with an M above the GateLinear
|
||||
threshold (1 for 128 experts), so the forward actually reaches the kernel."""
|
||||
dg = SimpleNamespace(
|
||||
get_bf16_mega_gate_config=lambda m, *_: {"num_sms": -(-m // 16)},
|
||||
get_block_m_for_mega_moe=lambda *_: 16,
|
||||
)
|
||||
monkeypatch.setattr(deep_gemm_warmup, "_import_deep_gemm", lambda: dg)
|
||||
monkeypatch.setattr(deep_gemm_warmup, "_mega_moe_uses_mega_gate", lambda _: True)
|
||||
monkeypatch.setattr(
|
||||
deep_gemm_warmup, "get_ep_group", lambda: SimpleNamespace(world_size=8)
|
||||
)
|
||||
buffer = SimpleNamespace(num_max_tokens_per_rank=64)
|
||||
module = SimpleNamespace(
|
||||
gate=SimpleNamespace(input_size=4096, output_size=128),
|
||||
experts=SimpleNamespace(
|
||||
top_k=6, num_experts=128, get_symm_buffer=lambda: buffer
|
||||
),
|
||||
use_sequence_parallel=False,
|
||||
)
|
||||
assert deep_gemm_warmup._get_fp8_fp4_mega_moe_m_values(module, 40) == [16, 32, 40]
|
||||
|
||||
@@ -386,3 +386,34 @@ def test_rejected_cache_retunes_every_rank_in_its_tuning_group(autotune_run, pp,
|
||||
rerun.assert_collectives_match()
|
||||
assert set(rerun.profile_groups) == set(range(tp))
|
||||
assert {rank for rank, _, _ in rerun.saves} == set(range(tp))
|
||||
|
||||
|
||||
def test_autotune_cache_key_normalizes_kv_init_fields():
|
||||
"""mamba_block_size/kv_cache_layout only resolve at engine KV-init time;
|
||||
they must not feed the autotune cache key, or nothing can compute the key
|
||||
before startup (the daemon tunes before any KV init). block_size must
|
||||
stay in the key: the daemon replays the engine's block-size resolution
|
||||
before tuning, and different block sizes may select different ops."""
|
||||
from vllm.config.cache import CacheConfig
|
||||
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
|
||||
_normalize_cache_config_for_hash,
|
||||
)
|
||||
|
||||
fresh = CacheConfig()
|
||||
# Simulate the engine's KV-init write-backs (hybrid model values).
|
||||
mutated = CacheConfig()
|
||||
mutated.mamba_block_size = 512
|
||||
mutated.kv_cache_layout = "LBHNC"
|
||||
|
||||
# Negative control: the write-backs are hashed factors.
|
||||
assert mutated.compute_hash() != fresh.compute_hash()
|
||||
normalized = _normalize_cache_config_for_hash(mutated)
|
||||
assert normalized.compute_hash() == fresh.compute_hash()
|
||||
# Factory state is returned untouched.
|
||||
assert _normalize_cache_config_for_hash(fresh) is fresh
|
||||
# block_size stays significant (the daemon keys on the resolved value).
|
||||
resolved = CacheConfig(block_size=64)
|
||||
assert (
|
||||
_normalize_cache_config_for_hash(resolved).compute_hash()
|
||||
!= fresh.compute_hash()
|
||||
)
|
||||
|
||||
@@ -553,12 +553,10 @@ def test_compute_slot_mapping_uses_named_launcher_inputs(monkeypatch) -> None:
|
||||
owner = ComputeSlotMappingKernel()
|
||||
compile_key = owner.CompileKey(
|
||||
kv_cache_block_size=16,
|
||||
blocks_per_kv_block=1,
|
||||
total_cp_world_size=2,
|
||||
total_cp_rank=1,
|
||||
cp_kv_cache_interleave_size=1,
|
||||
block_table_stride=128,
|
||||
block_size=16,
|
||||
)
|
||||
launches: list[tuple[Any, ...]] = []
|
||||
|
||||
@@ -574,7 +572,6 @@ def test_compute_slot_mapping_uses_named_launcher_inputs(monkeypatch) -> None:
|
||||
assert inputs["block_table_stride"] == 128
|
||||
assert kwargs == {
|
||||
"KV_CACHE_BLOCK_SIZE": 16,
|
||||
"BLOCKS_PER_KV_BLOCK": 1,
|
||||
"TOTAL_CP_WORLD_SIZE": 2,
|
||||
"TOTAL_CP_RANK": 1,
|
||||
"CP_KV_CACHE_INTERLEAVE_SIZE": 1,
|
||||
|
||||
@@ -6,6 +6,9 @@ import torch
|
||||
from vllm.model_executor.models.minicpmv import MiniCPMV2_6
|
||||
from vllm.model_executor.models.minicpmv4_6 import (
|
||||
MiniCPMV4_6ForConditionalGeneration,
|
||||
_expand_flags,
|
||||
_select_vision_items,
|
||||
_vision_downsample_mode,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
@@ -54,3 +57,114 @@ def test_image_and_video_embeds_stay_in_their_own_modality():
|
||||
|
||||
assert torch.equal(modalities["images"]["image_embeds"], image_embeds)
|
||||
assert torch.equal(modalities["videos"]["image_embeds"], video_embeds)
|
||||
|
||||
|
||||
class TestPerItemMergerFlags:
|
||||
"""The merger flag is per request but items from several requests can be
|
||||
batched together, so it has to be applied per item."""
|
||||
|
||||
def test_expand_flags_broadcasts_a_single_value(self):
|
||||
assert _expand_flags(None, 3) == [None, None, None]
|
||||
assert _expand_flags(torch.tensor([True]), 3) == [True, True, True]
|
||||
assert _expand_flags(True, 2) == [True, True]
|
||||
|
||||
def test_expand_flags_keeps_per_item_values(self):
|
||||
flags = [torch.tensor([True]), torch.tensor([False])]
|
||||
|
||||
assert _expand_flags(flags, 2) == [True, False]
|
||||
|
||||
def test_expand_flags_falls_back_on_a_shape_mismatch(self):
|
||||
# Too many values to map onto the items: keep one decision for the
|
||||
# batch rather than failing the request.
|
||||
assert _expand_flags([True, False, True], 2) == [True, True]
|
||||
|
||||
def test_vision_downsample_mode(self):
|
||||
assert _vision_downsample_mode(None) is None
|
||||
assert _vision_downsample_mode(True) == "16x"
|
||||
assert _vision_downsample_mode(False) == "4x"
|
||||
|
||||
def test_select_vision_items_slices_by_item(self):
|
||||
# 3 items holding 2, 1 and 2 slices; one channel, one patch, width 4.
|
||||
pixel_values = [torch.full((1, 1, 4), float(i)) for i in range(5)]
|
||||
tgt_sizes = torch.arange(10, dtype=torch.long).reshape(5, 2)
|
||||
image_input = {
|
||||
"pixel_values": pixel_values,
|
||||
"tgt_sizes": tgt_sizes,
|
||||
"num_slices": torch.tensor([2, 1, 2]),
|
||||
}
|
||||
|
||||
selected = _select_vision_items(image_input, [2], [2, 1, 2])
|
||||
|
||||
assert len(selected["pixel_values"]) == 2
|
||||
assert bool(selected["pixel_values"][0][0, 0, 0] == 3.0)
|
||||
assert bool(selected["pixel_values"][1][0, 0, 0] == 4.0)
|
||||
assert selected["num_slices"].tolist() == [2]
|
||||
assert torch.equal(selected["tgt_sizes"], tgt_sizes[3:5])
|
||||
|
||||
def test_select_vision_items_preserves_order(self):
|
||||
pixel_values = [torch.full((1, 1, 4), float(i)) for i in range(3)]
|
||||
image_input = {
|
||||
"pixel_values": pixel_values,
|
||||
"tgt_sizes": torch.zeros(3, 2, dtype=torch.long),
|
||||
"num_slices": torch.tensor([1, 1, 1]),
|
||||
}
|
||||
|
||||
selected = _select_vision_items(image_input, [2, 0], [1, 1, 1])
|
||||
|
||||
assert selected["num_slices"].tolist() == [1, 1]
|
||||
assert bool(selected["pixel_values"][0][0, 0, 0] == 2.0)
|
||||
assert bool(selected["pixel_values"][1][0, 0, 0] == 0.0)
|
||||
|
||||
def test_process_vision_input_applies_flags_per_item(self):
|
||||
calls: list[str | None] = []
|
||||
|
||||
def fake_get_vision_hidden_states(data, downsample_mode=None):
|
||||
calls.append(downsample_mode)
|
||||
tag = 1.0 if downsample_mode == "16x" else 0.0
|
||||
return [torch.full((1,), tag) for _ in data["pixel_values"]]
|
||||
|
||||
model = object.__new__(MiniCPMV4_6ForConditionalGeneration)
|
||||
model.get_vision_hidden_states = fake_get_vision_hidden_states
|
||||
|
||||
image_input = {
|
||||
"type": "pixel_values",
|
||||
"pixel_values": [torch.zeros(1, 1, 4) for _ in range(3)],
|
||||
"tgt_sizes": torch.zeros(3, 2, dtype=torch.long),
|
||||
"num_slices": torch.tensor([1, 1, 1]),
|
||||
}
|
||||
|
||||
result = model._process_vision_input(
|
||||
image_input,
|
||||
use_vit_merger=[
|
||||
torch.tensor([True]),
|
||||
torch.tensor([False]),
|
||||
torch.tensor([True]),
|
||||
],
|
||||
)
|
||||
|
||||
# Each item keeps its own mode even though they share one call.
|
||||
assert [float(r[0]) for r in result] == [1.0, 0.0, 1.0]
|
||||
# Mixed flags are encoded as separate groups, not one collapsed batch.
|
||||
assert calls == ["16x", "4x"]
|
||||
|
||||
def test_process_vision_input_uses_one_group_for_uniform_flags(self):
|
||||
calls: list[str | None] = []
|
||||
|
||||
def fake_get_vision_hidden_states(data, downsample_mode=None):
|
||||
calls.append(downsample_mode)
|
||||
return [torch.zeros(1) for _ in data["pixel_values"]]
|
||||
|
||||
model = object.__new__(MiniCPMV4_6ForConditionalGeneration)
|
||||
model.get_vision_hidden_states = fake_get_vision_hidden_states
|
||||
|
||||
image_input = {
|
||||
"type": "pixel_values",
|
||||
"pixel_values": [torch.zeros(1, 1, 4) for _ in range(2)],
|
||||
"tgt_sizes": torch.zeros(2, 2, dtype=torch.long),
|
||||
"num_slices": torch.tensor([1, 1]),
|
||||
}
|
||||
|
||||
model._process_vision_input(image_input, use_vit_merger=torch.tensor([True]))
|
||||
|
||||
# The common case still makes a single encoder call.
|
||||
assert calls == ["16x"]
|
||||
|
||||
@@ -0,0 +1,310 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.minicpmv4_6 import _stack_vit_merger_qkv
|
||||
from vllm.model_executor.models.minicpmv4_7 import (
|
||||
MiniCPMV4_7MultiModalProcessor,
|
||||
_compute_canvas_single,
|
||||
build_image_bounds,
|
||||
canvas_rope_delta,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
|
||||
def test_canvas_mrope_sizes_survive_prefix_cache_stripping():
|
||||
"""A cached image must keep `tgt_sizes`, or canvas M-RoPE falls back.
|
||||
|
||||
`strip_covered_mm_data` drops the payload of a prefix-cache-covered item and
|
||||
keeps only CPU-side metadata. Canvas M-RoPE places slices from `tgt_sizes`,
|
||||
so 4.7 marks it `keep_on_cpu` while the shared 4.6 config stays as it was.
|
||||
"""
|
||||
from vllm.model_executor.models.minicpmv4_6 import _minicpmv4_6_field_config
|
||||
|
||||
shared = _minicpmv4_6_field_config({})
|
||||
assert shared["tgt_sizes"].field.keep_on_cpu is False
|
||||
assert shared["video_tgt_sizes"].field.keep_on_cpu is False
|
||||
|
||||
processor = MiniCPMV4_7MultiModalProcessor.__new__(MiniCPMV4_7MultiModalProcessor)
|
||||
fields = processor._get_mm_fields_config({}, {})
|
||||
|
||||
assert fields["tgt_sizes"].field.keep_on_cpu is True
|
||||
assert fields["video_tgt_sizes"].field.keep_on_cpu is True
|
||||
# Other fields keep their previous setting.
|
||||
assert fields["pixel_values"].field.keep_on_cpu is False
|
||||
assert fields["video_image_sizes"].field.keep_on_cpu is True
|
||||
|
||||
|
||||
def test_modality_mm_kwargs_resolution():
|
||||
"""Flat keys reach both modalities; scopes stay scoped, and win on conflict.
|
||||
|
||||
`_merge_and_resolve_mm_processor_kwargs` routes a flat key into every scope
|
||||
whose HF processor declares it, so reading is a plain scope lookup. The
|
||||
routing itself is upstream's, tested here against the MiniCPM-V schema to
|
||||
pin the precedence rule (`scoped > flat`).
|
||||
"""
|
||||
from vllm.model_executor.models.minicpmv4_6 import (
|
||||
_flat_processor_kwargs,
|
||||
_scoped_mm_kwarg,
|
||||
)
|
||||
from vllm.multimodal.processing.context import _resolve_mm_processor_kwargs
|
||||
|
||||
# What MiniCPMV4_6{Image,Video}ProcessorKwargs declare.
|
||||
processor_keys = {"downsample_mode", "max_slice_nums"}
|
||||
schema = {
|
||||
"text_kwargs": set(),
|
||||
"images_kwargs": set(processor_keys),
|
||||
"videos_kwargs": set(processor_keys),
|
||||
"audio_kwargs": set(),
|
||||
}
|
||||
|
||||
def read(mm_kwargs, modality, key):
|
||||
merged = _resolve_mm_processor_kwargs(dict(mm_kwargs), schema)
|
||||
return _scoped_mm_kwarg(merged, modality, key)
|
||||
|
||||
assert read({}, "image", "downsample_mode") is None
|
||||
# A flat key is routed into both scopes.
|
||||
assert read({"downsample_mode": "4x"}, "image", "downsample_mode") == "4x"
|
||||
assert read({"downsample_mode": "4x"}, "video", "downsample_mode") == "4x"
|
||||
# A scoped key stays in its scope.
|
||||
assert (
|
||||
read({"images_kwargs": {"downsample_mode": "16x"}}, "image", "downsample_mode")
|
||||
== "16x"
|
||||
)
|
||||
assert (
|
||||
read({"images_kwargs": {"downsample_mode": "16x"}}, "video", "downsample_mode")
|
||||
is None
|
||||
)
|
||||
videos_only = {"videos_kwargs": {"max_slice_nums": 1}}
|
||||
assert read(videos_only, "video", "max_slice_nums") == 1
|
||||
# Scoped to videos, so image must not see it.
|
||||
assert read(videos_only, "image", "max_slice_nums") is None
|
||||
# An explicit scope wins over the flat key it is merged with.
|
||||
assert (
|
||||
read(
|
||||
{"downsample_mode": "16x", "images_kwargs": {"downsample_mode": "4x"}},
|
||||
"image",
|
||||
"downsample_mode",
|
||||
)
|
||||
== "4x"
|
||||
)
|
||||
# A non-mapping scope is ignored rather than raising.
|
||||
assert read({"images_kwargs": None}, "image", "downsample_mode") is None
|
||||
|
||||
# `_flat_processor_kwargs` drops the scoped keys and pins the mode.
|
||||
flat = _flat_processor_kwargs(
|
||||
{
|
||||
"downsample_mode": "4x",
|
||||
"images_kwargs": {"a": 1},
|
||||
"videos_kwargs": {"b": 2},
|
||||
"audio_kwargs": {"c": 3},
|
||||
},
|
||||
"16x",
|
||||
)
|
||||
assert flat == {"downsample_mode": "16x"}
|
||||
|
||||
|
||||
def test_vit_merger_qkv_is_stacked_before_load():
|
||||
mapped = list(
|
||||
_stack_vit_merger_qkv(
|
||||
[
|
||||
(
|
||||
"model.vision_tower.vit_merger.self_attn.q_proj.weight",
|
||||
torch.zeros(1),
|
||||
),
|
||||
(
|
||||
"model.vision_tower.vit_merger.self_attn.k_proj.weight",
|
||||
torch.zeros(1),
|
||||
),
|
||||
(
|
||||
"model.vision_tower.vit_merger.self_attn.v_proj.weight",
|
||||
torch.zeros(1),
|
||||
),
|
||||
("vit_merger.self_attn.qkv_proj.weight", torch.zeros(1)),
|
||||
]
|
||||
)
|
||||
)
|
||||
names = [name for name, _ in mapped]
|
||||
shards = [getattr(tensor, "shard_id", None) for _, tensor in mapped]
|
||||
assert names == [
|
||||
"model.vision_tower.vit_merger.self_attn.qkv_proj.weight",
|
||||
"model.vision_tower.vit_merger.self_attn.qkv_proj.weight",
|
||||
"model.vision_tower.vit_merger.self_attn.qkv_proj.weight",
|
||||
"vit_merger.self_attn.qkv_proj.weight",
|
||||
]
|
||||
assert shards == ["q", "k", "v", None]
|
||||
|
||||
|
||||
# Canvas M-RoPE goldens, cross-checked against the reference implementation in
|
||||
# transformers PR 48979 (`modular_minicpmv4_7.py`). Synthetic ids keep the token
|
||||
# sequences readable: the canvas math only compares ids for equality.
|
||||
_IM_START = 10
|
||||
_IM_END = 11
|
||||
_SLICE_START = 12
|
||||
_SLICE_END = 13
|
||||
_NEWLINE = 14
|
||||
_IMAGE_PAD = 20
|
||||
_VIDEO_PAD = 21
|
||||
_TEXT = 100
|
||||
|
||||
_CANVAS_IDS = {
|
||||
"im_start_id": _IM_START,
|
||||
"im_end_id": _IM_END,
|
||||
"slice_start_id": _SLICE_START,
|
||||
"slice_end_id": _SLICE_END,
|
||||
"newline_id": _NEWLINE,
|
||||
}
|
||||
|
||||
|
||||
def _canvas_positions(tokens, target_sizes):
|
||||
input_ids = torch.tensor([tokens], dtype=torch.long)
|
||||
targets = torch.tensor(target_sizes, dtype=torch.long)
|
||||
bounds = build_image_bounds(input_ids[0], _CANVAS_IDS)
|
||||
positions = _compute_canvas_single(
|
||||
input_ids[0],
|
||||
torch.arange(len(tokens), dtype=torch.long),
|
||||
bounds,
|
||||
targets,
|
||||
_CANVAS_IDS,
|
||||
)
|
||||
return positions.tolist(), canvas_rope_delta(positions, len(tokens))
|
||||
|
||||
|
||||
def test_canvas_mrope_trailing_newline_stays_inside_the_canvas():
|
||||
"""A newline right after `</image>` belongs to the canvas, not to the text.
|
||||
|
||||
The chat template joins content parts with ``\\n``, so an image followed by
|
||||
text always leaves one there. The transformers reference closes a frame over
|
||||
every marker including newlines; leaving it out pushes every later position
|
||||
one step forward and makes `rope_delta` differ by one.
|
||||
"""
|
||||
tokens = [_TEXT, _IM_START, _IMAGE_PAD, _IM_END, _NEWLINE, _TEXT, _TEXT]
|
||||
positions, delta = _canvas_positions(tokens, [[4, 4]])
|
||||
|
||||
assert positions == [
|
||||
[0, 1, 1, 1, 1, 3, 4],
|
||||
[0, 0, 1, 2, 1, 3, 4],
|
||||
[0, 0, 1, 2, 1, 3, 4],
|
||||
]
|
||||
assert delta == -2
|
||||
|
||||
|
||||
def test_canvas_mrope_thumbnail_matches_reference():
|
||||
# <text> <image> PAD </image> <text>; 16x turns a 4x4 patch grid into 1 token.
|
||||
tokens = [_TEXT, _IM_START, _IMAGE_PAD, _IM_END, _TEXT]
|
||||
positions, delta = _canvas_positions(tokens, [[4, 4]])
|
||||
|
||||
assert positions == [
|
||||
[0, 1, 1, 1, 3],
|
||||
[0, 0, 1, 2, 3],
|
||||
[0, 0, 1, 2, 3],
|
||||
]
|
||||
assert delta == -1
|
||||
|
||||
|
||||
def test_canvas_mrope_slice_row_matches_reference():
|
||||
# Thumbnail then a 1x2 slice row; the slices tile the thumbnail's canvas.
|
||||
tokens = [
|
||||
_TEXT,
|
||||
_IM_START,
|
||||
_IMAGE_PAD,
|
||||
_IM_END,
|
||||
_SLICE_START,
|
||||
_IMAGE_PAD,
|
||||
_SLICE_END,
|
||||
_SLICE_START,
|
||||
_IMAGE_PAD,
|
||||
_SLICE_END,
|
||||
_TEXT,
|
||||
]
|
||||
positions, delta = _canvas_positions(tokens, [[4, 4], [4, 4], [4, 4]])
|
||||
|
||||
assert positions == [
|
||||
[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 4],
|
||||
[0, 0, 1, 2, 1, 1, 1, 1, 1, 1, 4],
|
||||
[0, 0, 1, 3, 1, 1, 1, 2, 2, 2, 4],
|
||||
]
|
||||
assert delta == -6
|
||||
|
||||
|
||||
def test_canvas_mrope_resamples_thumbnail_over_canvas():
|
||||
# A 8x8 patch grid is 2x2 LLM tokens, so the canvas is resampled (not arange).
|
||||
tokens = [_TEXT, _IM_START] + [_IMAGE_PAD] * 4 + [_IM_END, _TEXT]
|
||||
positions, delta = _canvas_positions(tokens, [[8, 8]])
|
||||
|
||||
assert positions == [
|
||||
[0, 1, 1, 1, 1, 1, 1, 4],
|
||||
[0, 0, 1, 1, 2, 2, 3, 4],
|
||||
[0, 0, 1, 2, 1, 2, 3, 4],
|
||||
]
|
||||
assert delta == -3
|
||||
|
||||
|
||||
def test_canvas_mrope_video_frames_share_one_canvas():
|
||||
tokens = [
|
||||
_TEXT,
|
||||
_IM_START,
|
||||
_VIDEO_PAD,
|
||||
_IM_END,
|
||||
_IM_START,
|
||||
_VIDEO_PAD,
|
||||
_IM_END,
|
||||
_TEXT,
|
||||
]
|
||||
positions, delta = _canvas_positions(tokens, [[4, 4], [4, 4]])
|
||||
|
||||
assert positions == [
|
||||
[0, 1, 1, 1, 3, 3, 3, 5],
|
||||
[0, 0, 1, 2, 2, 3, 4, 5],
|
||||
[0, 0, 1, 2, 2, 3, 4, 5],
|
||||
]
|
||||
assert delta == -2
|
||||
|
||||
|
||||
def test_canvas_mrope_interleaved_image_and_video():
|
||||
tokens = (
|
||||
[_TEXT, _IM_START]
|
||||
+ [_IMAGE_PAD] * 4
|
||||
+ [_IM_END, _IM_START]
|
||||
+ [_VIDEO_PAD] * 4
|
||||
+ [_IM_END, _TEXT]
|
||||
)
|
||||
positions, delta = _canvas_positions(tokens, [[8, 8], [8, 8]])
|
||||
|
||||
assert positions == [
|
||||
[0, 1, 1, 1, 1, 1, 1, 4, 4, 4, 4, 4, 4, 7],
|
||||
[0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7],
|
||||
[0, 0, 1, 2, 1, 2, 3, 3, 4, 5, 4, 5, 6, 7],
|
||||
]
|
||||
assert delta == -6
|
||||
|
||||
|
||||
def test_canvas_span_is_shared_by_image_and_video():
|
||||
"""A video frame and a same-geometry image must place identical offsets.
|
||||
|
||||
Both paths go through `_assign_canvas_span`; this pins that they cannot
|
||||
drift apart now that the duplicated geometry is gone.
|
||||
"""
|
||||
thumb = [_IM_START] + [_IMAGE_PAD] * 4 + [_IM_END]
|
||||
|
||||
img_pos, _ = _canvas_positions([_TEXT, *thumb, _TEXT], [[8, 8]])
|
||||
vid_pos, _ = _canvas_positions([_TEXT, *thumb, *thumb, _TEXT], [[8, 8], [8, 8]])
|
||||
|
||||
def offsets(pos, start):
|
||||
# The <im_start> halo is one below the canvas base.
|
||||
base = pos[1][start] + 1
|
||||
return [[c[start + k] - base for k in range(3)] for c in pos]
|
||||
|
||||
assert offsets(img_pos, 0) == offsets(vid_pos, 0)
|
||||
|
||||
|
||||
def test_canvas_mrope_delta_leaves_room_for_the_first_decoded_token():
|
||||
# First decoded token must land on max_prefill_position + 1, matching
|
||||
# transformers and the other native M-RoPE models.
|
||||
tokens = [_TEXT, _IM_START, _IMAGE_PAD, _IM_END, _TEXT]
|
||||
positions, delta = _canvas_positions(tokens, [[4, 4]])
|
||||
|
||||
max_position = max(max(channel) for channel in positions)
|
||||
assert delta == max_position + 1 - len(tokens)
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.warmup.qwen_vl_triton_warmup import (
|
||||
@@ -61,7 +62,10 @@ def test_vision_warmup_calls_only_position_and_rotary_paths() -> None:
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_mrope_warmup_launches_triton_shapes() -> None:
|
||||
@pytest.mark.parametrize(
|
||||
"owners_fused, skipped", [((), False), ((True,), True), ((True, False), False)]
|
||||
)
|
||||
def test_mrope_warmup_launches_triton_shapes(owners_fused, skipped) -> None:
|
||||
from vllm.model_executor.layers.rotary_embedding.mrope import MRotaryEmbedding
|
||||
|
||||
launched: list[tuple[torch.Size, torch.Size]] = []
|
||||
@@ -84,7 +88,13 @@ def test_mrope_warmup_launches_triton_shapes() -> None:
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.rotary_emb = FakeRope()
|
||||
rope = FakeRope()
|
||||
if not owners_fused:
|
||||
self.rotary_emb = rope
|
||||
for i, fused in enumerate(owners_fused):
|
||||
owner = torch.nn.Module()
|
||||
owner.rotary_emb, owner.use_fused_qk_norm_rope_gate = rope, fused
|
||||
self.add_module(f"attn{i}", owner)
|
||||
|
||||
def get_mrope_input_positions(self, input_tokens, mm_features):
|
||||
raise AssertionError("warmup must not build live M-RoPE positions")
|
||||
@@ -99,11 +109,8 @@ def test_mrope_warmup_launches_triton_shapes() -> None:
|
||||
device=torch.device("cpu"),
|
||||
)
|
||||
_warm_mrope(runner, FakeMropeModel())
|
||||
assert [shape for shape, _ in launched] == [
|
||||
torch.Size((3, 1)),
|
||||
torch.Size((3, 2)),
|
||||
torch.Size((3, 16)),
|
||||
]
|
||||
expected = [torch.Size((3, n)) for n in (1, 2, 16)]
|
||||
assert [shape for shape, _ in launched] == ([] if skipped else expected)
|
||||
|
||||
|
||||
def test_mrope_warmup_skips_models_without_supports_mrope() -> None:
|
||||
|
||||
@@ -6,13 +6,108 @@ from unittest.mock import Mock
|
||||
|
||||
import huggingface_hub.constants
|
||||
import pytest
|
||||
import torch
|
||||
from huggingface_hub.utils import LocalEntryNotFoundError
|
||||
|
||||
import vllm.model_executor.model_loader.weight_utils as weight_utils
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
download_weights_from_hf,
|
||||
drop_checkpoint_cache,
|
||||
initialize_single_dummy_weight,
|
||||
load_attention_sink,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"checkpoint_heads,runtime_heads,tp_size",
|
||||
[(8, 4, 2), (8, 8, 1), (8, 6, 2), (16, 12, 4)],
|
||||
)
|
||||
def test_attention_sink_loader_shapes(
|
||||
monkeypatch, checkpoint_heads, runtime_heads, tp_size
|
||||
):
|
||||
"""Each rank loads its checkpoint heads and disables only padded heads."""
|
||||
monkeypatch.setattr(
|
||||
weight_utils, "get_tensor_model_parallel_world_size", lambda: tp_size
|
||||
)
|
||||
checkpoint = torch.arange(checkpoint_heads, dtype=torch.float32)
|
||||
for tp_rank in range(tp_size):
|
||||
monkeypatch.setattr(
|
||||
weight_utils, "get_tensor_model_parallel_rank", lambda rank=tp_rank: rank
|
||||
)
|
||||
param = torch.nn.Parameter(torch.zeros(runtime_heads), requires_grad=False)
|
||||
load_attention_sink(param, checkpoint)
|
||||
local_heads = checkpoint.chunk(tp_size)[tp_rank]
|
||||
expected = torch.cat(
|
||||
(
|
||||
local_heads,
|
||||
torch.full((runtime_heads - local_heads.numel(),), -torch.inf),
|
||||
)
|
||||
)
|
||||
assert torch.equal(param, expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"checkpoint_shape,runtime_shape,error",
|
||||
[
|
||||
((2, 4), (4,), RuntimeError),
|
||||
((8,), (2, 2), AssertionError),
|
||||
((7,), (4,), ValueError),
|
||||
((8,), (3,), AssertionError),
|
||||
],
|
||||
)
|
||||
def test_attention_sink_loader_rejects_invalid_shapes(
|
||||
monkeypatch, checkpoint_shape, runtime_shape, error
|
||||
):
|
||||
"""Invalid sink shapes fail before changing the runtime parameter."""
|
||||
monkeypatch.setattr(weight_utils, "get_tensor_model_parallel_world_size", lambda: 2)
|
||||
monkeypatch.setattr(weight_utils, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
param = torch.nn.Parameter(torch.ones(runtime_shape), requires_grad=False)
|
||||
with pytest.raises(error):
|
||||
load_attention_sink(param, torch.zeros(checkpoint_shape))
|
||||
assert torch.equal(param, torch.ones(runtime_shape))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("param_dtype", [torch.bfloat16, torch.float32])
|
||||
def test_attention_sink_loader_keeps_param_dtype(monkeypatch, param_dtype):
|
||||
"""A checkpoint dtype different from the runtime parameter is cast, not copied."""
|
||||
monkeypatch.setattr(weight_utils, "get_tensor_model_parallel_world_size", lambda: 1)
|
||||
monkeypatch.setattr(weight_utils, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
param = torch.nn.Parameter(torch.zeros(4, dtype=param_dtype), requires_grad=False)
|
||||
load_attention_sink(param, torch.arange(4, dtype=torch.float32))
|
||||
assert param.dtype == param_dtype
|
||||
assert torch.equal(param.float(), torch.arange(4, dtype=torch.float32))
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires a GPU")
|
||||
def test_attention_sink_loader_accepts_cpu_checkpoint(monkeypatch):
|
||||
"""Sinks are normally loaded from CPU checkpoints into device parameters."""
|
||||
monkeypatch.setattr(weight_utils, "get_tensor_model_parallel_world_size", lambda: 2)
|
||||
monkeypatch.setattr(weight_utils, "get_tensor_model_parallel_rank", lambda: 1)
|
||||
param = torch.nn.Parameter(
|
||||
torch.full((6,), 123.0, device="cuda"), requires_grad=False
|
||||
)
|
||||
# The test suite sets the default device, so pin the checkpoint to CPU.
|
||||
checkpoint = torch.arange(8, dtype=torch.float32, device="cpu")
|
||||
load_attention_sink(param, checkpoint)
|
||||
assert param.device.type == "cuda"
|
||||
expected = torch.arange(4, 8, dtype=torch.float32, device="cpu")
|
||||
assert torch.equal(param[:4].cpu(), expected)
|
||||
assert torch.isinf(param[4:]).all() and (param[4:] < 0).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"num_heads,dtype", [(4, torch.float32), (8, torch.bfloat16), (6, None)]
|
||||
)
|
||||
def test_make_attention_sink_registers_loader_and_padding(num_heads, dtype):
|
||||
"""The factory is the single place that creates and registers a sink."""
|
||||
sink = weight_utils.make_attention_sink(num_heads, dtype)
|
||||
assert sink.shape == (num_heads,)
|
||||
assert sink.dtype == (dtype or torch.get_default_dtype())
|
||||
assert sink.requires_grad is False
|
||||
assert torch.isinf(sink).all() and (sink < 0).all()
|
||||
assert sink.weight_loader is load_attention_sink
|
||||
|
||||
|
||||
def test_download_weights_from_hf():
|
||||
@@ -307,5 +402,69 @@ def test_drop_checkpoint_cache_releases_local_weight_files(
|
||||
fadvise.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float8_e4m3fn, torch.float8_e5m2])
|
||||
def test_dummy_fp8_weight_is_not_degenerate(dtype):
|
||||
"""+-1e-3 underflows in FP8, so 8-bit floats default to +-1."""
|
||||
param = torch.empty(4096, dtype=dtype)
|
||||
initialize_single_dummy_weight(param)
|
||||
values = param.float()
|
||||
assert values.isfinite().all()
|
||||
assert values.abs().max() <= 1.0
|
||||
assert (values != 0).float().mean() > 0.99
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("dtype", "name"),
|
||||
[(torch.float8_e8m0fnu, "weight_scale_inv"), (torch.uint8, "w13_weight_scale")],
|
||||
)
|
||||
def test_dummy_e8m0_scale_is_fixed_exponent(dtype, name):
|
||||
"""E8M0 scales, including uint8-stored ones, get a finite 2**-7."""
|
||||
param = torch.empty(64, dtype=dtype)
|
||||
initialize_single_dummy_weight(param, name=name)
|
||||
assert (param.view(torch.uint8) == 120).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.uint8, torch.int8, torch.float4_e2m1fn_x2])
|
||||
def test_dummy_packed_weight_gets_random_bytes(dtype):
|
||||
"""Packed weights are filled deterministically instead of left uninitialized."""
|
||||
# Some backends can't fill packed FP4 directly, so view zeroed bytes.
|
||||
params = [torch.zeros(4096, dtype=torch.uint8).view(dtype) for _ in range(2)]
|
||||
for param in params:
|
||||
initialize_single_dummy_weight(param, name="w13_weight")
|
||||
raw = params[0].view(torch.uint8)
|
||||
assert raw.unique().numel() > 250
|
||||
assert torch.equal(raw, params[1].view(torch.uint8))
|
||||
|
||||
|
||||
@pytest.mark.skipif(current_platform.is_rocm(), reason="ROCm zeroes integer params")
|
||||
def test_dummy_wide_integer_param_is_untouched():
|
||||
"""Wider integer params may be indices, so random values are unsafe."""
|
||||
param = torch.arange(16, dtype=torch.int32)
|
||||
initialize_single_dummy_weight(param, name="g_idx")
|
||||
assert torch.equal(param, torch.arange(16, dtype=torch.int32))
|
||||
|
||||
|
||||
def test_dummy_norm_weights_are_neutral(default_vllm_config):
|
||||
"""Dummy norms stay identity-like, keeping activations at a realistic scale."""
|
||||
from vllm.model_executor.layers.layernorm import (
|
||||
GemmaRMSNorm,
|
||||
LayerNorm,
|
||||
RMSNorm,
|
||||
RMSNormGated,
|
||||
)
|
||||
|
||||
expected = {
|
||||
RMSNorm(16): {"weight": 1.0},
|
||||
GemmaRMSNorm(16): {"weight": 0.0},
|
||||
LayerNorm(16): {"weight": 1.0, "bias": 0.0},
|
||||
RMSNormGated(16): {"weight": 1.0},
|
||||
}
|
||||
for norm, values in expected.items():
|
||||
for name, param in norm.named_parameters():
|
||||
param.data.uniform_(-5, 5)
|
||||
initialize_single_dummy_weight(param, name=name)
|
||||
assert torch.all(param == values[name]), (type(norm).__name__, name)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_download_weights_from_hf()
|
||||
|
||||
Reference in New Issue
Block a user