Merge remote-tracking branch 'origin/main' into sync/lesj/qwen4exp-bugfixes

Signed-off-by: lesj0610 <lesj0610@users.noreply.github.com>
This commit is contained in:
lesj0610
2026-09-20 09:27:57 +09:00
682 changed files with 26127 additions and 5580 deletions
@@ -0,0 +1,38 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from functools import partial
import pytest
import torch
import torch.nn as nn
from vllm.model_executor.layers.activation import GeluAndMul, SiluAndMul
from vllm.platforms import CpuArchEnum, current_platform
@pytest.mark.skipif(
not current_platform.is_cpu()
or current_platform.get_cpu_architecture() == CpuArchEnum.POWERPC,
reason="Native CPU activations require a non-PowerPC CPU platform",
)
@pytest.mark.parametrize(
"activation_factory,reference",
[
(SiluAndMul, nn.SiLU()),
(GeluAndMul, nn.GELU()),
(partial(GeluAndMul, approximate="tanh"), nn.GELU(approximate="tanh")),
],
ids=["silu", "gelu", "gelu_tanh"],
)
def test_cpu_act_and_mul_without_compiled_ops(
default_vllm_config, monkeypatch, activation_factory, reference
):
"""CPU activations must construct and run without compiled wheel operators."""
default_vllm_config.compilation_config.custom_ops = ["all"]
monkeypatch.setattr(torch.ops, "_C", object())
x = torch.tensor([[-2.0, -1.0, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0]], device="cpu")
layer = activation_factory()
torch.testing.assert_close(layer(x), reference(x[:, :4]) * x[:, 4:])
@@ -365,6 +365,26 @@ def test_select_candidate_blocks_tolerates_empty_rows():
assert (out[0] >= 0).all() and (out[2, :2] >= 0).all()
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
@pytest.mark.parametrize("score", [0.0, float("nan")])
@pytest.mark.parametrize("start", [0, 3])
def test_candidate_selection_keeps_valid_blocks_with_nan_scores(score, start):
"""NaN warmup scores must not turn real blocks into repeated sparse padding."""
logits = torch.full((3, 64), score, device="cuda")
starts = torch.full((3,), start, device="cuda", dtype=torch.int32)
lengths = torch.tensor([0, 17, 35], device="cuda", dtype=torch.int32)
out = torch.empty(3, 8, device="cuda", dtype=torch.int32)
sparse_indexer._select_candidate_blocks(logits, starts, starts + lengths, 8, 8, out)
for row, count in enumerate([0, 3, 5]):
valid = out[row][out[row] >= 0].sort().values
torch.testing.assert_close(
valid, torch.arange(count, device="cuda", dtype=torch.int32)
)
assert (out[row] == -1).sum().item() == 8 - count
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
@pytest.mark.parametrize(
"width,block_size,k,decode",
@@ -413,7 +433,7 @@ def test_candidate_kernels_preserve_packed_bounds_and_padding(
top = reduced.topk(min(k, nblocks), dim=-1)
expected = torch.full((rows, k), -1, device="cuda", dtype=torch.int32)
expected[:, : top.indices.shape[1]] = torch.where(
top.values > -torch.inf, top.indices, -1
top.values != -torch.inf, top.indices, -1
).int()
actual = torch.empty(rows, k * 2, device="cuda", dtype=torch.int32)[:, ::2]
select_candidate_blocks(logits, starts, ends, k, block_size, actual, repeat)
@@ -608,7 +628,10 @@ def test_candidate_blocks_to_sparse_indices_math():
@pytest.mark.skipif(not torch.cuda.is_available(), reason="Triton kernel")
@pytest.mark.parametrize("cbk,sbk", [(8, 8), (16, 8), (16, 16)])
def test_candidate_blocks_to_sparse_indices_matches_reference(cbk: int, sbk: int):
@pytest.mark.parametrize("end_offset", [0, 1])
def test_candidate_blocks_to_sparse_indices_matches_reference(
cbk: int, sbk: int, end_offset: int
):
"""The Triton expansion equals the PyTorch reference on production-shaped
random inputs, including strided (column-sliced) candidate rows and
strided output buffers."""
@@ -643,7 +666,7 @@ def test_candidate_blocks_to_sparse_indices_matches_reference(cbk: int, sbk: int
ke,
cbk,
sbk,
out=(si_buf[:, : num_candidates * ratio], end_buf[::2]),
out=(si_buf[:, : num_candidates * ratio], end_buf[end_offset::2]),
)
assert torch.equal(got_indices, ref_indices)
assert torch.equal(got_end, ref_end)
@@ -651,7 +674,7 @@ def test_candidate_blocks_to_sparse_indices_matches_reference(cbk: int, sbk: int
cand_buf[:, num_candidates:], torch.full_like(cand_buf[:, num_candidates:], -7)
)
assert not si_buf[:, num_candidates * ratio :].any()
assert not end_buf[1::2].any()
assert not end_buf[1 - end_offset :: 2].any()
def _skip_unless_sm100_sparse_kernels():
@@ -5,6 +5,7 @@ import importlib.machinery
import inspect
import sys
import types
from types import SimpleNamespace
from unittest.mock import Mock
from weakref import WeakKeyDictionary, ref
@@ -467,6 +468,111 @@ def test_model_cleanup(dist_init, default_vllm_config):
assert len(mock_info_dict) == 0
@pytest.mark.parametrize("is_gated", [False, True])
@pytest.mark.parametrize("has_bias", [False, True])
@pytest.mark.parametrize("tp_rank", [0, 1])
@pytest.mark.parametrize("padded", [False, True])
def test_padded_moe_reload_releases_each_layer(
monkeypatch, is_gated, has_bias, tp_rank, padded
):
"""Checkpoint-sized copies finish each layer without global finalization."""
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
UnquantizedFusedMoEMethod,
)
hidden, intermediate, experts = 4, 3, 2
stored_hidden, stored_intermediate = (8, 8) if padded else (hidden, intermediate)
config = SimpleNamespace(
hidden_dim_unpadded=hidden,
intermediate_size_per_partition_unpadded=intermediate,
is_act_and_mul=is_gated,
has_bias=has_bias,
tp_rank=tp_rank,
tp_shard_with_padding=False,
moe_parallel_config=SimpleNamespace(tp_size=2),
)
model = torch.nn.ModuleList()
processed: list[torch.nn.Module] = []
for _ in range(2):
method = object.__new__(UnquantizedFusedMoEMethod)
torch.nn.Module.__init__(method)
method.moe = config
# The regression concerns streaming reload, not kernel conversion.
monkeypatch.setattr(method, "process_weights_after_loading", processed.append)
layer = object.__new__(RoutedExperts)
torch.nn.Module.__init__(layer)
layer.moe_config = config
layer.quant_config = None
layer.quant_method = method
layer.expert_map_manager = SimpleNamespace(map_global_to_local=lambda i: i)
layer._loaded_expert_biases = set()
method.create_weights(
layer,
experts,
stored_hidden,
stored_intermediate,
torch.float32,
weight_loader=layer.weight_loader,
)
model.append(layer)
record_metadata_for_reloading(model)
original_params = [dict(layer.named_parameters()) for layer in model]
shards = ["w1", "w3", "w2"] if is_gated else ["w1", "w2"]
for cycle in range(2):
initialize_layerwise_reload(model)
for layer_index, layer in enumerate(model):
info = reload_layerwise.get_layerwise_info(layer)
inputs = []
expected = {
name: torch.full_like(p, float("nan"))
for name, p in original_params[layer_index].items()
}
params = dict(layer.named_parameters())
calls = [
(e, s, b)
for e in range(experts)
for s in shards
for b in ([False, True] if has_bias else [False])
]
for call_index, (expert, shard, bias) in enumerate(calls):
name = ("w2" if shard == "w2" else "w13") + (
"_bias" if bias else "_weight"
)
shape = (
((hidden,) if bias else (hidden, 2 * intermediate))
if shard == "w2"
else ((2 * intermediate,) if bias else (2 * intermediate, hidden))
)
weight = torch.arange(
torch.Size(shape).numel(), dtype=torch.float32
).reshape(shape)
weight = weight + 100 * (1 + call_index + cycle)
inputs.append(ref(weight))
# Direct checkpoint loading is the reference for deferred reload.
layer.weight_loader(expected[name], weight, name, shard, expert)
param = params[name]
param.weight_loader(param, weight, name, shard, expert)
del weight
if call_index != len(calls) - 1:
assert info.can_load(), (
"Layer processed before its final checkpoint shard"
)
assert not info.can_load(), (
"Padding must not defer the layer until finalization"
)
assert not info.loaded_weights
assert len(processed) == cycle * len(model) + layer_index + 1
assert all(source() is None for source in inputs)
for name, original in original_params[layer_index].items():
assert getattr(layer, name) is original
# Kernel-specific tests cover padding, which is not checkpoint data.
mask = torch.isfinite(expected[name])
assert torch.equal(original[mask], expected[name][mask])
def test_get_numel_loaded():
param = torch.empty(10, device="meta")
loaded_weight = torch.empty(10)