Merge remote-tracking branch 'origin/main' into sync/lesj/qwen4exp-bugfixes
Signed-off-by: lesj0610 <lesj0610@users.noreply.github.com>
This commit is contained in:
@@ -0,0 +1,38 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from functools import partial
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.model_executor.layers.activation import GeluAndMul, SiluAndMul
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cpu()
|
||||
or current_platform.get_cpu_architecture() == CpuArchEnum.POWERPC,
|
||||
reason="Native CPU activations require a non-PowerPC CPU platform",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"activation_factory,reference",
|
||||
[
|
||||
(SiluAndMul, nn.SiLU()),
|
||||
(GeluAndMul, nn.GELU()),
|
||||
(partial(GeluAndMul, approximate="tanh"), nn.GELU(approximate="tanh")),
|
||||
],
|
||||
ids=["silu", "gelu", "gelu_tanh"],
|
||||
)
|
||||
def test_cpu_act_and_mul_without_compiled_ops(
|
||||
default_vllm_config, monkeypatch, activation_factory, reference
|
||||
):
|
||||
"""CPU activations must construct and run without compiled wheel operators."""
|
||||
default_vllm_config.compilation_config.custom_ops = ["all"]
|
||||
monkeypatch.setattr(torch.ops, "_C", object())
|
||||
x = torch.tensor([[-2.0, -1.0, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0]], device="cpu")
|
||||
|
||||
layer = activation_factory()
|
||||
|
||||
torch.testing.assert_close(layer(x), reference(x[:, :4]) * x[:, 4:])
|
||||
@@ -365,6 +365,26 @@ def test_select_candidate_blocks_tolerates_empty_rows():
|
||||
assert (out[0] >= 0).all() and (out[2, :2] >= 0).all()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
@pytest.mark.parametrize("score", [0.0, float("nan")])
|
||||
@pytest.mark.parametrize("start", [0, 3])
|
||||
def test_candidate_selection_keeps_valid_blocks_with_nan_scores(score, start):
|
||||
"""NaN warmup scores must not turn real blocks into repeated sparse padding."""
|
||||
logits = torch.full((3, 64), score, device="cuda")
|
||||
starts = torch.full((3,), start, device="cuda", dtype=torch.int32)
|
||||
lengths = torch.tensor([0, 17, 35], device="cuda", dtype=torch.int32)
|
||||
out = torch.empty(3, 8, device="cuda", dtype=torch.int32)
|
||||
|
||||
sparse_indexer._select_candidate_blocks(logits, starts, starts + lengths, 8, 8, out)
|
||||
|
||||
for row, count in enumerate([0, 3, 5]):
|
||||
valid = out[row][out[row] >= 0].sort().values
|
||||
torch.testing.assert_close(
|
||||
valid, torch.arange(count, device="cuda", dtype=torch.int32)
|
||||
)
|
||||
assert (out[row] == -1).sum().item() == 8 - count
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
@pytest.mark.parametrize(
|
||||
"width,block_size,k,decode",
|
||||
@@ -413,7 +433,7 @@ def test_candidate_kernels_preserve_packed_bounds_and_padding(
|
||||
top = reduced.topk(min(k, nblocks), dim=-1)
|
||||
expected = torch.full((rows, k), -1, device="cuda", dtype=torch.int32)
|
||||
expected[:, : top.indices.shape[1]] = torch.where(
|
||||
top.values > -torch.inf, top.indices, -1
|
||||
top.values != -torch.inf, top.indices, -1
|
||||
).int()
|
||||
actual = torch.empty(rows, k * 2, device="cuda", dtype=torch.int32)[:, ::2]
|
||||
select_candidate_blocks(logits, starts, ends, k, block_size, actual, repeat)
|
||||
@@ -608,7 +628,10 @@ def test_candidate_blocks_to_sparse_indices_math():
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="Triton kernel")
|
||||
@pytest.mark.parametrize("cbk,sbk", [(8, 8), (16, 8), (16, 16)])
|
||||
def test_candidate_blocks_to_sparse_indices_matches_reference(cbk: int, sbk: int):
|
||||
@pytest.mark.parametrize("end_offset", [0, 1])
|
||||
def test_candidate_blocks_to_sparse_indices_matches_reference(
|
||||
cbk: int, sbk: int, end_offset: int
|
||||
):
|
||||
"""The Triton expansion equals the PyTorch reference on production-shaped
|
||||
random inputs, including strided (column-sliced) candidate rows and
|
||||
strided output buffers."""
|
||||
@@ -643,7 +666,7 @@ def test_candidate_blocks_to_sparse_indices_matches_reference(cbk: int, sbk: int
|
||||
ke,
|
||||
cbk,
|
||||
sbk,
|
||||
out=(si_buf[:, : num_candidates * ratio], end_buf[::2]),
|
||||
out=(si_buf[:, : num_candidates * ratio], end_buf[end_offset::2]),
|
||||
)
|
||||
assert torch.equal(got_indices, ref_indices)
|
||||
assert torch.equal(got_end, ref_end)
|
||||
@@ -651,7 +674,7 @@ def test_candidate_blocks_to_sparse_indices_matches_reference(cbk: int, sbk: int
|
||||
cand_buf[:, num_candidates:], torch.full_like(cand_buf[:, num_candidates:], -7)
|
||||
)
|
||||
assert not si_buf[:, num_candidates * ratio :].any()
|
||||
assert not end_buf[1::2].any()
|
||||
assert not end_buf[1 - end_offset :: 2].any()
|
||||
|
||||
|
||||
def _skip_unless_sm100_sparse_kernels():
|
||||
|
||||
@@ -5,6 +5,7 @@ import importlib.machinery
|
||||
import inspect
|
||||
import sys
|
||||
import types
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
from weakref import WeakKeyDictionary, ref
|
||||
|
||||
@@ -467,6 +468,111 @@ def test_model_cleanup(dist_init, default_vllm_config):
|
||||
assert len(mock_info_dict) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_gated", [False, True])
|
||||
@pytest.mark.parametrize("has_bias", [False, True])
|
||||
@pytest.mark.parametrize("tp_rank", [0, 1])
|
||||
@pytest.mark.parametrize("padded", [False, True])
|
||||
def test_padded_moe_reload_releases_each_layer(
|
||||
monkeypatch, is_gated, has_bias, tp_rank, padded
|
||||
):
|
||||
"""Checkpoint-sized copies finish each layer without global finalization."""
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
|
||||
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
|
||||
hidden, intermediate, experts = 4, 3, 2
|
||||
stored_hidden, stored_intermediate = (8, 8) if padded else (hidden, intermediate)
|
||||
config = SimpleNamespace(
|
||||
hidden_dim_unpadded=hidden,
|
||||
intermediate_size_per_partition_unpadded=intermediate,
|
||||
is_act_and_mul=is_gated,
|
||||
has_bias=has_bias,
|
||||
tp_rank=tp_rank,
|
||||
tp_shard_with_padding=False,
|
||||
moe_parallel_config=SimpleNamespace(tp_size=2),
|
||||
)
|
||||
model = torch.nn.ModuleList()
|
||||
processed: list[torch.nn.Module] = []
|
||||
for _ in range(2):
|
||||
method = object.__new__(UnquantizedFusedMoEMethod)
|
||||
torch.nn.Module.__init__(method)
|
||||
method.moe = config
|
||||
# The regression concerns streaming reload, not kernel conversion.
|
||||
monkeypatch.setattr(method, "process_weights_after_loading", processed.append)
|
||||
layer = object.__new__(RoutedExperts)
|
||||
torch.nn.Module.__init__(layer)
|
||||
layer.moe_config = config
|
||||
layer.quant_config = None
|
||||
layer.quant_method = method
|
||||
layer.expert_map_manager = SimpleNamespace(map_global_to_local=lambda i: i)
|
||||
layer._loaded_expert_biases = set()
|
||||
method.create_weights(
|
||||
layer,
|
||||
experts,
|
||||
stored_hidden,
|
||||
stored_intermediate,
|
||||
torch.float32,
|
||||
weight_loader=layer.weight_loader,
|
||||
)
|
||||
model.append(layer)
|
||||
|
||||
record_metadata_for_reloading(model)
|
||||
original_params = [dict(layer.named_parameters()) for layer in model]
|
||||
shards = ["w1", "w3", "w2"] if is_gated else ["w1", "w2"]
|
||||
for cycle in range(2):
|
||||
initialize_layerwise_reload(model)
|
||||
for layer_index, layer in enumerate(model):
|
||||
info = reload_layerwise.get_layerwise_info(layer)
|
||||
inputs = []
|
||||
expected = {
|
||||
name: torch.full_like(p, float("nan"))
|
||||
for name, p in original_params[layer_index].items()
|
||||
}
|
||||
params = dict(layer.named_parameters())
|
||||
calls = [
|
||||
(e, s, b)
|
||||
for e in range(experts)
|
||||
for s in shards
|
||||
for b in ([False, True] if has_bias else [False])
|
||||
]
|
||||
for call_index, (expert, shard, bias) in enumerate(calls):
|
||||
name = ("w2" if shard == "w2" else "w13") + (
|
||||
"_bias" if bias else "_weight"
|
||||
)
|
||||
shape = (
|
||||
((hidden,) if bias else (hidden, 2 * intermediate))
|
||||
if shard == "w2"
|
||||
else ((2 * intermediate,) if bias else (2 * intermediate, hidden))
|
||||
)
|
||||
weight = torch.arange(
|
||||
torch.Size(shape).numel(), dtype=torch.float32
|
||||
).reshape(shape)
|
||||
weight = weight + 100 * (1 + call_index + cycle)
|
||||
inputs.append(ref(weight))
|
||||
# Direct checkpoint loading is the reference for deferred reload.
|
||||
layer.weight_loader(expected[name], weight, name, shard, expert)
|
||||
param = params[name]
|
||||
param.weight_loader(param, weight, name, shard, expert)
|
||||
del weight
|
||||
if call_index != len(calls) - 1:
|
||||
assert info.can_load(), (
|
||||
"Layer processed before its final checkpoint shard"
|
||||
)
|
||||
|
||||
assert not info.can_load(), (
|
||||
"Padding must not defer the layer until finalization"
|
||||
)
|
||||
assert not info.loaded_weights
|
||||
assert len(processed) == cycle * len(model) + layer_index + 1
|
||||
assert all(source() is None for source in inputs)
|
||||
for name, original in original_params[layer_index].items():
|
||||
assert getattr(layer, name) is original
|
||||
# Kernel-specific tests cover padding, which is not checkpoint data.
|
||||
mask = torch.isfinite(expected[name])
|
||||
assert torch.equal(original[mask], expected[name][mask])
|
||||
|
||||
|
||||
def test_get_numel_loaded():
|
||||
param = torch.empty(10, device="meta")
|
||||
loaded_weight = torch.empty(10)
|
||||
|
||||
Reference in New Issue
Block a user