[Bugfix][MoE] Convert FlashInfer BF16 weights in place (#54699)

Signed-off-by: Yuchen Wang <93700456+yuchenwang3@users.noreply.github.com>
Signed-off-by: Dakai An <dakaian108@gmail.com>
Co-authored-by: Dakai An <dakaian108@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
Yuchen Wang
2026-09-17 22:00:04 +08:00
committed by GitHub
co-authored by Dakai An OpenAI Codex aoshen02
parent 117cf43fb6
commit 78fdf4efb8
9 changed files with 667 additions and 68 deletions
@@ -5,6 +5,7 @@ import importlib.machinery
import inspect
import sys
import types
from types import SimpleNamespace
from unittest.mock import Mock
from weakref import WeakKeyDictionary, ref
@@ -467,6 +468,110 @@ def test_model_cleanup(dist_init, default_vllm_config):
assert len(mock_info_dict) == 0
@pytest.mark.parametrize("is_gated", [False, True])
@pytest.mark.parametrize("has_bias", [False, True])
@pytest.mark.parametrize("tp_rank", [0, 1])
@pytest.mark.parametrize("padded", [False, True])
def test_padded_moe_reload_releases_each_layer(
monkeypatch, is_gated, has_bias, tp_rank, padded
):
"""Checkpoint-sized copies finish each layer without global finalization."""
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
UnquantizedFusedMoEMethod,
)
hidden, intermediate, experts = 4, 3, 2
stored_hidden, stored_intermediate = (8, 8) if padded else (hidden, intermediate)
config = SimpleNamespace(
hidden_dim_unpadded=hidden,
intermediate_size_per_partition_unpadded=intermediate,
is_act_and_mul=is_gated,
has_bias=has_bias,
tp_rank=tp_rank,
moe_parallel_config=SimpleNamespace(tp_size=2),
)
model = torch.nn.ModuleList()
processed: list[torch.nn.Module] = []
for _ in range(2):
method = object.__new__(UnquantizedFusedMoEMethod)
torch.nn.Module.__init__(method)
method.moe = config
# The regression concerns streaming reload, not kernel conversion.
monkeypatch.setattr(method, "process_weights_after_loading", processed.append)
layer = object.__new__(RoutedExperts)
torch.nn.Module.__init__(layer)
layer.moe_config = config
layer.quant_config = None
layer.quant_method = method
layer.expert_map_manager = SimpleNamespace(map_global_to_local=lambda i: i)
layer._loaded_expert_biases = set()
method.create_weights(
layer,
experts,
stored_hidden,
stored_intermediate,
torch.float32,
weight_loader=layer.weight_loader,
)
model.append(layer)
record_metadata_for_reloading(model)
original_params = [dict(layer.named_parameters()) for layer in model]
shards = ["w1", "w3", "w2"] if is_gated else ["w1", "w2"]
for cycle in range(2):
initialize_layerwise_reload(model)
for layer_index, layer in enumerate(model):
info = reload_layerwise.get_layerwise_info(layer)
inputs = []
expected = {
name: torch.full_like(p, float("nan"))
for name, p in original_params[layer_index].items()
}
params = dict(layer.named_parameters())
calls = [
(e, s, b)
for e in range(experts)
for s in shards
for b in ([False, True] if has_bias else [False])
]
for call_index, (expert, shard, bias) in enumerate(calls):
name = ("w2" if shard == "w2" else "w13") + (
"_bias" if bias else "_weight"
)
shape = (
((hidden,) if bias else (hidden, 2 * intermediate))
if shard == "w2"
else ((2 * intermediate,) if bias else (2 * intermediate, hidden))
)
weight = torch.arange(
torch.Size(shape).numel(), dtype=torch.float32
).reshape(shape)
weight = weight + 100 * (1 + call_index + cycle)
inputs.append(ref(weight))
# Direct checkpoint loading is the reference for deferred reload.
layer.weight_loader(expected[name], weight, name, shard, expert)
param = params[name]
param.weight_loader(param, weight, name, shard, expert)
del weight
if call_index != len(calls) - 1:
assert info.can_load(), (
"Layer processed before its final checkpoint shard"
)
assert not info.can_load(), (
"Padding must not defer the layer until finalization"
)
assert not info.loaded_weights
assert len(processed) == cycle * len(model) + layer_index + 1
assert all(source() is None for source in inputs)
for name, original in original_params[layer_index].items():
assert getattr(layer, name) is original
# Kernel-specific tests cover padding, which is not checkpoint data.
mask = torch.isfinite(expected[name])
assert torch.equal(original[mask], expected[name][mask])
def test_get_numel_loaded():
param = torch.empty(10, device="meta")
loaded_weight = torch.empty(10)