[Bugfix][MoE] Convert FlashInfer BF16 weights in place (#54699)

Signed-off-by: Yuchen Wang <93700456+yuchenwang3@users.noreply.github.com>
Signed-off-by: Dakai An <dakaian108@gmail.com>
Co-authored-by: Dakai An <dakaian108@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
Yuchen Wang
2026-09-17 22:00:04 +08:00
committed by GitHub
co-authored by Dakai An OpenAI Codex aoshen02
parent 117cf43fb6
commit 78fdf4efb8
9 changed files with 667 additions and 68 deletions
+258
View File
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import multiprocessing
from dataclasses import dataclass from dataclasses import dataclass
import pytest import pytest
@@ -433,6 +434,263 @@ def test_convert_moe_weights_to_flashinfer_trtllm_block_layout(
assert w13_converted.shape[0] == num_experts assert w13_converted.shape[0] == num_experts
assert w2_converted.shape[0] == num_experts assert w2_converted.shape[0] == num_experts
assert w13_converted.data_ptr() == w13.data_ptr()
assert w2_converted.data_ptr() == w2.data_ptr()
@pytest.mark.parametrize("is_gated_act_gemm", [True, False])
def test_convert_moe_weights_to_flashinfer_trtllm_block_layout_values(
is_gated_act_gemm,
):
from flashinfer.fused_moe.core import (
_maybe_get_cached_w3_w1_permute_indices,
get_w2_permute_indices_with_cache,
)
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
convert_moe_weights_to_flashinfer_trtllm_block_layout,
)
num_experts, intermediate, hidden = 2, 256, 256
w13_multiplier = 2 if is_gated_act_gemm else 1
w13 = torch.randn(
(num_experts, w13_multiplier * intermediate, hidden),
dtype=torch.bfloat16,
device="cuda",
)
w2 = torch.randn(
(num_experts, hidden, intermediate),
dtype=torch.bfloat16,
device="cuda",
)
def _reference_block_layout(
weight: torch.Tensor,
is_w13: bool,
cache: dict[torch.Size, torch.Tensor],
) -> torch.Tensor:
outputs = []
for expert in weight:
expert_uint8 = expert.view(torch.uint8)
if is_w13:
indices = _maybe_get_cached_w3_w1_permute_indices(
cache,
expert_uint8,
128,
is_gated_act_gemm=is_gated_act_gemm,
)
if is_gated_act_gemm:
indices = (indices + expert_uint8.shape[0] // 2) % (
expert_uint8.shape[0]
)
else:
indices = get_w2_permute_indices_with_cache(
cache,
expert_uint8,
128,
)
rows, cols = expert_uint8.shape
blocks = expert_uint8.view(rows, cols // 128, 128).permute(1, 0, 2)
outputs.append(torch.index_select(blocks, 1, indices.to(weight.device)))
return torch.stack(outputs).view(torch.bfloat16)
reference_cache: dict[torch.Size, torch.Tensor] = {}
expected_w13 = _reference_block_layout(w13, is_w13=True, cache=reference_cache)
expected_w2 = _reference_block_layout(w2, is_w13=False, cache=reference_cache)
w13_ptr = w13.data_ptr()
w2_ptr = w2.data_ptr()
actual_w13, actual_w2 = convert_moe_weights_to_flashinfer_trtllm_block_layout(
{},
w13,
w2,
is_gated_act_gemm=is_gated_act_gemm,
)
assert actual_w13.data_ptr() == w13_ptr
assert actual_w2.data_ptr() == w2_ptr
assert torch.equal(actual_w13, expected_w13)
assert torch.equal(actual_w2, expected_w2)
def _make_unquantized_flashinfer_test_layer(
monkeypatch, intermediate, is_gated, *, device="cuda"
):
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
UnquantizedMoeBackend,
)
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
UnquantizedFusedMoEMethod,
)
moe_config = make_dummy_moe_config(
num_experts=2,
hidden_dim=256,
intermediate_size=intermediate,
activation=MoEActivation.SILU if is_gated else MoEActivation.RELU2_NO_MUL,
)
moe_config.intermediate_size_per_partition = (intermediate + 127) // 128 * 128
method = object.__new__(UnquantizedFusedMoEMethod)
method.moe = moe_config
method.unquantized_backend = UnquantizedMoeBackend.FLASHINFER_TRTLLM
method.moe_kernel = None
monkeypatch.setattr(
method,
"_init_moe_kernel",
lambda _: setattr(method, "moe_kernel", object()),
)
layer = torch.nn.Module()
layer.moe_config = moe_config
with torch.device(device):
method.create_weights(
layer,
num_experts=2,
hidden_size=256,
intermediate_size_per_partition=moe_config.intermediate_size_per_partition,
params_dtype=torch.bfloat16,
)
return method, layer
@pytest.mark.parametrize("intermediate", [192, 256])
@pytest.mark.parametrize("is_gated", [True, False])
def test_unquantized_flashinfer_trtllm_weights_can_be_reprocessed(
monkeypatch, intermediate, is_gated
):
"""Each raw reload clears padding left dirty by prior in-place conversion."""
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
convert_moe_weights_to_flashinfer_trtllm_block_layout,
)
method, layer = _make_unquantized_flashinfer_test_layer(
monkeypatch, intermediate, is_gated
)
w13_shape, w2_shape = layer.w13_weight.shape, layer.w2_weight.shape
padded = w2_shape[-1]
w13_ptr = layer.w13_weight.data_ptr()
w2_ptr = layer.w2_weight.data_ptr()
for _ in range(2):
reloaded_w13 = torch.randn_like(layer.w13_weight)
reloaded_w2 = torch.randn_like(layer.w2_weight)
reloaded_w13[:, intermediate:padded].zero_()
if is_gated:
reloaded_w13[:, padded + intermediate :].zero_()
reloaded_w2[:, :, intermediate:].zero_()
expected_w13, expected_w2 = (
convert_moe_weights_to_flashinfer_trtllm_block_layout(
{},
reloaded_w13.clone(),
reloaded_w2.clone(),
is_gated_act_gemm=is_gated,
)
)
# The loader writes only logical checkpoint slices, not padding.
layer.w13_weight.fill_(float("nan"))
layer.w2_weight.fill_(float("nan"))
layer.w13_weight[:, :intermediate].copy_(reloaded_w13[:, :intermediate])
if is_gated:
layer.w13_weight[:, padded : padded + intermediate].copy_(
reloaded_w13[:, padded : padded + intermediate]
)
layer.w2_weight[:, :, :intermediate].copy_(reloaded_w2[:, :, :intermediate])
method.process_weights_after_loading(layer)
assert layer.w13_weight.shape == w13_shape
assert layer.w2_weight.shape == w2_shape
assert layer.w13_weight.data_ptr() == w13_ptr
assert layer.w2_weight.data_ptr() == w2_ptr
kernel_w13, kernel_w2 = method._kernel_weights(layer)
assert torch.equal(kernel_w13, expected_w13)
assert torch.equal(kernel_w2, expected_w2)
def _check_flashinfer_ipc_weights(entries, expected, is_gated, mode, device_index):
from vllm.model_executor.model_loader.weight_cache.ipc_loader import IpcModelLoader
from vllm.model_executor.utils import weights_already_processed
torch.accelerator.set_device_index(device_index)
with pytest.MonkeyPatch.context() as monkeypatch:
method, layer = _make_unquantized_flashinfer_test_layer(
monkeypatch, 192, is_gated, device="meta"
)
loader = object.__new__(IpcModelLoader)
loader.mode = mode
loader._apply_entries(layer, entries, {}, device_index)
pointers = (layer.w13_weight.data_ptr(), layer.w2_weight.data_ptr())
# Like ipc_cache: a fresh method, only tensor metadata, no _setup_kernel.
with weights_already_processed():
method.process_weights_after_loading(layer)
assert method.moe_kernel is not None
actual = method._kernel_weights(layer)
for weight, reference, pointer in zip(actual, expected, pointers):
assert weight.data_ptr() == pointer
assert torch.equal(weight.cpu(), reference)
# Cross-process addresses need not match; writes prove sharing/isolation.
with torch.no_grad():
for weight in actual:
weight.fill_(1.0)
torch.accelerator.synchronize()
@pytest.mark.parametrize("is_gated", [True, False])
@pytest.mark.parametrize("cache_ndim", [3, 4])
@pytest.mark.parametrize("mode", ["copy", "zero_copy"])
def test_unquantized_flashinfer_trtllm_cached_weights_need_no_method_state(
monkeypatch, is_gated, cache_ndim, mode
):
"""A spawned IPC consumer restores views without transient method state."""
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
convert_moe_weights_to_flashinfer_trtllm_block_layout,
)
from vllm.model_executor.model_loader.weight_cache.protocol import TensorEntry
method, layer = _make_unquantized_flashinfer_test_layer(
monkeypatch, 192, is_gated, device="meta"
)
packed = convert_moe_weights_to_flashinfer_trtllm_block_layout(
{},
torch.randn_like(layer.w13_weight, device="cuda"),
torch.randn_like(layer.w2_weight, device="cuda"),
is_gated_act_gemm=is_gated,
)
expected = [weight.cpu() for weight in packed]
entries = {}
for name, weight in zip(("w13_weight", "w2_weight"), packed):
cached = weight.view_as(getattr(layer, name)) if cache_ndim == 3 else weight
entries[name] = TensorEntry.from_tensor(cached, kind="param")
consumer = multiprocessing.get_context("spawn").Process(
target=_check_flashinfer_ipc_weights,
args=(
entries,
expected,
is_gated,
mode,
torch.accelerator.current_device_index(),
),
)
consumer.start()
try:
# Keep producer allocations alive until the consumer releases its views.
consumer.join(timeout=180)
assert not consumer.is_alive(), "IPC consumer timed out"
assert consumer.exitcode == 0, "IPC consumer failed; see child traceback"
for source, reference in zip(packed, expected):
expected_source = (
torch.ones_like(reference) if mode == "zero_copy" else reference
)
assert torch.equal(source.cpu(), expected_source)
finally:
if consumer.is_alive():
consumer.terminate()
consumer.join(timeout=10)
if consumer.is_alive():
consumer.kill()
consumer.join(timeout=10)
consumer.close()
@pytest.mark.parametrize( @pytest.mark.parametrize(
@@ -12,7 +12,14 @@ correctly handles this mismatch.
import pytest import pytest
import torch import torch
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
UnquantizedMoeBackend,
)
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
UnquantizedFusedMoEMethod,
)
from .utils import make_dummy_moe_config from .utils import make_dummy_moe_config
@@ -249,6 +256,57 @@ class TestNarrowExpertDataForPadding:
class TestWeightLoadingWithPaddedHiddenSize: class TestWeightLoadingWithPaddedHiddenSize:
"""Integration-style tests that simulate padded weight loading.""" """Integration-style tests that simulate padded weight loading."""
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float32])
@pytest.mark.parametrize("tp_rank", [0, 1])
def test_load_w2_chunks_noncontiguous_cpu_source_to_padded_cuda(
self, dtype, tp_rank
):
hidden = 4096
intermediate = 1024
loaded_weight = (
torch.arange(hidden * intermediate * 2)
.remainder(251)
.to(dtype)
.reshape(hidden, intermediate * 2)
)
tp_source = loaded_weight[
:, intermediate * tp_rank : intermediate * (tp_rank + 1)
]
expert_data_full = torch.zeros(
hidden + 8, intermediate + 8, device="cuda", dtype=dtype
)
destination = expert_data_full[:hidden, :intermediate]
assert not tp_source.is_contiguous()
assert tp_source.nbytes > 1 << 20
assert not destination.is_contiguous()
experts = object.__new__(RoutedExperts)
torch.nn.Module.__init__(experts)
experts.moe_config = make_dummy_moe_config()
experts.moe_config.moe_parallel_config.tp_size = 2
torch.accelerator.synchronize()
allocated_before = torch.accelerator.memory_allocated()
torch.accelerator.reset_peak_memory_stats()
experts._load_w2(
expert_data=expert_data_full,
shard_dim=1,
loaded_weight=loaded_weight,
tp_rank=tp_rank,
)
torch.accelerator.synchronize()
peak_extra = torch.accelerator.max_memory_allocated() - allocated_before
# A single copy into the strided CUDA view allocates a full-shard
# temporary. Correct values alone would not catch that regression.
assert peak_extra < tp_source.nbytes
torch.testing.assert_close(destination.cpu(), tp_source)
assert torch.count_nonzero(expert_data_full[hidden:, :]) == 0
assert torch.count_nonzero(expert_data_full[:hidden, intermediate:]) == 0
def test_load_w2_with_padding(self): def test_load_w2_with_padding(self):
"""Simulate loading w2 weights when hidden_size is padded.""" """Simulate loading w2 weights when hidden_size is padded."""
padded_hidden = 3072 padded_hidden = 3072
@@ -364,6 +422,74 @@ class TestWeightLoadingWithPaddedHiddenSize:
) )
class TestUnquantizedTrtLlmPrePadding:
@staticmethod
def _make_method(
intermediate: int,
backend: UnquantizedMoeBackend = UnquantizedMoeBackend.FLASHINFER_TRTLLM,
) -> UnquantizedFusedMoEMethod:
moe_config = make_dummy_moe_config(
num_experts=2,
hidden_dim=64,
intermediate_size=intermediate,
)
method = object.__new__(UnquantizedFusedMoEMethod)
method.moe = moe_config
method.unquantized_backend = backend
method.moe_kernel = None
return method
@pytest.mark.parametrize(
"backend,original,expected",
[
(UnquantizedMoeBackend.FLASHINFER_TRTLLM, 1344, 1408),
(UnquantizedMoeBackend.FLASHINFER_TRTLLM, 1408, 1408),
(UnquantizedMoeBackend.TRITON, 1344, 1344),
],
)
def test_rounds_intermediate_before_weight_allocation(
self,
backend: UnquantizedMoeBackend,
original: int,
expected: int,
):
method = self._make_method(original, backend)
hidden, intermediate = method.maybe_roundup_sizes(
hidden_size=64,
intermediate_size_per_partition=original,
act_dtype=torch.bfloat16,
moe_parallel_config=method.moe.moe_parallel_config,
)
assert hidden == 64
assert intermediate == expected
@pytest.mark.parametrize("is_act_and_mul", [True, False])
def test_create_weights_zeroes_preallocated_padding(self, is_act_and_mul: bool):
method = self._make_method(1344)
if not is_act_and_mul:
method.moe.activation = MoEActivation.RELU2_NO_MUL
method.moe.intermediate_size_per_partition = 1408
layer = torch.nn.Module()
method.create_weights(
layer=layer,
num_experts=2,
hidden_size=64,
intermediate_size_per_partition=1408,
params_dtype=torch.bfloat16,
)
up_mult = 2 if is_act_and_mul else 1
assert layer.w13_weight.shape == (2, up_mult * 1408, 64)
assert layer.w2_weight.shape == (2, 64, 1408)
assert torch.count_nonzero(layer.w13_weight[:, 1344:1408]) == 0
if is_act_and_mul:
assert torch.count_nonzero(layer.w13_weight[:, 2752:]) == 0
assert torch.count_nonzero(layer.w2_weight[:, :, 1344:]) == 0
class TestLoadWeightsExpertBias: class TestLoadWeightsExpertBias:
"""Some quantized exports (e.g. GPTQ, llm-compressor NVFP4) materialize """Some quantized exports (e.g. GPTQ, llm-compressor NVFP4) materialize
all-zero per-expert `.bias` tensors for models whose experts have no bias all-zero per-expert `.bias` tensors for models whose experts have no bias
+6 -2
View File
@@ -75,6 +75,10 @@ def test_trtllm_bf16_moe_modular_no_graph(
a = torch.randn((m, k), device="cuda", dtype=dtype) / 10 a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 10 w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 10
w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 10 w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 10
# The FlashInfer conversion may rewrite unpadded input storage in place.
# Preserve the original layout for the independent torch reference.
reference_w1 = w1.clone()
reference_w2 = w2.clone()
score = torch.randn((m, e), device="cuda", dtype=dtype) score = torch.randn((m, e), device="cuda", dtype=dtype)
scores = torch.softmax(score, dim=-1, dtype=torch.float32) scores = torch.softmax(score, dim=-1, dtype=torch.float32)
topk_weights, topk_ids = torch.topk(scores, topk) topk_weights, topk_ids = torch.topk(scores, topk)
@@ -133,8 +137,8 @@ def test_trtllm_bf16_moe_modular_no_graph(
torch_output = torch_moe( torch_output = torch_moe(
a, a,
w1, reference_w1,
w2, reference_w2,
score, score,
topk, topk,
activation=MoEActivation.SILU, activation=MoEActivation.SILU,
@@ -5,6 +5,7 @@ import importlib.machinery
import inspect import inspect
import sys import sys
import types import types
from types import SimpleNamespace
from unittest.mock import Mock from unittest.mock import Mock
from weakref import WeakKeyDictionary, ref from weakref import WeakKeyDictionary, ref
@@ -467,6 +468,110 @@ def test_model_cleanup(dist_init, default_vllm_config):
assert len(mock_info_dict) == 0 assert len(mock_info_dict) == 0
@pytest.mark.parametrize("is_gated", [False, True])
@pytest.mark.parametrize("has_bias", [False, True])
@pytest.mark.parametrize("tp_rank", [0, 1])
@pytest.mark.parametrize("padded", [False, True])
def test_padded_moe_reload_releases_each_layer(
monkeypatch, is_gated, has_bias, tp_rank, padded
):
"""Checkpoint-sized copies finish each layer without global finalization."""
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
UnquantizedFusedMoEMethod,
)
hidden, intermediate, experts = 4, 3, 2
stored_hidden, stored_intermediate = (8, 8) if padded else (hidden, intermediate)
config = SimpleNamespace(
hidden_dim_unpadded=hidden,
intermediate_size_per_partition_unpadded=intermediate,
is_act_and_mul=is_gated,
has_bias=has_bias,
tp_rank=tp_rank,
moe_parallel_config=SimpleNamespace(tp_size=2),
)
model = torch.nn.ModuleList()
processed: list[torch.nn.Module] = []
for _ in range(2):
method = object.__new__(UnquantizedFusedMoEMethod)
torch.nn.Module.__init__(method)
method.moe = config
# The regression concerns streaming reload, not kernel conversion.
monkeypatch.setattr(method, "process_weights_after_loading", processed.append)
layer = object.__new__(RoutedExperts)
torch.nn.Module.__init__(layer)
layer.moe_config = config
layer.quant_config = None
layer.quant_method = method
layer.expert_map_manager = SimpleNamespace(map_global_to_local=lambda i: i)
layer._loaded_expert_biases = set()
method.create_weights(
layer,
experts,
stored_hidden,
stored_intermediate,
torch.float32,
weight_loader=layer.weight_loader,
)
model.append(layer)
record_metadata_for_reloading(model)
original_params = [dict(layer.named_parameters()) for layer in model]
shards = ["w1", "w3", "w2"] if is_gated else ["w1", "w2"]
for cycle in range(2):
initialize_layerwise_reload(model)
for layer_index, layer in enumerate(model):
info = reload_layerwise.get_layerwise_info(layer)
inputs = []
expected = {
name: torch.full_like(p, float("nan"))
for name, p in original_params[layer_index].items()
}
params = dict(layer.named_parameters())
calls = [
(e, s, b)
for e in range(experts)
for s in shards
for b in ([False, True] if has_bias else [False])
]
for call_index, (expert, shard, bias) in enumerate(calls):
name = ("w2" if shard == "w2" else "w13") + (
"_bias" if bias else "_weight"
)
shape = (
((hidden,) if bias else (hidden, 2 * intermediate))
if shard == "w2"
else ((2 * intermediate,) if bias else (2 * intermediate, hidden))
)
weight = torch.arange(
torch.Size(shape).numel(), dtype=torch.float32
).reshape(shape)
weight = weight + 100 * (1 + call_index + cycle)
inputs.append(ref(weight))
# Direct checkpoint loading is the reference for deferred reload.
layer.weight_loader(expected[name], weight, name, shard, expert)
param = params[name]
param.weight_loader(param, weight, name, shard, expert)
del weight
if call_index != len(calls) - 1:
assert info.can_load(), (
"Layer processed before its final checkpoint shard"
)
assert not info.can_load(), (
"Padding must not defer the layer until finalization"
)
assert not info.loaded_weights
assert len(processed) == cycle * len(model) + layer_index + 1
assert all(source() is None for source in inputs)
for name, original in original_params[layer_index].items():
assert getattr(layer, name) is original
# Kernel-specific tests cover padding, which is not checkpoint data.
mask = torch.isfinite(expected[name])
assert torch.equal(original[mask], expected[name][mask])
def test_get_numel_loaded(): def test_get_numel_loaded():
param = torch.empty(10, device="meta") param = torch.empty(10, device="meta")
loaded_weight = torch.empty(10) loaded_weight = torch.empty(10)
@@ -27,6 +27,7 @@ from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig, QuantizationConfig,
resolve_quant_method, resolve_quant_method,
) )
from vllm.utils.math_utils import cdiv
if TYPE_CHECKING: if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe.runner.shared_experts import SharedExperts from vllm.model_executor.layers.fused_moe.runner.shared_experts import SharedExperts
@@ -566,6 +567,23 @@ class RoutedExperts(PluggableLayer):
hidden_dim=hidden_dim, hidden_dim=hidden_dim,
shard_dim=shard_dim, shard_dim=shard_dim,
) )
if (
loaded_weight.device.type == "cpu"
and expert_data.device.type == "cuda"
and not expert_data.is_contiguous()
and expert_data.ndim == 2
):
# A strided CPU-to-CUDA copy can allocate a full-shard CUDA
# temporary. Make the TP slice contiguous on CPU and copy rows
# in chunks to limit the temporary used for the padded view.
loaded_weight = loaded_weight.contiguous()
num_chunks = max(cdiv(loaded_weight.nbytes, 1 << 20), 1)
for dst, src in zip(
expert_data.chunk(num_chunks, dim=0),
loaded_weight.chunk(num_chunks, dim=0),
):
dst.copy_(src)
return
expert_data.copy_(loaded_weight) expert_data.copy_(loaded_weight)
def _load_single_value( def _load_single_value(
@@ -11,6 +11,7 @@ from vllm.logger import init_logger
from vllm.model_executor.custom_op import CustomOp from vllm.model_executor.custom_op import CustomOp
from vllm.model_executor.layers.fused_moe.config import ( from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig, FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig, FusedMoEQuantConfig,
biased_moe_quant_config, biased_moe_quant_config,
) )
@@ -33,6 +34,7 @@ from vllm.model_executor.utils import (
set_weight_attrs, set_weight_attrs,
) )
from vllm.platforms import current_platform from vllm.platforms import current_platform
from vllm.utils.math_utils import round_up
if TYPE_CHECKING: if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
@@ -68,22 +70,38 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
params_dtype: torch.dtype, params_dtype: torch.dtype,
**extra_weight_attrs, **extra_weight_attrs,
): ):
unpadded_intermediate = self.moe.intermediate_size_per_partition_unpadded
assert unpadded_intermediate is not None
has_intermediate_padding = (
intermediate_size_per_partition > unpadded_intermediate
)
if self.moe.is_act_and_mul: if self.moe.is_act_and_mul:
w13_up_dim = 2 * intermediate_size_per_partition w13_up_dim = 2 * intermediate_size_per_partition
else: else:
w13_up_dim = intermediate_size_per_partition w13_up_dim = intermediate_size_per_partition
unpadded_hidden = self.moe.hidden_dim_unpadded
assert unpadded_hidden is not None
unpadded_up = unpadded_intermediate * (2 if self.moe.is_act_and_mul else 1)
# Fused gate_up_proj (column parallel) # Fused gate_up_proj (column parallel)
w13_weight = torch.nn.Parameter( w13_data = torch.empty(
torch.empty( num_experts,
num_experts, w13_up_dim,
w13_up_dim, hidden_size,
hidden_size, dtype=params_dtype,
dtype=params_dtype,
),
requires_grad=False,
) )
if has_intermediate_padding:
# The checkpoint loader only writes the unpadded slices. Zero the
# tails now so no full-size padding copy is needed after loading.
w13_data[:, unpadded_intermediate:intermediate_size_per_partition].zero_()
if self.moe.is_act_and_mul:
w13_data[
:,
intermediate_size_per_partition + unpadded_intermediate :,
].zero_()
w13_weight = torch.nn.Parameter(w13_data, requires_grad=False)
layer.register_parameter("w13_weight", w13_weight) layer.register_parameter("w13_weight", w13_weight)
set_weight_attrs(w13_weight, extra_weight_attrs) set_weight_attrs(w13_weight, extra_weight_attrs)
w13_weight.weight_loader_numel = num_experts * unpadded_up * unpadded_hidden
if self.moe.has_bias: if self.moe.has_bias:
w13_bias = torch.nn.Parameter( w13_bias = torch.nn.Parameter(
torch.zeros(num_experts, w13_up_dim, dtype=params_dtype), torch.zeros(num_experts, w13_up_dim, dtype=params_dtype),
@@ -91,18 +109,22 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
) )
layer.register_parameter("w13_bias", w13_bias) layer.register_parameter("w13_bias", w13_bias)
set_weight_attrs(w13_bias, extra_weight_attrs) set_weight_attrs(w13_bias, extra_weight_attrs)
w13_bias.weight_loader_numel = num_experts * unpadded_up
# down_proj (row parallel) # down_proj (row parallel)
w2_weight = torch.nn.Parameter( w2_data = torch.empty(
torch.empty( num_experts,
num_experts, hidden_size,
hidden_size, intermediate_size_per_partition,
intermediate_size_per_partition, dtype=params_dtype,
dtype=params_dtype,
),
requires_grad=False,
) )
if has_intermediate_padding:
w2_data[:, :, unpadded_intermediate:].zero_()
w2_weight = torch.nn.Parameter(w2_data, requires_grad=False)
layer.register_parameter("w2_weight", w2_weight) layer.register_parameter("w2_weight", w2_weight)
set_weight_attrs(w2_weight, extra_weight_attrs) set_weight_attrs(w2_weight, extra_weight_attrs)
w2_weight.weight_loader_numel = (
num_experts * unpadded_hidden * unpadded_intermediate
)
if self.moe.has_bias: if self.moe.has_bias:
w2_bias = torch.nn.Parameter( w2_bias = torch.nn.Parameter(
torch.zeros(num_experts, hidden_size, dtype=params_dtype), torch.zeros(num_experts, hidden_size, dtype=params_dtype),
@@ -110,6 +132,26 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
) )
layer.register_parameter("w2_bias", w2_bias) layer.register_parameter("w2_bias", w2_bias)
set_weight_attrs(w2_bias, extra_weight_attrs) set_weight_attrs(w2_bias, extra_weight_attrs)
w2_bias.weight_loader_numel = num_experts * unpadded_hidden
def maybe_roundup_sizes(
self,
hidden_size: int,
intermediate_size_per_partition: int,
act_dtype: torch.dtype,
moe_parallel_config: FusedMoEParallelConfig,
) -> tuple[int, int]:
hidden_size, intermediate_size_per_partition = super().maybe_roundup_sizes(
hidden_size,
intermediate_size_per_partition,
act_dtype,
moe_parallel_config,
)
if self.unquantized_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM:
intermediate_size_per_partition = round_up(
intermediate_size_per_partition, 128
)
return hidden_size, intermediate_size_per_partition
def _maybe_pad_weight(self, weight: torch.Tensor) -> torch.Tensor: def _maybe_pad_weight(self, weight: torch.Tensor) -> torch.Tensor:
# Pad the weight tensor. This is an optimization on ROCm platform, which # Pad the weight tensor. This is an optimization on ROCm platform, which
@@ -135,6 +177,21 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
w13: torch.Tensor, w13: torch.Tensor,
w2: torch.Tensor, w2: torch.Tensor,
) -> None: ) -> None:
w13_parameter_shape = w13.shape
w2_parameter_shape = w2.shape
if self.unquantized_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM:
unpadded = self.moe.intermediate_size_per_partition_unpadded
assert unpadded is not None
intermediate = w2.shape[-1]
# Reloads overwrite only checkpoint slices; the previous in-place
# permutation can leave nonzero values in the raw padding slots.
if intermediate > unpadded:
w13[:, unpadded:intermediate].zero_()
if self.moe.is_act_and_mul:
w13[:, intermediate + unpadded :].zero_()
w2[:, :, unpadded:].zero_()
# Shuffle weights to runtime format. # Shuffle weights to runtime format.
w13_new, w2_new = convert_to_unquantized_kernel_format( w13_new, w2_new = convert_to_unquantized_kernel_format(
self.unquantized_backend, self.unquantized_backend,
@@ -142,6 +199,10 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
w13_weight=w13, w13_weight=w13,
w2_weight=w2, w2_weight=w2,
) )
if self.unquantized_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM:
w13_new = w13_new.view(w13_parameter_shape)
w2_new = w2_new.view(w2_parameter_shape)
# `moe_kernel` is initialized to None in FusedMoEMethodBase.__init__; # `moe_kernel` is initialized to None in FusedMoEMethodBase.__init__;
# On the first call we replace the parameter normally. On subsequent # On the first call we replace the parameter normally. On subsequent
# calls (e.g. RL weight updates that re-trigger # calls (e.g. RL weight updates that re-trigger
@@ -170,6 +231,25 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
assert self.moe_kernel is not None assert self.moe_kernel is not None
self.moe_kernel.fused_experts.process_weights_after_loading(layer) self.moe_kernel.fused_experts.process_weights_after_loading(layer)
def _kernel_weights(
self, layer: "RoutedExperts"
) -> tuple[torch.Tensor, torch.Tensor]:
if self.unquantized_backend != UnquantizedMoeBackend.FLASHINFER_TRTLLM:
return layer.w13_weight, layer.w2_weight
# IPC caches restore tensor shapes/storage, not method attributes.
# BlockMajorK uses 128-byte blocks, i.e. 64 BF16 elements.
def block_view(weight: torch.Tensor) -> torch.Tensor:
if weight.ndim == 4:
return weight
experts, rows, cols = weight.shape
return weight.view(experts, cols // 64, rows, 64)
return (
block_view(layer.w13_weight),
block_view(layer.w2_weight),
)
def _init_moe_kernel(self, layer: "RoutedExperts") -> None: def _init_moe_kernel(self, layer: "RoutedExperts") -> None:
"""Build the MoE kernel from the layer's current (shuffled) weights.""" """Build the MoE kernel from the layer's current (shuffled) weights."""
self.moe_quant_config = self.get_fused_moe_quant_config(layer) self.moe_quant_config = self.get_fused_moe_quant_config(layer)
@@ -274,10 +354,11 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
shared_experts_input: torch.Tensor | None, shared_experts_input: torch.Tensor | None,
) -> torch.Tensor: ) -> torch.Tensor:
assert self.moe_kernel is not None assert self.moe_kernel is not None
w13, w2 = self._kernel_weights(layer)
return self.moe_kernel.apply( return self.moe_kernel.apply(
hidden_states=x, hidden_states=x,
w1=layer.w13_weight, w1=w13,
w2=layer.w2_weight, w2=w2,
topk_weights=topk_weights, topk_weights=topk_weights,
topk_ids=topk_ids, topk_ids=topk_ids,
activation=layer.activation, activation=layer.activation,
@@ -315,10 +396,11 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
) -> torch.Tensor | UnfinalizedMoEOutput: ) -> torch.Tensor | UnfinalizedMoEOutput:
assert self.is_monolithic assert self.is_monolithic
assert self.moe_kernel is not None assert self.moe_kernel is not None
w13, w2 = self._kernel_weights(layer)
return self.moe_kernel.apply_monolithic( return self.moe_kernel.apply_monolithic(
x, x,
layer.w13_weight, w13,
layer.w2_weight, w2,
router_logits, router_logits,
activation=layer.activation, activation=layer.activation,
global_num_experts=layer.global_num_experts, global_num_experts=layer.global_num_experts,
@@ -112,8 +112,9 @@ def convert_moe_weights_to_flashinfer_trtllm_block_layout(
) -> tuple[torch.Tensor, torch.Tensor]: ) -> tuple[torch.Tensor, torch.Tensor]:
"""Convert expert weights to FlashInfer's block layout. """Convert expert weights to FlashInfer's block layout.
This reorders W13 and W2 into the expected epilogue-tiled block layout and This reorders W13 and W2 in place into the expected epilogue-tiled block
returns the shuffled weight tensors. layout and returns views of the shuffled weight tensors. Using one expert
as scratch space avoids allocating another full copy of both weights.
""" """
if w13_weight.dtype != torch.bfloat16 or w2_weight.dtype != torch.bfloat16: if w13_weight.dtype != torch.bfloat16 or w2_weight.dtype != torch.bfloat16:
raise ValueError( raise ValueError(
@@ -147,51 +148,50 @@ def convert_moe_weights_to_flashinfer_trtllm_block_layout(
out=out, out=out,
) )
w13_rows, w13_cols = w13_weight[0].view(torch.uint8).shape def _convert_weight_in_place(
w2_rows, w2_cols = w2_weight[0].view(torch.uint8).shape weight: torch.Tensor,
w13_weights_shuffled_tensor = torch.empty( is_w13: bool,
(num_experts, w13_cols // block_k, w13_rows, block_k), ) -> torch.Tensor:
dtype=torch.uint8, rows, cols = weight[0].view(torch.uint8).shape
device=w13_weight.device, block_layout_shape = (num_experts, cols // block_k, rows, block_k)
) expert_scratch = torch.empty(
w2_weights_shuffled_tensor = torch.empty( block_layout_shape[1:],
(num_experts, w2_cols // block_k, w2_rows, block_k), dtype=torch.uint8,
dtype=torch.uint8, device=weight.device,
device=w2_weight.device,
)
for i in range(num_experts):
w13_expert_uint8 = w13_weight[i].view(torch.uint8)
permute_indices = _maybe_get_cached_w3_w1_permute_indices(
cache_permute_indices,
w13_expert_uint8,
epilogue_tile_m,
is_gated_act_gemm=is_gated_act_gemm,
)
if is_gated_act_gemm:
rows = w13_expert_uint8.shape[0]
permute_indices = (permute_indices + rows // 2) % rows
_copy_permuted_expert_to_block_layout(
w13_weights_shuffled_tensor[i],
w13_expert_uint8,
permute_indices,
) )
permute_indices = get_w2_permute_indices_with_cache( for i in range(num_experts):
cache_permute_indices, expert_uint8 = weight[i].view(torch.uint8)
w2_weight[i].view(torch.uint8), if is_w13:
epilogue_tile_m, permute_indices = _maybe_get_cached_w3_w1_permute_indices(
) cache_permute_indices,
_copy_permuted_expert_to_block_layout( expert_uint8,
w2_weights_shuffled_tensor[i], epilogue_tile_m,
w2_weight[i].view(torch.uint8), is_gated_act_gemm=is_gated_act_gemm,
permute_indices, )
) if is_gated_act_gemm:
permute_indices = (
permute_indices + expert_uint8.shape[0] // 2
) % expert_uint8.shape[0]
else:
permute_indices = get_w2_permute_indices_with_cache(
cache_permute_indices,
expert_uint8,
epilogue_tile_m,
)
_copy_permuted_expert_to_block_layout(
expert_scratch,
expert_uint8,
permute_indices,
)
expert_uint8.view(-1).copy_(expert_scratch.view(-1))
return weight.view(torch.uint8).view(block_layout_shape).view(torch.bfloat16)
return ( return (
w13_weights_shuffled_tensor.view(torch.bfloat16), _convert_weight_in_place(w13_weight, is_w13=True),
w2_weights_shuffled_tensor.view(torch.bfloat16), _convert_weight_in_place(w2_weight, is_w13=False),
) )
@@ -9,7 +9,7 @@ from torch.utils._python_dispatch import TorchDispatchMode
from .sanitize import restore_layer_refs, sanitize_layer_refs from .sanitize import restore_layer_refs, sanitize_layer_refs
from .types import LayerReloadingInfo, LayerTensors from .types import LayerReloadingInfo, LayerTensors
from .utils import get_layer_params_buffers, get_layer_tensors from .utils import get_layer_params_buffers, get_layer_tensors, get_tensor_load_numel
__all__ = [ __all__ = [
"to_meta_tensor", "to_meta_tensor",
@@ -206,5 +206,5 @@ def get_numel_loaded(
numel = counter.copied_numel numel = counter.copied_numel
param = args.arguments.get("param", None) param = args.arguments.get("param", None)
if isinstance(param, torch.Tensor): if isinstance(param, torch.Tensor):
numel = min(numel, param.numel()) numel = min(numel, get_tensor_load_numel(param))
return numel, return_value return numel, return_value
@@ -11,6 +11,7 @@ __all__ = [
"get_layer_tensors", "get_layer_tensors",
"get_layer_params_buffers", "get_layer_params_buffers",
"get_layer_size", "get_layer_size",
"get_tensor_load_numel",
"has_device_tensors", "has_device_tensors",
"get_info_size", "get_info_size",
] ]
@@ -30,6 +31,11 @@ def get_layer_params_buffers(layer: torch.nn.Module) -> LayerTensors:
) )
def get_tensor_load_numel(tensor: torch.Tensor) -> int:
"""Count checkpoint elements, excluding padding declared by the weight creator."""
return getattr(tensor, "weight_loader_numel", tensor.numel())
def get_layer_size(layer: torch.nn.Module) -> int: def get_layer_size(layer: torch.nn.Module) -> int:
"""Calculate total number of elements across loadable tensors in a layer. """Calculate total number of elements across loadable tensors in a layer.
@@ -39,7 +45,7 @@ def get_layer_size(layer: torch.nn.Module) -> int:
from .meta import SKIP_LOAD_TENSORS from .meta import SKIP_LOAD_TENSORS
return sum( return sum(
tensor.numel() get_tensor_load_numel(tensor)
for name, tensor in get_layer_tensors(layer).items() for name, tensor in get_layer_tensors(layer).items()
if name not in SKIP_LOAD_TENSORS if name not in SKIP_LOAD_TENSORS
) )