[Bugfix][MoE] Convert FlashInfer BF16 weights in place (#54699)

Signed-off-by: Yuchen Wang <93700456+yuchenwang3@users.noreply.github.com>
Signed-off-by: Dakai An <dakaian108@gmail.com>
Co-authored-by: Dakai An <dakaian108@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
Yuchen Wang
2026-09-17 22:00:04 +08:00
committed by GitHub
co-authored by Dakai An OpenAI Codex aoshen02
parent 117cf43fb6
commit 78fdf4efb8
9 changed files with 667 additions and 68 deletions
@@ -27,6 +27,7 @@ from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
resolve_quant_method,
)
from vllm.utils.math_utils import cdiv
if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe.runner.shared_experts import SharedExperts
@@ -566,6 +567,23 @@ class RoutedExperts(PluggableLayer):
hidden_dim=hidden_dim,
shard_dim=shard_dim,
)
if (
loaded_weight.device.type == "cpu"
and expert_data.device.type == "cuda"
and not expert_data.is_contiguous()
and expert_data.ndim == 2
):
# A strided CPU-to-CUDA copy can allocate a full-shard CUDA
# temporary. Make the TP slice contiguous on CPU and copy rows
# in chunks to limit the temporary used for the padded view.
loaded_weight = loaded_weight.contiguous()
num_chunks = max(cdiv(loaded_weight.nbytes, 1 << 20), 1)
for dst, src in zip(
expert_data.chunk(num_chunks, dim=0),
loaded_weight.chunk(num_chunks, dim=0),
):
dst.copy_(src)
return
expert_data.copy_(loaded_weight)
def _load_single_value(
@@ -11,6 +11,7 @@ from vllm.logger import init_logger
from vllm.model_executor.custom_op import CustomOp
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
biased_moe_quant_config,
)
@@ -33,6 +34,7 @@ from vllm.model_executor.utils import (
set_weight_attrs,
)
from vllm.platforms import current_platform
from vllm.utils.math_utils import round_up
if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
@@ -68,22 +70,38 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
params_dtype: torch.dtype,
**extra_weight_attrs,
):
unpadded_intermediate = self.moe.intermediate_size_per_partition_unpadded
assert unpadded_intermediate is not None
has_intermediate_padding = (
intermediate_size_per_partition > unpadded_intermediate
)
if self.moe.is_act_and_mul:
w13_up_dim = 2 * intermediate_size_per_partition
else:
w13_up_dim = intermediate_size_per_partition
unpadded_hidden = self.moe.hidden_dim_unpadded
assert unpadded_hidden is not None
unpadded_up = unpadded_intermediate * (2 if self.moe.is_act_and_mul else 1)
# Fused gate_up_proj (column parallel)
w13_weight = torch.nn.Parameter(
torch.empty(
num_experts,
w13_up_dim,
hidden_size,
dtype=params_dtype,
),
requires_grad=False,
w13_data = torch.empty(
num_experts,
w13_up_dim,
hidden_size,
dtype=params_dtype,
)
if has_intermediate_padding:
# The checkpoint loader only writes the unpadded slices. Zero the
# tails now so no full-size padding copy is needed after loading.
w13_data[:, unpadded_intermediate:intermediate_size_per_partition].zero_()
if self.moe.is_act_and_mul:
w13_data[
:,
intermediate_size_per_partition + unpadded_intermediate :,
].zero_()
w13_weight = torch.nn.Parameter(w13_data, requires_grad=False)
layer.register_parameter("w13_weight", w13_weight)
set_weight_attrs(w13_weight, extra_weight_attrs)
w13_weight.weight_loader_numel = num_experts * unpadded_up * unpadded_hidden
if self.moe.has_bias:
w13_bias = torch.nn.Parameter(
torch.zeros(num_experts, w13_up_dim, dtype=params_dtype),
@@ -91,18 +109,22 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
)
layer.register_parameter("w13_bias", w13_bias)
set_weight_attrs(w13_bias, extra_weight_attrs)
w13_bias.weight_loader_numel = num_experts * unpadded_up
# down_proj (row parallel)
w2_weight = torch.nn.Parameter(
torch.empty(
num_experts,
hidden_size,
intermediate_size_per_partition,
dtype=params_dtype,
),
requires_grad=False,
w2_data = torch.empty(
num_experts,
hidden_size,
intermediate_size_per_partition,
dtype=params_dtype,
)
if has_intermediate_padding:
w2_data[:, :, unpadded_intermediate:].zero_()
w2_weight = torch.nn.Parameter(w2_data, requires_grad=False)
layer.register_parameter("w2_weight", w2_weight)
set_weight_attrs(w2_weight, extra_weight_attrs)
w2_weight.weight_loader_numel = (
num_experts * unpadded_hidden * unpadded_intermediate
)
if self.moe.has_bias:
w2_bias = torch.nn.Parameter(
torch.zeros(num_experts, hidden_size, dtype=params_dtype),
@@ -110,6 +132,26 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
)
layer.register_parameter("w2_bias", w2_bias)
set_weight_attrs(w2_bias, extra_weight_attrs)
w2_bias.weight_loader_numel = num_experts * unpadded_hidden
def maybe_roundup_sizes(
self,
hidden_size: int,
intermediate_size_per_partition: int,
act_dtype: torch.dtype,
moe_parallel_config: FusedMoEParallelConfig,
) -> tuple[int, int]:
hidden_size, intermediate_size_per_partition = super().maybe_roundup_sizes(
hidden_size,
intermediate_size_per_partition,
act_dtype,
moe_parallel_config,
)
if self.unquantized_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM:
intermediate_size_per_partition = round_up(
intermediate_size_per_partition, 128
)
return hidden_size, intermediate_size_per_partition
def _maybe_pad_weight(self, weight: torch.Tensor) -> torch.Tensor:
# Pad the weight tensor. This is an optimization on ROCm platform, which
@@ -135,6 +177,21 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
w13: torch.Tensor,
w2: torch.Tensor,
) -> None:
w13_parameter_shape = w13.shape
w2_parameter_shape = w2.shape
if self.unquantized_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM:
unpadded = self.moe.intermediate_size_per_partition_unpadded
assert unpadded is not None
intermediate = w2.shape[-1]
# Reloads overwrite only checkpoint slices; the previous in-place
# permutation can leave nonzero values in the raw padding slots.
if intermediate > unpadded:
w13[:, unpadded:intermediate].zero_()
if self.moe.is_act_and_mul:
w13[:, intermediate + unpadded :].zero_()
w2[:, :, unpadded:].zero_()
# Shuffle weights to runtime format.
w13_new, w2_new = convert_to_unquantized_kernel_format(
self.unquantized_backend,
@@ -142,6 +199,10 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
w13_weight=w13,
w2_weight=w2,
)
if self.unquantized_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM:
w13_new = w13_new.view(w13_parameter_shape)
w2_new = w2_new.view(w2_parameter_shape)
# `moe_kernel` is initialized to None in FusedMoEMethodBase.__init__;
# On the first call we replace the parameter normally. On subsequent
# calls (e.g. RL weight updates that re-trigger
@@ -170,6 +231,25 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
assert self.moe_kernel is not None
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
def _kernel_weights(
self, layer: "RoutedExperts"
) -> tuple[torch.Tensor, torch.Tensor]:
if self.unquantized_backend != UnquantizedMoeBackend.FLASHINFER_TRTLLM:
return layer.w13_weight, layer.w2_weight
# IPC caches restore tensor shapes/storage, not method attributes.
# BlockMajorK uses 128-byte blocks, i.e. 64 BF16 elements.
def block_view(weight: torch.Tensor) -> torch.Tensor:
if weight.ndim == 4:
return weight
experts, rows, cols = weight.shape
return weight.view(experts, cols // 64, rows, 64)
return (
block_view(layer.w13_weight),
block_view(layer.w2_weight),
)
def _init_moe_kernel(self, layer: "RoutedExperts") -> None:
"""Build the MoE kernel from the layer's current (shuffled) weights."""
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
@@ -274,10 +354,11 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
assert self.moe_kernel is not None
w13, w2 = self._kernel_weights(layer)
return self.moe_kernel.apply(
hidden_states=x,
w1=layer.w13_weight,
w2=layer.w2_weight,
w1=w13,
w2=w2,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=layer.activation,
@@ -315,10 +396,11 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
) -> torch.Tensor | UnfinalizedMoEOutput:
assert self.is_monolithic
assert self.moe_kernel is not None
w13, w2 = self._kernel_weights(layer)
return self.moe_kernel.apply_monolithic(
x,
layer.w13_weight,
layer.w2_weight,
w13,
w2,
router_logits,
activation=layer.activation,
global_num_experts=layer.global_num_experts,
@@ -112,8 +112,9 @@ def convert_moe_weights_to_flashinfer_trtllm_block_layout(
) -> tuple[torch.Tensor, torch.Tensor]:
"""Convert expert weights to FlashInfer's block layout.
This reorders W13 and W2 into the expected epilogue-tiled block layout and
returns the shuffled weight tensors.
This reorders W13 and W2 in place into the expected epilogue-tiled block
layout and returns views of the shuffled weight tensors. Using one expert
as scratch space avoids allocating another full copy of both weights.
"""
if w13_weight.dtype != torch.bfloat16 or w2_weight.dtype != torch.bfloat16:
raise ValueError(
@@ -147,51 +148,50 @@ def convert_moe_weights_to_flashinfer_trtllm_block_layout(
out=out,
)
w13_rows, w13_cols = w13_weight[0].view(torch.uint8).shape
w2_rows, w2_cols = w2_weight[0].view(torch.uint8).shape
w13_weights_shuffled_tensor = torch.empty(
(num_experts, w13_cols // block_k, w13_rows, block_k),
dtype=torch.uint8,
device=w13_weight.device,
)
w2_weights_shuffled_tensor = torch.empty(
(num_experts, w2_cols // block_k, w2_rows, block_k),
dtype=torch.uint8,
device=w2_weight.device,
)
for i in range(num_experts):
w13_expert_uint8 = w13_weight[i].view(torch.uint8)
permute_indices = _maybe_get_cached_w3_w1_permute_indices(
cache_permute_indices,
w13_expert_uint8,
epilogue_tile_m,
is_gated_act_gemm=is_gated_act_gemm,
)
if is_gated_act_gemm:
rows = w13_expert_uint8.shape[0]
permute_indices = (permute_indices + rows // 2) % rows
_copy_permuted_expert_to_block_layout(
w13_weights_shuffled_tensor[i],
w13_expert_uint8,
permute_indices,
def _convert_weight_in_place(
weight: torch.Tensor,
is_w13: bool,
) -> torch.Tensor:
rows, cols = weight[0].view(torch.uint8).shape
block_layout_shape = (num_experts, cols // block_k, rows, block_k)
expert_scratch = torch.empty(
block_layout_shape[1:],
dtype=torch.uint8,
device=weight.device,
)
permute_indices = get_w2_permute_indices_with_cache(
cache_permute_indices,
w2_weight[i].view(torch.uint8),
epilogue_tile_m,
)
_copy_permuted_expert_to_block_layout(
w2_weights_shuffled_tensor[i],
w2_weight[i].view(torch.uint8),
permute_indices,
)
for i in range(num_experts):
expert_uint8 = weight[i].view(torch.uint8)
if is_w13:
permute_indices = _maybe_get_cached_w3_w1_permute_indices(
cache_permute_indices,
expert_uint8,
epilogue_tile_m,
is_gated_act_gemm=is_gated_act_gemm,
)
if is_gated_act_gemm:
permute_indices = (
permute_indices + expert_uint8.shape[0] // 2
) % expert_uint8.shape[0]
else:
permute_indices = get_w2_permute_indices_with_cache(
cache_permute_indices,
expert_uint8,
epilogue_tile_m,
)
_copy_permuted_expert_to_block_layout(
expert_scratch,
expert_uint8,
permute_indices,
)
expert_uint8.view(-1).copy_(expert_scratch.view(-1))
return weight.view(torch.uint8).view(block_layout_shape).view(torch.bfloat16)
return (
w13_weights_shuffled_tensor.view(torch.bfloat16),
w2_weights_shuffled_tensor.view(torch.bfloat16),
_convert_weight_in_place(w13_weight, is_w13=True),
_convert_weight_in_place(w2_weight, is_w13=False),
)
@@ -9,7 +9,7 @@ from torch.utils._python_dispatch import TorchDispatchMode
from .sanitize import restore_layer_refs, sanitize_layer_refs
from .types import LayerReloadingInfo, LayerTensors
from .utils import get_layer_params_buffers, get_layer_tensors
from .utils import get_layer_params_buffers, get_layer_tensors, get_tensor_load_numel
__all__ = [
"to_meta_tensor",
@@ -206,5 +206,5 @@ def get_numel_loaded(
numel = counter.copied_numel
param = args.arguments.get("param", None)
if isinstance(param, torch.Tensor):
numel = min(numel, param.numel())
numel = min(numel, get_tensor_load_numel(param))
return numel, return_value
@@ -11,6 +11,7 @@ __all__ = [
"get_layer_tensors",
"get_layer_params_buffers",
"get_layer_size",
"get_tensor_load_numel",
"has_device_tensors",
"get_info_size",
]
@@ -30,6 +31,11 @@ def get_layer_params_buffers(layer: torch.nn.Module) -> LayerTensors:
)
def get_tensor_load_numel(tensor: torch.Tensor) -> int:
"""Count checkpoint elements, excluding padding declared by the weight creator."""
return getattr(tensor, "weight_loader_numel", tensor.numel())
def get_layer_size(layer: torch.nn.Module) -> int:
"""Calculate total number of elements across loadable tensors in a layer.
@@ -39,7 +45,7 @@ def get_layer_size(layer: torch.nn.Module) -> int:
from .meta import SKIP_LOAD_TENSORS
return sum(
tensor.numel()
get_tensor_load_numel(tensor)
for name, tensor in get_layer_tensors(layer).items()
if name not in SKIP_LOAD_TENSORS
)