[Bugfix][MoE] Convert FlashInfer BF16 weights in place (#54699)
Signed-off-by: Yuchen Wang <93700456+yuchenwang3@users.noreply.github.com> Signed-off-by: Dakai An <dakaian108@gmail.com> Co-authored-by: Dakai An <dakaian108@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
co-authored by
Dakai An
OpenAI Codex
aoshen02
parent
117cf43fb6
commit
78fdf4efb8
@@ -27,6 +27,7 @@ from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
resolve_quant_method,
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.fused_moe.runner.shared_experts import SharedExperts
|
||||
@@ -566,6 +567,23 @@ class RoutedExperts(PluggableLayer):
|
||||
hidden_dim=hidden_dim,
|
||||
shard_dim=shard_dim,
|
||||
)
|
||||
if (
|
||||
loaded_weight.device.type == "cpu"
|
||||
and expert_data.device.type == "cuda"
|
||||
and not expert_data.is_contiguous()
|
||||
and expert_data.ndim == 2
|
||||
):
|
||||
# A strided CPU-to-CUDA copy can allocate a full-shard CUDA
|
||||
# temporary. Make the TP slice contiguous on CPU and copy rows
|
||||
# in chunks to limit the temporary used for the padded view.
|
||||
loaded_weight = loaded_weight.contiguous()
|
||||
num_chunks = max(cdiv(loaded_weight.nbytes, 1 << 20), 1)
|
||||
for dst, src in zip(
|
||||
expert_data.chunk(num_chunks, dim=0),
|
||||
loaded_weight.chunk(num_chunks, dim=0),
|
||||
):
|
||||
dst.copy_(src)
|
||||
return
|
||||
expert_data.copy_(loaded_weight)
|
||||
|
||||
def _load_single_value(
|
||||
|
||||
@@ -11,6 +11,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
FusedMoEQuantConfig,
|
||||
biased_moe_quant_config,
|
||||
)
|
||||
@@ -33,6 +34,7 @@ from vllm.model_executor.utils import (
|
||||
set_weight_attrs,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
|
||||
@@ -68,22 +70,38 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
unpadded_intermediate = self.moe.intermediate_size_per_partition_unpadded
|
||||
assert unpadded_intermediate is not None
|
||||
has_intermediate_padding = (
|
||||
intermediate_size_per_partition > unpadded_intermediate
|
||||
)
|
||||
if self.moe.is_act_and_mul:
|
||||
w13_up_dim = 2 * intermediate_size_per_partition
|
||||
else:
|
||||
w13_up_dim = intermediate_size_per_partition
|
||||
unpadded_hidden = self.moe.hidden_dim_unpadded
|
||||
assert unpadded_hidden is not None
|
||||
unpadded_up = unpadded_intermediate * (2 if self.moe.is_act_and_mul else 1)
|
||||
# Fused gate_up_proj (column parallel)
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
w13_up_dim,
|
||||
hidden_size,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
requires_grad=False,
|
||||
w13_data = torch.empty(
|
||||
num_experts,
|
||||
w13_up_dim,
|
||||
hidden_size,
|
||||
dtype=params_dtype,
|
||||
)
|
||||
if has_intermediate_padding:
|
||||
# The checkpoint loader only writes the unpadded slices. Zero the
|
||||
# tails now so no full-size padding copy is needed after loading.
|
||||
w13_data[:, unpadded_intermediate:intermediate_size_per_partition].zero_()
|
||||
if self.moe.is_act_and_mul:
|
||||
w13_data[
|
||||
:,
|
||||
intermediate_size_per_partition + unpadded_intermediate :,
|
||||
].zero_()
|
||||
w13_weight = torch.nn.Parameter(w13_data, requires_grad=False)
|
||||
layer.register_parameter("w13_weight", w13_weight)
|
||||
set_weight_attrs(w13_weight, extra_weight_attrs)
|
||||
w13_weight.weight_loader_numel = num_experts * unpadded_up * unpadded_hidden
|
||||
if self.moe.has_bias:
|
||||
w13_bias = torch.nn.Parameter(
|
||||
torch.zeros(num_experts, w13_up_dim, dtype=params_dtype),
|
||||
@@ -91,18 +109,22 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
)
|
||||
layer.register_parameter("w13_bias", w13_bias)
|
||||
set_weight_attrs(w13_bias, extra_weight_attrs)
|
||||
w13_bias.weight_loader_numel = num_experts * unpadded_up
|
||||
# down_proj (row parallel)
|
||||
w2_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
requires_grad=False,
|
||||
w2_data = torch.empty(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition,
|
||||
dtype=params_dtype,
|
||||
)
|
||||
if has_intermediate_padding:
|
||||
w2_data[:, :, unpadded_intermediate:].zero_()
|
||||
w2_weight = torch.nn.Parameter(w2_data, requires_grad=False)
|
||||
layer.register_parameter("w2_weight", w2_weight)
|
||||
set_weight_attrs(w2_weight, extra_weight_attrs)
|
||||
w2_weight.weight_loader_numel = (
|
||||
num_experts * unpadded_hidden * unpadded_intermediate
|
||||
)
|
||||
if self.moe.has_bias:
|
||||
w2_bias = torch.nn.Parameter(
|
||||
torch.zeros(num_experts, hidden_size, dtype=params_dtype),
|
||||
@@ -110,6 +132,26 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
)
|
||||
layer.register_parameter("w2_bias", w2_bias)
|
||||
set_weight_attrs(w2_bias, extra_weight_attrs)
|
||||
w2_bias.weight_loader_numel = num_experts * unpadded_hidden
|
||||
|
||||
def maybe_roundup_sizes(
|
||||
self,
|
||||
hidden_size: int,
|
||||
intermediate_size_per_partition: int,
|
||||
act_dtype: torch.dtype,
|
||||
moe_parallel_config: FusedMoEParallelConfig,
|
||||
) -> tuple[int, int]:
|
||||
hidden_size, intermediate_size_per_partition = super().maybe_roundup_sizes(
|
||||
hidden_size,
|
||||
intermediate_size_per_partition,
|
||||
act_dtype,
|
||||
moe_parallel_config,
|
||||
)
|
||||
if self.unquantized_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM:
|
||||
intermediate_size_per_partition = round_up(
|
||||
intermediate_size_per_partition, 128
|
||||
)
|
||||
return hidden_size, intermediate_size_per_partition
|
||||
|
||||
def _maybe_pad_weight(self, weight: torch.Tensor) -> torch.Tensor:
|
||||
# Pad the weight tensor. This is an optimization on ROCm platform, which
|
||||
@@ -135,6 +177,21 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
w13: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
) -> None:
|
||||
w13_parameter_shape = w13.shape
|
||||
w2_parameter_shape = w2.shape
|
||||
|
||||
if self.unquantized_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM:
|
||||
unpadded = self.moe.intermediate_size_per_partition_unpadded
|
||||
assert unpadded is not None
|
||||
intermediate = w2.shape[-1]
|
||||
# Reloads overwrite only checkpoint slices; the previous in-place
|
||||
# permutation can leave nonzero values in the raw padding slots.
|
||||
if intermediate > unpadded:
|
||||
w13[:, unpadded:intermediate].zero_()
|
||||
if self.moe.is_act_and_mul:
|
||||
w13[:, intermediate + unpadded :].zero_()
|
||||
w2[:, :, unpadded:].zero_()
|
||||
|
||||
# Shuffle weights to runtime format.
|
||||
w13_new, w2_new = convert_to_unquantized_kernel_format(
|
||||
self.unquantized_backend,
|
||||
@@ -142,6 +199,10 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
w13_weight=w13,
|
||||
w2_weight=w2,
|
||||
)
|
||||
if self.unquantized_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM:
|
||||
w13_new = w13_new.view(w13_parameter_shape)
|
||||
w2_new = w2_new.view(w2_parameter_shape)
|
||||
|
||||
# `moe_kernel` is initialized to None in FusedMoEMethodBase.__init__;
|
||||
# On the first call we replace the parameter normally. On subsequent
|
||||
# calls (e.g. RL weight updates that re-trigger
|
||||
@@ -170,6 +231,25 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
assert self.moe_kernel is not None
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
def _kernel_weights(
|
||||
self, layer: "RoutedExperts"
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if self.unquantized_backend != UnquantizedMoeBackend.FLASHINFER_TRTLLM:
|
||||
return layer.w13_weight, layer.w2_weight
|
||||
|
||||
# IPC caches restore tensor shapes/storage, not method attributes.
|
||||
# BlockMajorK uses 128-byte blocks, i.e. 64 BF16 elements.
|
||||
def block_view(weight: torch.Tensor) -> torch.Tensor:
|
||||
if weight.ndim == 4:
|
||||
return weight
|
||||
experts, rows, cols = weight.shape
|
||||
return weight.view(experts, cols // 64, rows, 64)
|
||||
|
||||
return (
|
||||
block_view(layer.w13_weight),
|
||||
block_view(layer.w2_weight),
|
||||
)
|
||||
|
||||
def _init_moe_kernel(self, layer: "RoutedExperts") -> None:
|
||||
"""Build the MoE kernel from the layer's current (shuffled) weights."""
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
@@ -274,10 +354,11 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
assert self.moe_kernel is not None
|
||||
w13, w2 = self._kernel_weights(layer)
|
||||
return self.moe_kernel.apply(
|
||||
hidden_states=x,
|
||||
w1=layer.w13_weight,
|
||||
w2=layer.w2_weight,
|
||||
w1=w13,
|
||||
w2=w2,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=layer.activation,
|
||||
@@ -315,10 +396,11 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert self.is_monolithic
|
||||
assert self.moe_kernel is not None
|
||||
w13, w2 = self._kernel_weights(layer)
|
||||
return self.moe_kernel.apply_monolithic(
|
||||
x,
|
||||
layer.w13_weight,
|
||||
layer.w2_weight,
|
||||
w13,
|
||||
w2,
|
||||
router_logits,
|
||||
activation=layer.activation,
|
||||
global_num_experts=layer.global_num_experts,
|
||||
|
||||
@@ -112,8 +112,9 @@ def convert_moe_weights_to_flashinfer_trtllm_block_layout(
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Convert expert weights to FlashInfer's block layout.
|
||||
|
||||
This reorders W13 and W2 into the expected epilogue-tiled block layout and
|
||||
returns the shuffled weight tensors.
|
||||
This reorders W13 and W2 in place into the expected epilogue-tiled block
|
||||
layout and returns views of the shuffled weight tensors. Using one expert
|
||||
as scratch space avoids allocating another full copy of both weights.
|
||||
"""
|
||||
if w13_weight.dtype != torch.bfloat16 or w2_weight.dtype != torch.bfloat16:
|
||||
raise ValueError(
|
||||
@@ -147,51 +148,50 @@ def convert_moe_weights_to_flashinfer_trtllm_block_layout(
|
||||
out=out,
|
||||
)
|
||||
|
||||
w13_rows, w13_cols = w13_weight[0].view(torch.uint8).shape
|
||||
w2_rows, w2_cols = w2_weight[0].view(torch.uint8).shape
|
||||
w13_weights_shuffled_tensor = torch.empty(
|
||||
(num_experts, w13_cols // block_k, w13_rows, block_k),
|
||||
dtype=torch.uint8,
|
||||
device=w13_weight.device,
|
||||
)
|
||||
w2_weights_shuffled_tensor = torch.empty(
|
||||
(num_experts, w2_cols // block_k, w2_rows, block_k),
|
||||
dtype=torch.uint8,
|
||||
device=w2_weight.device,
|
||||
)
|
||||
|
||||
for i in range(num_experts):
|
||||
w13_expert_uint8 = w13_weight[i].view(torch.uint8)
|
||||
|
||||
permute_indices = _maybe_get_cached_w3_w1_permute_indices(
|
||||
cache_permute_indices,
|
||||
w13_expert_uint8,
|
||||
epilogue_tile_m,
|
||||
is_gated_act_gemm=is_gated_act_gemm,
|
||||
)
|
||||
if is_gated_act_gemm:
|
||||
rows = w13_expert_uint8.shape[0]
|
||||
permute_indices = (permute_indices + rows // 2) % rows
|
||||
_copy_permuted_expert_to_block_layout(
|
||||
w13_weights_shuffled_tensor[i],
|
||||
w13_expert_uint8,
|
||||
permute_indices,
|
||||
def _convert_weight_in_place(
|
||||
weight: torch.Tensor,
|
||||
is_w13: bool,
|
||||
) -> torch.Tensor:
|
||||
rows, cols = weight[0].view(torch.uint8).shape
|
||||
block_layout_shape = (num_experts, cols // block_k, rows, block_k)
|
||||
expert_scratch = torch.empty(
|
||||
block_layout_shape[1:],
|
||||
dtype=torch.uint8,
|
||||
device=weight.device,
|
||||
)
|
||||
|
||||
permute_indices = get_w2_permute_indices_with_cache(
|
||||
cache_permute_indices,
|
||||
w2_weight[i].view(torch.uint8),
|
||||
epilogue_tile_m,
|
||||
)
|
||||
_copy_permuted_expert_to_block_layout(
|
||||
w2_weights_shuffled_tensor[i],
|
||||
w2_weight[i].view(torch.uint8),
|
||||
permute_indices,
|
||||
)
|
||||
for i in range(num_experts):
|
||||
expert_uint8 = weight[i].view(torch.uint8)
|
||||
if is_w13:
|
||||
permute_indices = _maybe_get_cached_w3_w1_permute_indices(
|
||||
cache_permute_indices,
|
||||
expert_uint8,
|
||||
epilogue_tile_m,
|
||||
is_gated_act_gemm=is_gated_act_gemm,
|
||||
)
|
||||
if is_gated_act_gemm:
|
||||
permute_indices = (
|
||||
permute_indices + expert_uint8.shape[0] // 2
|
||||
) % expert_uint8.shape[0]
|
||||
else:
|
||||
permute_indices = get_w2_permute_indices_with_cache(
|
||||
cache_permute_indices,
|
||||
expert_uint8,
|
||||
epilogue_tile_m,
|
||||
)
|
||||
|
||||
_copy_permuted_expert_to_block_layout(
|
||||
expert_scratch,
|
||||
expert_uint8,
|
||||
permute_indices,
|
||||
)
|
||||
expert_uint8.view(-1).copy_(expert_scratch.view(-1))
|
||||
|
||||
return weight.view(torch.uint8).view(block_layout_shape).view(torch.bfloat16)
|
||||
|
||||
return (
|
||||
w13_weights_shuffled_tensor.view(torch.bfloat16),
|
||||
w2_weights_shuffled_tensor.view(torch.bfloat16),
|
||||
_convert_weight_in_place(w13_weight, is_w13=True),
|
||||
_convert_weight_in_place(w2_weight, is_w13=False),
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user