[Weight processing] Copy over new_data attributes in replace_parameter (#49601)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
b38e111d3e
commit
81be2e09ae
@@ -145,11 +145,6 @@ class AiterMxfp8Experts(Mxfp8TritonExpertsBase):
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
# Re-tag the preshuffled weights: replace_parameter drops the
|
||||
# is_shuffled flag, without which aiter picks a broken CK kernel.
|
||||
w1.is_shuffled = True
|
||||
w2.is_shuffled = True
|
||||
|
||||
limit = self.quant_config.gemm1_clamp_limit
|
||||
swiglu_limit = 0.0 if limit is None else float(limit)
|
||||
|
||||
|
||||
@@ -483,10 +483,14 @@ def convert_to_fp8_moe_kernel_format(
|
||||
)
|
||||
elif fp8_backend == Fp8MoeBackend.AITER:
|
||||
w13, w2 = rocm_aiter_ops.shuffle_weights(w13, w2)
|
||||
w13.is_shuffled = True
|
||||
w2.is_shuffled = True
|
||||
elif fp8_backend == Fp8MoeBackend.AITER_MXFP8:
|
||||
w13, w2, w13_scale, w2_scale = rocm_aiter_ops.shuffle_mxfp8_moe_weights(
|
||||
w13, w2, w13_scale, w2_scale
|
||||
)
|
||||
w13.is_shuffled = True
|
||||
w2.is_shuffled = True
|
||||
elif fp8_backend == Fp8MoeBackend.HUMMING:
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
convert_to_humming_moe_kernel_format,
|
||||
|
||||
@@ -1081,6 +1081,9 @@ def convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
|
||||
.view(-1, n)
|
||||
)
|
||||
|
||||
w13_weight.is_shuffled = True
|
||||
w2_weight.is_shuffled = True
|
||||
|
||||
return (
|
||||
w13_weight,
|
||||
w2_weight,
|
||||
@@ -1504,6 +1507,9 @@ def convert_weight_to_mxfp4_moe_kernel_format(
|
||||
False,
|
||||
)
|
||||
|
||||
w13_weight.is_shuffled = True
|
||||
w2_weight.is_shuffled = True
|
||||
|
||||
return (
|
||||
w13_weight,
|
||||
w2_weight,
|
||||
|
||||
@@ -332,6 +332,8 @@ def convert_to_unquantized_kernel_format(
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if unquantized_backend == UnquantizedMoeBackend.AITER:
|
||||
w13_weight, w2_weight = rocm_aiter_ops.shuffle_weights(w13_weight, w2_weight)
|
||||
w13_weight.is_shuffled = True
|
||||
w2_weight.is_shuffled = True
|
||||
|
||||
elif unquantized_backend == UnquantizedMoeBackend.FLASHINFER_CUTLASS:
|
||||
if moe_config.is_act_and_mul:
|
||||
|
||||
@@ -146,11 +146,6 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
replace_parameter(layer, "w13_weight", w13_new, prefer_copy=is_weight_update)
|
||||
replace_parameter(layer, "w2_weight", w2_new, prefer_copy=is_weight_update)
|
||||
|
||||
# AITER backend requires weights to be marked as shuffled.
|
||||
if self.unquantized_backend == UnquantizedMoeBackend.AITER:
|
||||
layer.w13_weight.is_shuffled = True
|
||||
layer.w2_weight.is_shuffled = True
|
||||
|
||||
if not is_weight_update:
|
||||
# Setup moe kernel only on the first call. For the unquantized
|
||||
# method, moe_quant_config carries no quantized scales -- only
|
||||
|
||||
@@ -28,7 +28,6 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
Fp8MoeBackend,
|
||||
convert_to_fp8_moe_kernel_format,
|
||||
make_fp8_moe_kernel,
|
||||
make_fp8_moe_quant_config,
|
||||
@@ -671,11 +670,6 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
replace_parameter(layer, f"w13_{self.weight_scale_name}", w13_scale)
|
||||
replace_parameter(layer, f"w2_{self.weight_scale_name}", w2_scale)
|
||||
|
||||
# AITER backend requires weights to be marked as shuffled.
|
||||
if self.fp8_backend == Fp8MoeBackend.AITER:
|
||||
layer.w13_weight.is_shuffled = True
|
||||
layer.w2_weight.is_shuffled = True
|
||||
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
assert self.moe_quant_config is not None
|
||||
assert self.experts_cls is not None
|
||||
|
||||
@@ -366,11 +366,6 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
self.w13_precision_config = w13_scale
|
||||
self.w2_precision_config = w2_scale
|
||||
|
||||
# AITER backend requires weights to be marked as shuffled.
|
||||
if self.mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_BF16:
|
||||
layer.w13_weight.is_shuffled = True
|
||||
layer.w2_weight.is_shuffled = True
|
||||
|
||||
if w13_bias is not None and w2_bias is not None:
|
||||
replace_parameter(layer, "w13_bias", w13_bias)
|
||||
replace_parameter(layer, "w2_bias", w2_bias)
|
||||
@@ -760,14 +755,6 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
self.w13_precision_config = w13_scale
|
||||
self.w2_precision_config = w2_scale
|
||||
|
||||
# AITER backend requires weights to be marked as shuffled.
|
||||
if (
|
||||
self.mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_BF16
|
||||
and not uses_triton_weight_format
|
||||
):
|
||||
layer.w13_weight.is_shuffled = True
|
||||
layer.w2_weight.is_shuffled = True
|
||||
|
||||
if w13_bias is not None and w2_bias is not None:
|
||||
replace_parameter(layer, "w13_bias", w13_bias)
|
||||
replace_parameter(layer, "w2_bias", w2_bias)
|
||||
@@ -816,6 +803,10 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
w13_scale_raw.view(-1, w13_scale_raw.shape[-1]), num_experts, guinterleave
|
||||
)
|
||||
w2_scale = e8m0_shuffle(w2_scale_raw.view(-1, w2_scale_raw.shape[-1]))
|
||||
|
||||
w13.is_shuffled = True
|
||||
w2.is_shuffled = True
|
||||
|
||||
return w13, w2, w13_scale, w2_scale
|
||||
|
||||
def process_weights_after_loading(self, layer):
|
||||
|
||||
@@ -27,7 +27,6 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
ocp_mx_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
Fp8MoeBackend,
|
||||
convert_to_fp8_moe_kernel_format,
|
||||
make_fp8_moe_kernel,
|
||||
make_fp8_moe_quant_config,
|
||||
@@ -447,10 +446,6 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
replace_parameter(layer, "w13_weight_scale", w13_scale)
|
||||
replace_parameter(layer, "w2_weight_scale", w2_scale)
|
||||
|
||||
if self.fp8_backend == Fp8MoeBackend.AITER:
|
||||
layer.w13_weight.is_shuffled = True
|
||||
layer.w2_weight.is_shuffled = True
|
||||
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
assert self.moe_quant_config is not None
|
||||
assert self.experts_cls is not None
|
||||
@@ -1328,10 +1323,6 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
replace_parameter(layer, "w13_bias", w13_bias)
|
||||
replace_parameter(layer, "w2_bias", w2_bias)
|
||||
|
||||
if self.mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_MXFP4:
|
||||
layer.w13_weight.is_shuffled = True
|
||||
layer.w2_weight.is_shuffled = True
|
||||
|
||||
torch.accelerator.empty_cache()
|
||||
|
||||
# Build quant config and kernel
|
||||
|
||||
@@ -47,21 +47,29 @@ def set_weight_attrs(
|
||||
def replace_parameter(
|
||||
layer: torch.nn.Module,
|
||||
param_name: str,
|
||||
new_data: torch.Tensor | None,
|
||||
new_tensor: torch.Tensor | None,
|
||||
prefer_copy: bool = False,
|
||||
):
|
||||
"""
|
||||
Replace a parameter of a layer while maintaining the ability to reload the weight.
|
||||
Called within implementations of the `process_weights_after_loading` method.
|
||||
|
||||
Custom attributes set on ``new_tensor`` (e.g. kernel dispatch flags such as
|
||||
``is_shuffled``) are carried over to the replacement parameter, except
|
||||
``weight_loader``, which is always taken from the existing parameter.
|
||||
|
||||
Attributes of the existing parameter are otherwise dropped when a new
|
||||
parameter is registered, but kept when ``prefer_copy`` reuses it in place.
|
||||
|
||||
This function should not be called on weights which are tied/shared
|
||||
|
||||
Args:
|
||||
layer: Layer containing parameter to replace
|
||||
param_name: Name of parameter to replace
|
||||
new_data: New data of the new parameter, or None to set the parameter to None
|
||||
new_tensor: New data of the new parameter, or None to set the parameter
|
||||
to None
|
||||
prefer_copy: If True and the existing parameter is compatible with
|
||||
``new_data`` (same shape, dtype, and device), copy ``new_data``
|
||||
``new_tensor`` (same shape, dtype, and device), copy ``new_tensor``
|
||||
into the existing parameter in place rather than re-registering
|
||||
a new parameter. This preserves the parameter's storage address
|
||||
(``data_ptr``), which is required for captured CUDA graphs to
|
||||
@@ -69,27 +77,41 @@ def replace_parameter(
|
||||
"""
|
||||
# should not be used on a tied/shared param
|
||||
|
||||
# If new_data is None, set the parameter to None
|
||||
if new_data is None:
|
||||
# If new_tensor is None, set the parameter to None
|
||||
if new_tensor is None:
|
||||
setattr(layer, param_name, None)
|
||||
return
|
||||
|
||||
if isinstance(new_data, torch.nn.Parameter):
|
||||
new_data = new_data.data
|
||||
|
||||
old_param: torch.nn.Parameter | None = getattr(layer, param_name, None)
|
||||
|
||||
new_tensor_attrs = dict(new_tensor.__dict__)
|
||||
|
||||
# `weight_loader` is the only attribute not ported over from new_tensor,
|
||||
# old_param.weight_loader only is supported. `_weight_loader` is
|
||||
# `BasevLLMParameter`'s backing field for its `weight_loader` property, so
|
||||
# a loader riding along under that name has to be dropped as well.
|
||||
new_tensor_attrs.pop("weight_loader", None)
|
||||
new_tensor_attrs.pop("_weight_loader", None)
|
||||
|
||||
if isinstance(new_tensor, torch.nn.Parameter):
|
||||
new_tensor = new_tensor.data
|
||||
|
||||
if (
|
||||
prefer_copy
|
||||
and old_param is not None
|
||||
and old_param.shape == new_data.shape
|
||||
and old_param.dtype == new_data.dtype
|
||||
and old_param.device == new_data.device
|
||||
and old_param.shape == new_tensor.shape
|
||||
and old_param.dtype == new_tensor.dtype
|
||||
and old_param.device == new_tensor.device
|
||||
):
|
||||
old_param.copy_(new_data)
|
||||
old_param.copy_(new_tensor)
|
||||
for attr_name, attr in new_tensor_attrs.items():
|
||||
setattr(old_param, attr_name, attr)
|
||||
return
|
||||
|
||||
new_param = torch.nn.Parameter(new_data, requires_grad=False)
|
||||
new_param = torch.nn.Parameter(new_tensor, requires_grad=False)
|
||||
|
||||
for attr_name, attr in new_tensor_attrs.items():
|
||||
setattr(new_param, attr_name, attr)
|
||||
|
||||
if old_param is not None and hasattr(old_param, "weight_loader"):
|
||||
weight_loader = old_param.weight_loader
|
||||
|
||||
Reference in New Issue
Block a user