[Bugfix][Rocm]Aiter MoE re-uses existing tensor addresses after weight update. (#40390)

Signed-off-by: Yuankai Chen <yuankach@amd.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
Yuankai Chen
2026-05-06 10:32:26 +00:00
committed by GitHub
co-authored by gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
parent e43a791284
commit 2e777d21a8
2 changed files with 59 additions and 18 deletions
@@ -154,32 +154,52 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
w2: torch.Tensor,
) -> None:
# Shuffle weights to runtime format.
w13, w2 = convert_to_unquantized_kernel_format(
w13_new, w2_new = convert_to_unquantized_kernel_format(
self.unquantized_backend,
layer=layer,
w13_weight=w13,
w2_weight=w2,
)
replace_parameter(layer, "w13_weight", w13)
replace_parameter(layer, "w2_weight", w2)
# `moe_kernel` is initialized to None in FusedMoEMethodBase.__init__;
# On the first call we replace the parameter normally. On subsequent
# calls (e.g. RL weight updates that re-trigger
# process_weights_after_loading) the moe kernel has already been set
# up and CUDA graphs may have captured the parameter addresses, so
# we copy the shuffled data into the existing storage instead of
# re-registering a new Parameter.
is_weight_update = self.moe_kernel is not None # type: ignore[has-type]
replace_parameter(layer, "w13_weight", w13_new, prefer_copy=is_weight_update)
replace_parameter(layer, "w2_weight", w2_new, prefer_copy=is_weight_update)
# Setup moe kernel.
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
assert self.moe_quant_config is not None
assert self.experts_cls is not None
self.moe_kernel = make_unquantized_moe_kernel(
quant_config=self.moe_quant_config,
moe_config=self.moe,
backend=self.unquantized_backend,
experts_cls=self.experts_cls,
routing_tables=layer._maybe_init_expert_routing_tables(),
shared_experts=layer.shared_experts,
)
if not is_weight_update:
# Setup moe kernel only on the first call. For the unquantized
# method, moe_quant_config is either the constant
# FUSED_MOE_UNQUANTIZED_CONFIG or biased_moe_quant_config(...)
# which references layer.w{13,2}_bias; since weight updates
# mutate those bias tensors in place, the kernel does not need
# to be re-built.
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
assert self.moe_quant_config is not None
assert self.experts_cls is not None
self.moe_kernel = make_unquantized_moe_kernel(
quant_config=self.moe_quant_config,
moe_config=self.moe,
backend=self.unquantized_backend,
experts_cls=self.experts_cls,
routing_tables=layer._maybe_init_expert_routing_tables(),
shared_experts=layer.shared_experts,
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
super().process_weights_after_loading(layer)
# Padding the weight for better performance on ROCm
# Padding the weight for better performance on ROCm.
# _maybe_pad_weight is idempotent: on the first call it allocates a
# padded storage and returns a strided view; on subsequent calls
# (weight updates) the stride condition no longer matches so it
# returns the input unchanged. The reassignment to .data is therefore
# a no-op on updates and preserves the storage address (data_ptr)
# used by captured CUDA graphs.
layer.w13_weight.data = self._maybe_pad_weight(layer.w13_weight.data)
layer.w2_weight.data = self._maybe_pad_weight(layer.w2_weight.data)
+23 -2
View File
@@ -45,7 +45,10 @@ def set_weight_attrs(
def replace_parameter(
layer: torch.nn.Module, param_name: str, new_data: torch.Tensor | None
layer: torch.nn.Module,
param_name: str,
new_data: torch.Tensor | None,
prefer_copy: bool = False,
):
"""
Replace a parameter of a layer while maintaining the ability to reload the weight.
@@ -57,6 +60,12 @@ def replace_parameter(
layer: Layer containing parameter to replace
param_name: Name of parameter to replace
new_data: New data of the new parameter, or None to set the parameter to None
prefer_copy: If True and the existing parameter is compatible with
``new_data`` (same shape, dtype, and device), copy ``new_data``
into the existing parameter in place rather than re-registering
a new parameter. This preserves the parameter's storage address
(``data_ptr``), which is required for captured CUDA graphs to
remain valid across weight updates (e.g. in RL training loops).
"""
# should not be used on a tied/shared param
@@ -67,9 +76,21 @@ def replace_parameter(
if isinstance(new_data, torch.nn.Parameter):
new_data = new_data.data
new_param = torch.nn.Parameter(new_data, requires_grad=False)
old_param: torch.nn.Parameter | None = getattr(layer, param_name, None)
if (
prefer_copy
and old_param is not None
and old_param.shape == new_data.shape
and old_param.dtype == new_data.dtype
and old_param.device == new_data.device
):
old_param.copy_(new_data)
return
new_param = torch.nn.Parameter(new_data, requires_grad=False)
if old_param is not None and hasattr(old_param, "weight_loader"):
weight_loader = old_param.weight_loader
set_weight_attrs(new_param, {"weight_loader": weight_loader})