From 2e777d21a82c232ee8bc315c13400cd9d9afd489 Mon Sep 17 00:00:00 2001 From: Yuankai Chen Date: Wed, 6 May 2026 03:32:26 -0700 Subject: [PATCH] [Bugfix][Rocm]Aiter MoE re-uses existing tensor addresses after weight update. (#40390) Signed-off-by: Yuankai Chen Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- .../fused_moe/unquantized_fused_moe_method.py | 52 +++++++++++++------ vllm/model_executor/utils.py | 25 ++++++++- 2 files changed, 59 insertions(+), 18 deletions(-) diff --git a/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py b/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py index 8969703340..91de16f79c 100644 --- a/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py +++ b/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py @@ -154,32 +154,52 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): w2: torch.Tensor, ) -> None: # Shuffle weights to runtime format. - w13, w2 = convert_to_unquantized_kernel_format( + w13_new, w2_new = convert_to_unquantized_kernel_format( self.unquantized_backend, layer=layer, w13_weight=w13, w2_weight=w2, ) - replace_parameter(layer, "w13_weight", w13) - replace_parameter(layer, "w2_weight", w2) + # `moe_kernel` is initialized to None in FusedMoEMethodBase.__init__; + # On the first call we replace the parameter normally. On subsequent + # calls (e.g. RL weight updates that re-trigger + # process_weights_after_loading) the moe kernel has already been set + # up and CUDA graphs may have captured the parameter addresses, so + # we copy the shuffled data into the existing storage instead of + # re-registering a new Parameter. + is_weight_update = self.moe_kernel is not None # type: ignore[has-type] + replace_parameter(layer, "w13_weight", w13_new, prefer_copy=is_weight_update) + replace_parameter(layer, "w2_weight", w2_new, prefer_copy=is_weight_update) - # Setup moe kernel. - self.moe_quant_config = self.get_fused_moe_quant_config(layer) - assert self.moe_quant_config is not None - assert self.experts_cls is not None - self.moe_kernel = make_unquantized_moe_kernel( - quant_config=self.moe_quant_config, - moe_config=self.moe, - backend=self.unquantized_backend, - experts_cls=self.experts_cls, - routing_tables=layer._maybe_init_expert_routing_tables(), - shared_experts=layer.shared_experts, - ) + if not is_weight_update: + # Setup moe kernel only on the first call. For the unquantized + # method, moe_quant_config is either the constant + # FUSED_MOE_UNQUANTIZED_CONFIG or biased_moe_quant_config(...) + # which references layer.w{13,2}_bias; since weight updates + # mutate those bias tensors in place, the kernel does not need + # to be re-built. + self.moe_quant_config = self.get_fused_moe_quant_config(layer) + assert self.moe_quant_config is not None + assert self.experts_cls is not None + self.moe_kernel = make_unquantized_moe_kernel( + quant_config=self.moe_quant_config, + moe_config=self.moe, + backend=self.unquantized_backend, + experts_cls=self.experts_cls, + routing_tables=layer._maybe_init_expert_routing_tables(), + shared_experts=layer.shared_experts, + ) def process_weights_after_loading(self, layer: torch.nn.Module) -> None: super().process_weights_after_loading(layer) - # Padding the weight for better performance on ROCm + # Padding the weight for better performance on ROCm. + # _maybe_pad_weight is idempotent: on the first call it allocates a + # padded storage and returns a strided view; on subsequent calls + # (weight updates) the stride condition no longer matches so it + # returns the input unchanged. The reassignment to .data is therefore + # a no-op on updates and preserves the storage address (data_ptr) + # used by captured CUDA graphs. layer.w13_weight.data = self._maybe_pad_weight(layer.w13_weight.data) layer.w2_weight.data = self._maybe_pad_weight(layer.w2_weight.data) diff --git a/vllm/model_executor/utils.py b/vllm/model_executor/utils.py index 0b844d1493..a0269be855 100644 --- a/vllm/model_executor/utils.py +++ b/vllm/model_executor/utils.py @@ -45,7 +45,10 @@ def set_weight_attrs( def replace_parameter( - layer: torch.nn.Module, param_name: str, new_data: torch.Tensor | None + layer: torch.nn.Module, + param_name: str, + new_data: torch.Tensor | None, + prefer_copy: bool = False, ): """ Replace a parameter of a layer while maintaining the ability to reload the weight. @@ -57,6 +60,12 @@ def replace_parameter( layer: Layer containing parameter to replace param_name: Name of parameter to replace new_data: New data of the new parameter, or None to set the parameter to None + prefer_copy: If True and the existing parameter is compatible with + ``new_data`` (same shape, dtype, and device), copy ``new_data`` + into the existing parameter in place rather than re-registering + a new parameter. This preserves the parameter's storage address + (``data_ptr``), which is required for captured CUDA graphs to + remain valid across weight updates (e.g. in RL training loops). """ # should not be used on a tied/shared param @@ -67,9 +76,21 @@ def replace_parameter( if isinstance(new_data, torch.nn.Parameter): new_data = new_data.data - new_param = torch.nn.Parameter(new_data, requires_grad=False) old_param: torch.nn.Parameter | None = getattr(layer, param_name, None) + + if ( + prefer_copy + and old_param is not None + and old_param.shape == new_data.shape + and old_param.dtype == new_data.dtype + and old_param.device == new_data.device + ): + old_param.copy_(new_data) + return + + new_param = torch.nn.Parameter(new_data, requires_grad=False) + if old_param is not None and hasattr(old_param, "weight_loader"): weight_loader = old_param.weight_loader set_weight_attrs(new_param, {"weight_loader": weight_loader})