[Bugfix][MoE] Preserve unquantized weight storage on ROCm (#46009)

Signed-off-by: bingzhaodong <aaab8b@gmail.com>
Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
Co-authored-by: Zhaodong Bing <zhaobing@smc300x-ccs-aus-gpuf2c5.prov.aus.ccs.cpe.ice.amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Zhaodong Bing <zhaobing@smc300x-ccs-aus-gpuf2a0.prov.aus.ccs.cpe.ice.amd.com>
This commit is contained in:
Zhaodong Bing
2026-09-03 11:06:09 +08:00
committed by GitHub
co-authored by TJian Zhaodong Bing Cursor mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Zhaodong Bing
parent e47356c63e
commit 5d09eb2cf3
4 changed files with 315 additions and 9 deletions
@@ -336,6 +336,7 @@ def convert_to_unquantized_kernel_format(
w13_weight, w2_weight = rocm_aiter_ops.shuffle_weights(w13_weight, w2_weight)
w13_weight.is_shuffled = True
w2_weight.is_shuffled = True
return w13_weight, w2_weight
elif unquantized_backend == UnquantizedMoeBackend.FLASHINFER_CUTLASS:
if moe_config.is_act_and_mul:
@@ -175,15 +175,11 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
def process_weights_after_loading(self, layer: "RoutedExperts") -> None:
super().process_weights_after_loading(layer)
# Padding the weight for better performance on ROCm.
# _maybe_pad_weight is idempotent: on the first call it allocates a
# padded storage and returns a strided view; on subsequent calls
# (weight updates) the stride condition no longer matches so it
# returns the input unchanged. The reassignment to .data is therefore
# a no-op on updates and preserves the storage address (data_ptr)
# used by captured CUDA graphs.
layer.w13_weight.data = self._maybe_pad_weight(layer.w13_weight.data)
layer.w2_weight.data = self._maybe_pad_weight(layer.w2_weight.data)
# Padding may allocate a same-shaped strided view. Copy it back instead
# of rebinding `.data`, since CUDA graphs capture the parameter storage
# address.
layer.w13_weight.data.copy_(self._maybe_pad_weight(layer.w13_weight.data))
layer.w2_weight.data.copy_(self._maybe_pad_weight(layer.w2_weight.data))
if self.unquantized_backend in [
UnquantizedMoeBackend.TPU,