[ROCM] Fix AITER Fused AllReduce RMSNorm for Transformers Backend (#49673)

Signed-off-by: BadrBasowid <badr.basowid@gmail.com>
This commit is contained in:
BadrBasowid
2026-07-24 07:16:17 -07:00
committed by GitHub
parent a454a1dd25
commit 8eac21a602
2 changed files with 18 additions and 4 deletions
+16 -2
View File
@@ -181,8 +181,13 @@ def test_tp2_ar_rms_fp4_fusions(
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize(
"model_name, matches_fn, model_kwargs, hf_overrides",
[llama3_8b, qwen3_a3b, gpt_oss_20b],
"model_name, matches_fn, model_kwargs, hf_overrides, model_impl",
[
(*llama3_8b, "auto"),
(*llama3_8b, "transformers"),
(*qwen3_a3b, "auto"),
(*gpt_oss_20b, "auto"),
],
)
@pytest.mark.parametrize(
"attn_backend",
@@ -202,17 +207,26 @@ def test_tp2_ar_rms_fusions(
matches_fn: Callable[[int], Matches],
model_kwargs: dict,
hf_overrides: Callable[[int], dict],
model_impl: str,
attn_backend: AttentionBackendCase,
n_layers: int,
custom_ops: str,
inductor_graph_partition: bool,
run_e2e_fusion_test,
):
if model_impl == "transformers" and not current_platform.is_rocm():
pytest.skip("Transformers 3D AR+RMS regression is ROCm-only")
matches = matches_fn(n_layers)
if model_impl == "transformers":
# TODO(BadrBasowid): Match the vLLM backend's fusion count once the
# separate residual add and RMSNorm operations are fused.
matches = matches._replace(aiter_ar_rms_fusion=1)
# Reduce size of model and skip weight loading time
model_kwargs["hf_overrides"] = hf_overrides(n_layers)
model_kwargs["load_format"] = "dummy"
model_kwargs["model_impl"] = model_impl
model_kwargs["max_model_len"] = 1024
model_kwargs["kernel_config"] = {"enable_flashinfer_autotune": False}
model_kwargs["disable_custom_all_reduce"] = False
+2 -2
View File
@@ -764,7 +764,7 @@ def _rocm_aiter_fused_allreduce_rmsnorm_impl(
total_bytes = input_.numel() * input_.element_size()
hidden_dim = input_.shape[-1]
token_num = input_.shape[0]
token_num = input_.numel() // hidden_dim
if input_.dtype in (torch.bfloat16, torch.float16):
pack_size = 16 // input_.element_size()
hidden_ok = hidden_dim % pack_size == 0 and hidden_dim // pack_size <= 1024
@@ -824,7 +824,7 @@ def _rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_impl(
total_bytes = input_.numel() * input_.element_size()
hidden_dim = input_.shape[-1]
token_num = input_.shape[0]
token_num = input_.numel() // hidden_dim
if input_.dtype in (torch.bfloat16, torch.float16):
pack_size = 16 // input_.element_size()
hidden_ok = hidden_dim % pack_size == 0 and hidden_dim // pack_size <= 1024