[ROCM] Fix AITER Fused AllReduce RMSNorm for Transformers Backend (#49673)
Signed-off-by: BadrBasowid <badr.basowid@gmail.com>
This commit is contained in:
@@ -181,8 +181,13 @@ def test_tp2_ar_rms_fp4_fusions(
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize(
|
||||
"model_name, matches_fn, model_kwargs, hf_overrides",
|
||||
[llama3_8b, qwen3_a3b, gpt_oss_20b],
|
||||
"model_name, matches_fn, model_kwargs, hf_overrides, model_impl",
|
||||
[
|
||||
(*llama3_8b, "auto"),
|
||||
(*llama3_8b, "transformers"),
|
||||
(*qwen3_a3b, "auto"),
|
||||
(*gpt_oss_20b, "auto"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"attn_backend",
|
||||
@@ -202,17 +207,26 @@ def test_tp2_ar_rms_fusions(
|
||||
matches_fn: Callable[[int], Matches],
|
||||
model_kwargs: dict,
|
||||
hf_overrides: Callable[[int], dict],
|
||||
model_impl: str,
|
||||
attn_backend: AttentionBackendCase,
|
||||
n_layers: int,
|
||||
custom_ops: str,
|
||||
inductor_graph_partition: bool,
|
||||
run_e2e_fusion_test,
|
||||
):
|
||||
if model_impl == "transformers" and not current_platform.is_rocm():
|
||||
pytest.skip("Transformers 3D AR+RMS regression is ROCm-only")
|
||||
|
||||
matches = matches_fn(n_layers)
|
||||
if model_impl == "transformers":
|
||||
# TODO(BadrBasowid): Match the vLLM backend's fusion count once the
|
||||
# separate residual add and RMSNorm operations are fused.
|
||||
matches = matches._replace(aiter_ar_rms_fusion=1)
|
||||
|
||||
# Reduce size of model and skip weight loading time
|
||||
model_kwargs["hf_overrides"] = hf_overrides(n_layers)
|
||||
model_kwargs["load_format"] = "dummy"
|
||||
model_kwargs["model_impl"] = model_impl
|
||||
model_kwargs["max_model_len"] = 1024
|
||||
model_kwargs["kernel_config"] = {"enable_flashinfer_autotune": False}
|
||||
model_kwargs["disable_custom_all_reduce"] = False
|
||||
|
||||
+2
-2
@@ -764,7 +764,7 @@ def _rocm_aiter_fused_allreduce_rmsnorm_impl(
|
||||
|
||||
total_bytes = input_.numel() * input_.element_size()
|
||||
hidden_dim = input_.shape[-1]
|
||||
token_num = input_.shape[0]
|
||||
token_num = input_.numel() // hidden_dim
|
||||
if input_.dtype in (torch.bfloat16, torch.float16):
|
||||
pack_size = 16 // input_.element_size()
|
||||
hidden_ok = hidden_dim % pack_size == 0 and hidden_dim // pack_size <= 1024
|
||||
@@ -824,7 +824,7 @@ def _rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_impl(
|
||||
|
||||
total_bytes = input_.numel() * input_.element_size()
|
||||
hidden_dim = input_.shape[-1]
|
||||
token_num = input_.shape[0]
|
||||
token_num = input_.numel() // hidden_dim
|
||||
if input_.dtype in (torch.bfloat16, torch.float16):
|
||||
pack_size = 16 // input_.element_size()
|
||||
hidden_ok = hidden_dim % pack_size == 0 and hidden_dim // pack_size <= 1024
|
||||
|
||||
Reference in New Issue
Block a user