diff --git a/tests/compile/fusions_e2e/test_tp2_ar_rms.py b/tests/compile/fusions_e2e/test_tp2_ar_rms.py index b18c41658f..a969e91daa 100644 --- a/tests/compile/fusions_e2e/test_tp2_ar_rms.py +++ b/tests/compile/fusions_e2e/test_tp2_ar_rms.py @@ -181,8 +181,13 @@ def test_tp2_ar_rms_fp4_fusions( @multi_gpu_test(num_gpus=2) @pytest.mark.parametrize( - "model_name, matches_fn, model_kwargs, hf_overrides", - [llama3_8b, qwen3_a3b, gpt_oss_20b], + "model_name, matches_fn, model_kwargs, hf_overrides, model_impl", + [ + (*llama3_8b, "auto"), + (*llama3_8b, "transformers"), + (*qwen3_a3b, "auto"), + (*gpt_oss_20b, "auto"), + ], ) @pytest.mark.parametrize( "attn_backend", @@ -202,17 +207,26 @@ def test_tp2_ar_rms_fusions( matches_fn: Callable[[int], Matches], model_kwargs: dict, hf_overrides: Callable[[int], dict], + model_impl: str, attn_backend: AttentionBackendCase, n_layers: int, custom_ops: str, inductor_graph_partition: bool, run_e2e_fusion_test, ): + if model_impl == "transformers" and not current_platform.is_rocm(): + pytest.skip("Transformers 3D AR+RMS regression is ROCm-only") + matches = matches_fn(n_layers) + if model_impl == "transformers": + # TODO(BadrBasowid): Match the vLLM backend's fusion count once the + # separate residual add and RMSNorm operations are fused. + matches = matches._replace(aiter_ar_rms_fusion=1) # Reduce size of model and skip weight loading time model_kwargs["hf_overrides"] = hf_overrides(n_layers) model_kwargs["load_format"] = "dummy" + model_kwargs["model_impl"] = model_impl model_kwargs["max_model_len"] = 1024 model_kwargs["kernel_config"] = {"enable_flashinfer_autotune": False} model_kwargs["disable_custom_all_reduce"] = False diff --git a/vllm/_aiter_ops.py b/vllm/_aiter_ops.py index 4a915c0dd6..e4b08e90ac 100644 --- a/vllm/_aiter_ops.py +++ b/vllm/_aiter_ops.py @@ -764,7 +764,7 @@ def _rocm_aiter_fused_allreduce_rmsnorm_impl( total_bytes = input_.numel() * input_.element_size() hidden_dim = input_.shape[-1] - token_num = input_.shape[0] + token_num = input_.numel() // hidden_dim if input_.dtype in (torch.bfloat16, torch.float16): pack_size = 16 // input_.element_size() hidden_ok = hidden_dim % pack_size == 0 and hidden_dim // pack_size <= 1024 @@ -824,7 +824,7 @@ def _rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_impl( total_bytes = input_.numel() * input_.element_size() hidden_dim = input_.shape[-1] - token_num = input_.shape[0] + token_num = input_.numel() // hidden_dim if input_.dtype in (torch.bfloat16, torch.float16): pack_size = 16 // input_.element_size() hidden_ok = hidden_dim % pack_size == 0 and hidden_dim // pack_size <= 1024