diff --git a/tests/compile/passes/distributed/test_fusion_all_reduce.py b/tests/compile/passes/distributed/test_fusion_all_reduce.py index 4805863057..83ce458aaf 100644 --- a/tests/compile/passes/distributed/test_fusion_all_reduce.py +++ b/tests/compile/passes/distributed/test_fusion_all_reduce.py @@ -189,6 +189,118 @@ class TestAllReduceRMSNormStaticQuantFP8Model(torch.nn.Module): ] +class TestAiterAllReduceRMSNormGroupQuantFP8Model(torch.nn.Module): + """Exercises the new ROCm AITER AR+RMS+per-group-FP8-quant patterns. + + Four ``rms_norm`` sites that together hit every pattern registered by + ``RocmAiterAllReduceFusionPass`` for the per-group FP8 quant path: + + * ``norm[0]``: ``all_reduce -> rms_norm -> group_fp8_quant`` (no residual) + -> ``AiterAllreduceFusedRMSNormGroupQuantFP8Pattern`` + * ``norm[1]``: ``all_reduce -> fused_add_rms_norm -> group_fp8_quant`` + (single ``rms`` consumer) + -> ``AiterAllreduceFusedAddRMSNormGroupQuantFP8Pattern`` + * ``norm[2..3]``: ``all_reduce -> fused_add_rms_norm + -> (group_fp8_quant + rocm_unquantized_gemm)`` (two ``rms`` consumers, + modeling the DSv3.2 indexer fan-out) + -> ``AiterAllreduceFusedAddRMSNormGroupQuantWithIndexerPattern`` + + The chain feeds the next AllReduce by dequantizing the FP8 output (FP8 + cast back to bf16 multiplied by the per-group scale), which is enough to + keep the matmul chain bf16 without depending on a real FP8 block-scaled + GEMM kernel. + """ + + quant_group_size = 128 + indexer_out_dim = 8 + + def __init__( + self, + hidden_size=128, + token_num=16, + eps=1e-6, + dtype: torch.dtype = torch.bfloat16, + use_triton_quant: bool = False, + ): + super().__init__() + self.hidden_size = hidden_size + self.eps = eps + self.use_triton_quant = use_triton_quant + assert hidden_size % self.quant_group_size == 0, ( + f"hidden_size ({hidden_size}) must be a multiple of " + f"quant_group_size ({self.quant_group_size}) for per-group FP8 quant" + ) + self.norm = [RMSNorm(hidden_size, eps) for _ in range(4)] + self.w = [torch.rand(hidden_size, hidden_size, dtype=dtype) for _ in range(3)] + self.indexer_w = [ + torch.rand(self.indexer_out_dim, hidden_size, dtype=dtype) for _ in range(2) + ] + + def _group_quant(self, rms: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + if self.use_triton_quant: + return torch.ops.vllm.triton_per_token_group_quant_fp8( + rms, self.quant_group_size + ) + return torch.ops.vllm.rocm_aiter_group_fp8_quant.default( + rms, self.quant_group_size + ) + + def _dequantize_to_bf16( + self, q: torch.Tensor, s: torch.Tensor, ref: torch.Tensor + ) -> torch.Tensor: + # Broadcast the per-group scale across each group of `quant_group_size` + # so we can chain the FP8 output back into a bf16 matmul. This avoids + # depending on a real FP8 block-scaled GEMM kernel in the test. + s_full = s.repeat_interleave(self.quant_group_size, dim=-1).to(ref.dtype) + return q.to(ref.dtype) * s_full + + def forward(self, hidden_states): + z = torch.relu(hidden_states) + x = resid = tensor_model_parallel_all_reduce(z) + rms = self.norm[0](x) + q0, s0 = self._group_quant(rms) + y = self._dequantize_to_bf16(q0, s0, rms) + + z2 = torch.mm(y, self.w[0]) + x2 = tensor_model_parallel_all_reduce(z2) + rms2, resid = self.norm[1](x2, resid) + q1, s1 = self._group_quant(rms2) + y2 = self._dequantize_to_bf16(q1, s1, rms2) + + z3 = torch.mm(y2, self.w[1]) + x3 = tensor_model_parallel_all_reduce(z3) + rms3, resid = self.norm[2](x3, resid) + q2, s2 = self._group_quant(rms3) + # Second consumer of ``rms3``: forces the with-indexer pattern. + idx2 = torch.ops.vllm.rocm_unquantized_gemm(rms3, self.indexer_w[0], None) + y3 = self._dequantize_to_bf16(q2, s2, rms3) + + z4 = torch.mm(y3, self.w[2]) + x4 = tensor_model_parallel_all_reduce(z4) + rms4, resid = self.norm[3](x4, resid) + q3, s3 = self._group_quant(rms4) + # Second consumer of ``rms4``: forces the with-indexer pattern. + idx3 = torch.ops.vllm.rocm_unquantized_gemm(rms4, self.indexer_w[1], None) + y4 = self._dequantize_to_bf16(q3, s3, rms4) + return y4, idx2, idx3 + + def ops_in_model_before(self): + return [ + torch.ops.vllm.all_reduce.default, + ( + torch.ops.vllm.triton_per_token_group_quant_fp8.default + if self.use_triton_quant + else torch.ops.vllm.rocm_aiter_group_fp8_quant.default + ), + ] + + def ops_in_model_after(self): + return [ + rocm_aiter_ops.get_fused_allreduce_rmsnorm_quant_per_group_op(), + rocm_aiter_ops.get_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_op(), # noqa: E501 + ] + + class TestAllReduceFusedAddRMSNormStaticQuantFP4Model(torch.nn.Module): def __init__( self, hidden_size=16, token_num=16, eps=1e-6, dtype: torch.dtype = torch.float16 @@ -463,3 +575,167 @@ def all_reduce_fusion_pass_on_test_model( assert fused_nodes assert all(n.kwargs.get("weight_bias") == 1.0 for n in fused_nodes) del all_reduce_fusion_pass + + +@multi_gpu_test(num_gpus=2) +@pytest.mark.parametrize("use_triton_quant", [True, False]) +@pytest.mark.parametrize("batch_size", [8]) +@pytest.mark.parametrize("seq_len", [8]) +@pytest.mark.parametrize("hidden_size", [128]) +@pytest.mark.parametrize("dtype", [torch.bfloat16]) +@pytest.mark.parametrize("enable_rms_norm_custom_op", [True, False]) +@pytest.mark.skipif( + not current_platform.is_rocm(), + reason="ROCm AITER AR+RMS+per-group-FP8-quant fusion is ROCm-only", +) +@pytest.mark.skipif(not IS_AITER_FOUND, reason="aiter is not found") +def test_rocm_aiter_all_reduce_rmsnorm_group_quant_fp8_fusion_pass_replace( + batch_size: int, + seq_len: int, + hidden_size: int, + dtype: torch.dtype, + enable_rms_norm_custom_op: bool, + use_triton_quant: bool, + monkeypatch: pytest.MonkeyPatch, +): + """Sibling of ``test_all_reduce_fusion_pass_replace`` for the new + ROCm AITER AR+RMS+per-group-FP8-quant fusion patterns. + + Validates the three new ``VllmPatternReplacement`` patterns added to + ``RocmAiterAllReduceFusionPass``: + + * ``AiterAllreduceFusedRMSNormGroupQuantFP8Pattern`` (no-residual) + * ``AiterAllreduceFusedAddRMSNormGroupQuantFP8Pattern`` (with-residual, + single ``rms`` consumer) + * ``AiterAllreduceFusedAddRMSNormGroupQuantWithIndexerPattern`` (with- + residual, DSv3.2 indexer fan-out; parametrized over both + ``triton_per_token_group_quant_fp8`` and ``rocm_aiter_group_fp8_quant`` + producers). + """ + with monkeypatch.context() as m: + m.setenv("VLLM_ROCM_USE_AITER", "1") + rocm_aiter_ops.refresh_env_variables() + + if not rocm_aiter_ops.has_fused_allreduce_rmsnorm_quant_per_group(): + pytest.skip( + "aiter build is missing 'fused_ar_rms_per_group_quant' (needs " + "ROCm/aiter PR #2823); the new patterns aren't registered." + ) + + num_processes = 2 + + def run_torch_spawn(fn, nprocs): + torch.multiprocessing.spawn( + fn, + args=( + num_processes, + TestAiterAllReduceRMSNormGroupQuantFP8Model, + batch_size, + seq_len, + hidden_size, + dtype, + enable_rms_norm_custom_op, + use_triton_quant, + monkeypatch, + ), + nprocs=nprocs, + ) + + run_torch_spawn(rocm_aiter_group_quant_fusion_pass_on_test_model, num_processes) + + +def rocm_aiter_group_quant_fusion_pass_on_test_model( + local_rank: int, + world_size: int, + test_model_cls: torch.nn.Module, + batch_size: int, + seq_len: int, + hidden_size: int, + dtype: torch.dtype, + enable_rms_norm_custom_op: bool, + use_triton_quant: bool, + monkeypatch: pytest.MonkeyPatch, +): + set_random_seed(0) + + device = torch.device(f"{DEVICE_TYPE}:{local_rank}") + torch.accelerator.set_device_index(device) + torch.set_default_device(device) + torch.set_default_dtype(dtype) + + update_environment_variables( + { + "RANK": str(local_rank), + "LOCAL_RANK": str(local_rank), + "WORLD_SIZE": str(world_size), + "MASTER_ADDR": "localhost", + "MASTER_PORT": "12345", + "VLLM_ROCM_USE_AITER": "1", + } + ) + rocm_aiter_ops.refresh_env_variables() + + init_distributed_environment() + + custom_ops = [] + if enable_rms_norm_custom_op: + custom_ops.append("+rms_norm") + # ``triton_per_token_group_quant_fp8`` is emitted by ``QuantFP8.forward_hip`` + # only when QuantFP8 is enabled as a custom op (and ``use_triton=True`` at + # the call site). The patterns in this PR are robust to both Triton and + # rocm_aiter forms; we always enable +quant_fp8 so the matcher's example + # trace finds the same form the test model uses. + custom_ops.append("+quant_fp8") + + vllm_config = VllmConfig( + compilation_config=CompilationConfig( + mode=CompilationMode.VLLM_COMPILE, custom_ops=custom_ops + ) + ) + vllm_config.compilation_config.pass_config = PassConfig( + fuse_allreduce_rms=True, eliminate_noops=True + ) + vllm_config.device_config = DeviceConfig(device=torch.device(DEVICE_TYPE)) + vllm_config.parallel_config.rank = local_rank + + model_name = "RedHatAI/Llama-3.2-1B-Instruct-FP8" + vllm_config.model_config = ModelConfig( + model=model_name, trust_remote_code=True, dtype=dtype, seed=42 + ) + with set_current_vllm_config(vllm_config): + initialize_model_parallel(tensor_model_parallel_size=world_size) + all_reduce_fusion_pass = RocmAiterAllReduceFusionPass(vllm_config) + noop_pass = NoOpEliminationPass(vllm_config) + func_pass = FixFunctionalizationPass(vllm_config) + cleanup_pass = PostCleanupPass(vllm_config) + + backend = TestBackend( + noop_pass, all_reduce_fusion_pass, func_pass, cleanup_pass + ) + + token_num = batch_size * seq_len + model = test_model_cls( + hidden_size, token_num, dtype=dtype, use_triton_quant=use_triton_quant + ) + + hidden_states = torch.randn((token_num, hidden_size), requires_grad=False) + + compiled_model = torch.compile(model, backend=backend) + compiled_model(hidden_states) + + results_unfused = model(hidden_states) + results_fused = compiled_model(hidden_states) + # The fused per-group AR+RMS+QUANT op is bit-equivalent to the unfused + # chain modulo the small AllReduce + RMSNorm reordering inside aiter. + # Per-group FP8 quant introduces step noise <=1 per group; use the + # same tolerance as the sibling FP8 static test. + torch.testing.assert_close(results_unfused, results_fused, atol=1e-2, rtol=1e-2) + + # Four pattern firings: norm[0] (no-add quant), norm[1] (add quant, + # single ``rms`` consumer), norm[2..3] (add quant + indexer fan-out). + assert all_reduce_fusion_pass.matched_count == 4, ( + f"{all_reduce_fusion_pass.matched_count=}" + ) + backend.check_before_ops(model.ops_in_model_before(), fully_replaced=False) + backend.check_after_ops(model.ops_in_model_after()) + del all_reduce_fusion_pass diff --git a/vllm/_aiter_ops.py b/vllm/_aiter_ops.py index eb12bedd7b..1d75b7c762 100644 --- a/vllm/_aiter_ops.py +++ b/vllm/_aiter_ops.py @@ -70,6 +70,21 @@ class AiterCustomAllreduceProto(Protocol): registered: bool = False, use_1stage: bool = False, ) -> tuple[torch.Tensor, torch.Tensor]: ... + def fused_ar_rms_per_group_quant( + self, + inp: torch.Tensor, + res_inp: torch.Tensor, + *, + w: torch.Tensor, + eps: float, + group_size: int = 128, + registered: bool = False, + use_1stage: bool = False, + emit_bf16: bool = False, + ) -> ( + tuple[torch.Tensor, torch.Tensor, torch.Tensor] + | tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor] + ): ... def should_custom_ar(self, inp: torch.Tensor) -> bool: ... @@ -850,6 +865,152 @@ def _rocm_aiter_fused_allreduce_rmsnorm_fake( return torch.empty_like(input_), torch.empty_like(residual) +def _rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_impl( + input_: torch.Tensor, + residual: torch.Tensor, + weight: torch.Tensor, + epsilon: float, + group_size: int, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Fused AllReduce + RMSNorm + per-group FP8 quant. + + Mirrors the eligibility logic of ``_rocm_aiter_fused_allreduce_rmsnorm_impl`` + for the 1-stage vs 2-stage AITER kernel dispatch (both variants run inside + AITER, the only choice we make here is the launcher to call into). + """ + aiter_ar = rocm_aiter_ops.get_aiter_allreduce() + assert aiter_ar is not None, "aiter allreduce must be initialized" + + total_bytes = input_.numel() * input_.element_size() + hidden_dim = input_.shape[-1] + token_num = input_.shape[0] + if input_.dtype in (torch.bfloat16, torch.float16): + pack_size = 16 // input_.element_size() + hidden_ok = hidden_dim % pack_size == 0 and hidden_dim // pack_size <= 1024 + else: + hidden_ok = False + token_ok = token_num <= 80 + world_size = aiter_ar.world_size + full_nvlink = aiter_ar.fully_connected + + if world_size == 2: + size_ok = True + elif full_nvlink and world_size <= 4: + size_ok = total_bytes < 256 * 1024 + elif full_nvlink and world_size <= 8: + size_ok = total_bytes < 128 * 1024 + else: + size_ok = False + + use_1stage = hidden_ok and token_ok and size_ok + + result = aiter_ar.fused_ar_rms_per_group_quant( + input_, + residual, + w=weight, + eps=epsilon, + group_size=group_size, + registered=torch.cuda.is_current_stream_capturing(), + use_1stage=use_1stage, + ) + assert result is not None + return result[0], result[1], result[2] + + +def _rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_fake( + input_: torch.Tensor, + residual: torch.Tensor, + weight: torch.Tensor, + epsilon: float, + group_size: int, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + hidden_dim = input_.shape[-1] + num_groups = hidden_dim // group_size + quant_out = torch.empty(input_.shape, dtype=FP8_DTYPE, device=input_.device) + residual_out = torch.empty_like(residual) + scale_out = torch.empty( + input_.shape[:-1] + (num_groups,), + dtype=torch.float32, + device=input_.device, + ) + return quant_out, residual_out, scale_out + + +def _rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_impl( + input_: torch.Tensor, + residual: torch.Tensor, + weight: torch.Tensor, + epsilon: float, + group_size: int, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Fused AllReduce + add-RMSNorm + per-group FP8 quant + bf16 normed act. + + Wraps the same AITER launcher as ``_rocm_aiter_fused_allreduce_rmsnorm_ + quant_per_group_impl`` with ``emit_bf16=True``, emitting the pre-quant + bf16/fp16 normed activation for a parallel consumer (DeepSeek V3.2 sparse + indexer ``wk_weights_proj``). + """ + aiter_ar = rocm_aiter_ops.get_aiter_allreduce() + assert aiter_ar is not None, "aiter allreduce must be initialized" + + total_bytes = input_.numel() * input_.element_size() + hidden_dim = input_.shape[-1] + token_num = input_.shape[0] + if input_.dtype in (torch.bfloat16, torch.float16): + pack_size = 16 // input_.element_size() + hidden_ok = hidden_dim % pack_size == 0 and hidden_dim // pack_size <= 1024 + else: + hidden_ok = False + token_ok = token_num <= 80 + world_size = aiter_ar.world_size + full_nvlink = aiter_ar.fully_connected + + if world_size == 2: + size_ok = True + elif full_nvlink and world_size <= 4: + size_ok = total_bytes < 256 * 1024 + elif full_nvlink and world_size <= 8: + size_ok = total_bytes < 128 * 1024 + else: + size_ok = False + + use_1stage = hidden_ok and token_ok and size_ok + + result = aiter_ar.fused_ar_rms_per_group_quant( + input_, + residual, + w=weight, + eps=epsilon, + group_size=group_size, + registered=torch.cuda.is_current_stream_capturing(), + use_1stage=use_1stage, + emit_bf16=True, + ) + assert result is not None + assert len(result) == 4, "emit_bf16=True must return four tensors from aiter" + return result[0], result[1], result[2], result[3] + + +def _rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_fake( + input_: torch.Tensor, + residual: torch.Tensor, + weight: torch.Tensor, + epsilon: float, + group_size: int, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + hidden_dim = input_.shape[-1] + num_groups = hidden_dim // group_size + quant_out = torch.empty(input_.shape, dtype=FP8_DTYPE, device=input_.device) + residual_out = torch.empty_like(residual) + scale_out = torch.empty( + input_.shape[:-1] + (num_groups,), + dtype=torch.float32, + device=input_.device, + ) + bf16_norm_out = torch.empty_like(input_) + return quant_out, residual_out, scale_out, bf16_norm_out + + def _rocm_aiter_per_tensor_quant_impl( x: torch.Tensor, quant_dtype: torch.dtype, @@ -1838,6 +1999,18 @@ class rocm_aiter_ops: fake_impl=_rocm_aiter_fused_allreduce_rmsnorm_fake, ) + direct_register_custom_op( + op_name="rocm_aiter_fused_allreduce_rmsnorm_quant_per_group", + op_func=(_rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_impl), + fake_impl=(_rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_fake), + ) + + direct_register_custom_op( + op_name="rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm", # noqa: E501 + op_func=_rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_impl, # noqa: E501 + fake_impl=_rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_fake, # noqa: E501 + ) + direct_register_custom_op( op_name="fused_mla_dual_rms_norm", op_func=_fused_mla_dual_rms_norm_impl, @@ -1892,6 +2065,29 @@ class rocm_aiter_ops: def get_fused_allreduce_rmsnorm_op() -> OpOverload: return torch.ops.vllm.rocm_aiter_fused_allreduce_rmsnorm.default + @staticmethod + def get_fused_allreduce_rmsnorm_quant_per_group_op() -> OpOverload: + return torch.ops.vllm.rocm_aiter_fused_allreduce_rmsnorm_quant_per_group.default + + @staticmethod + def get_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_op() -> OpOverload: # noqa: E501 + return torch.ops.vllm.rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm.default # noqa: E501 + + # TODO(frida-andersson): drop once vLLM pins AITER >= 0.1.14 (ROCm/aiter#2823). + @classmethod + def has_fused_allreduce_rmsnorm_quant_per_group(cls) -> bool: + """True if the running AITER build exposes the per-group AR+RMS+quant + kernel (added in ROCm/aiter PR #2823). + + The pattern registration in ``RocmAiterAllReduceFusionPass`` keys off + this so vLLM degrades to the AR+RMS-only fusion when run against an + older aiter that lacks the per-group launcher. + """ + aiter_ar = cls.get_aiter_allreduce() + return aiter_ar is not None and hasattr( + aiter_ar, "fused_ar_rms_per_group_quant" + ) + @staticmethod def get_fused_mla_dual_rms_norm_op() -> OpOverload: return torch.ops.vllm.fused_mla_dual_rms_norm.default diff --git a/vllm/compilation/passes/fusion/allreduce_rms_fusion.py b/vllm/compilation/passes/fusion/allreduce_rms_fusion.py index 324b0266b4..4de5c6cf7a 100644 --- a/vllm/compilation/passes/fusion/allreduce_rms_fusion.py +++ b/vllm/compilation/passes/fusion/allreduce_rms_fusion.py @@ -26,6 +26,9 @@ from vllm.distributed.parallel_state import ( ) from vllm.logger import init_logger from vllm.model_executor.layers.quantization.utils.quant_utils import ( + GroupShape, + QuantKey, + ScaleDesc, kFp8StaticTensorSym, ) from vllm.platforms import current_platform @@ -1109,7 +1112,7 @@ class AiterAllreduceFusedRMSNormPattern(BasePattern, VllmPatternReplacement): allreduce = self.FUSED_AR_RMSNORM_OP( input_=input, residual=residual, - weight=weight, + weight=weight.to(input.dtype), epsilon=self.epsilon, ) return allreduce[0], allreduce[1] @@ -1155,7 +1158,7 @@ class AiterAllreduceFusedAddRMSNormPattern(BasePattern, VllmPatternReplacement): allreduce = self.FUSED_AR_RMSNORM_OP( input_=input, residual=residual, - weight=weight, + weight=weight.to(input.dtype), epsilon=self.epsilon, ) return allreduce[0], allreduce[1] @@ -1163,6 +1166,279 @@ class AiterAllreduceFusedAddRMSNormPattern(BasePattern, VllmPatternReplacement): return _replacement +class AiterAllreduceFusedRMSNormGroupQuantFP8Pattern( + BasePattern, VllmPatternReplacement +): + """Fuse AllReduce + RMSNorm + per-group FP8 quant into a single AITER + custom op. + + Matches the AR-side analogue of ``AiterRMSFp8GroupQuantPattern`` in + ``rocm_aiter_fusion.py``: ``all_reduce -> rms_norm -> group_fp8_quant`` + fans out into ``rocm_aiter_fused_allreduce_rmsnorm_quant_per_group``. + + Without this pattern, ``RocmAiterAllReduceFusionPass`` would fuse the + ``all_reduce + rms_norm`` half (PR #41825 wires that), but the trailing + ``rocm_aiter_group_fp8_quant`` would still launch as a separate kernel. + That standalone quant accounts for ~535us / decode step on DSv3.2 MI355X + TP4 -- this pattern eliminates it by absorbing the quant into the AR + epilogue. Group size 128 matches the FP8 block-scaled MM kernel used by + DSv3.2's linear weights. + """ + + def __init__( + self, + epsilon: float, + dtype: torch.dtype, + device: str | None, + group_size: int = 128, + ) -> None: + super().__init__(dtype, device) + self.epsilon = epsilon + self.dtype = dtype + self.group_size = group_size + self.FUSED_AR_RMS_QUANT_OP = ( + rocm_aiter_ops.get_fused_allreduce_rmsnorm_quant_per_group_op() + ) + self.quant_dtype = current_platform.fp8_dtype() + self.quant_matcher = MatcherQuantFP8( + QuantKey( + dtype=self.quant_dtype, + scale=ScaleDesc(torch.float32, False, GroupShape(1, group_size)), + symmetric=True, + ), + match_rocm_aiter=True, + ) + + def get_inputs(self) -> list[torch.Tensor]: + # input, weight; hidden dim must be a group_size multiple so the + # group quant matcher's example trace is well-defined. + return [self.empty(5, self.group_size), self.empty(self.group_size)] + + @property + def pattern(self): + def _pattern( + input: torch.Tensor, weight: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor]: + allreduce_output = tensor_model_parallel_all_reduce(input) + rms = vllm.ir.ops.rms_norm(allreduce_output, weight, self.epsilon) + quant, scale = self.quant_matcher(rms) + return quant, scale + + return _pattern + + @property + def replacement(self): + def _replacement( + input: torch.Tensor, weight: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor]: + residual = torch.zeros_like(input) + result = self.FUSED_AR_RMS_QUANT_OP( + input_=input, + residual=residual, + weight=weight.to(input.dtype), + epsilon=self.epsilon, + group_size=self.group_size, + ) + # quant_out, scale_out (residual is unused on the no-add path, + # mirroring how AiterAllreduceFusedRMSNormPattern drops the + # residual output) + return result[0], result[2] + + return _replacement + + +class AiterAllreduceFusedAddRMSNormGroupQuantFP8Pattern( + BasePattern, VllmPatternReplacement +): + """``fused_add`` variant of ``AiterAllreduceFusedRMSNormGroupQuantFP8Pattern``. + + Targets the dominant DSv3.2-style post-attention / post-MLP path: + ``all_reduce -> fused_add_rms_norm -> group_fp8_quant``. Returns the + FP8 quant output, the residual carry-over, and the per-group scale. + """ + + def __init__( + self, + epsilon: float, + dtype: torch.dtype, + device: str | None, + group_size: int = 128, + ) -> None: + super().__init__(dtype, device) + self.epsilon = epsilon + self.dtype = dtype + self.group_size = group_size + self.FUSED_AR_RMS_QUANT_OP = ( + rocm_aiter_ops.get_fused_allreduce_rmsnorm_quant_per_group_op() + ) + self.quant_dtype = current_platform.fp8_dtype() + self.quant_matcher = MatcherQuantFP8( + QuantKey( + dtype=self.quant_dtype, + scale=ScaleDesc(torch.float32, False, GroupShape(1, group_size)), + symmetric=True, + ), + match_rocm_aiter=True, + ) + + def get_inputs(self) -> list[torch.Tensor]: + # residual, input, weight + return [ + self.empty(5, self.group_size), + self.empty(5, self.group_size), + self.empty(self.group_size), + ] + + @property + def pattern(self): + def _pattern( + residual: torch.Tensor, + input: torch.Tensor, + weight: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + allreduce_output = tensor_model_parallel_all_reduce(input) + rms, residual_out = vllm.ir.ops.fused_add_rms_norm( + allreduce_output, residual, weight, self.epsilon + ) + quant, scale = self.quant_matcher(rms) + return quant, scale, residual_out + + return _pattern + + @property + def replacement(self): + def _replacement( + residual: torch.Tensor, + input: torch.Tensor, + weight: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + result = self.FUSED_AR_RMS_QUANT_OP( + input_=input, + residual=residual, + weight=weight.to(input.dtype), + epsilon=self.epsilon, + group_size=self.group_size, + ) + # quant_out, scale_out, residual_out + return result[0], result[2], result[1] + + return _replacement + + +class AiterAllreduceFusedAddRMSNormGroupQuantWithIndexerPattern( + BasePattern, VllmPatternReplacement +): + """Indexer-fan-out variant of ``AiterAllreduceFusedAddRMSNormGroupQuantFP8Pattern``. + + Targets the DSv3.2 post-attention / post-MLP path where the post-AR normed + activation has two consumers: a per-group FP8 quant for ``fused_qkv_a_proj`` + *and* a bf16 ``rocm_unquantized_gemm`` for the indexer ``wk_weights_proj``. + The single-consumer pattern above cannot fire when this fan-out is present, + so without this pattern the standalone FP8 quant kernel survives unfused + (~535us / decode step on DSv3.2 MI355X TP4). + + Lowers to ``rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm`` + (the ``emit_bf16=True`` variant of the AR+RMS+QUANT launcher, which returns + FP8 quant + scales + bf16 normed activations in one kernel) and rewires the + indexer GEMM onto the emitted bf16 norm output. The RMS output is also a + graph output in DSv3.2's residual carry; it is returned as a pattern output + so the matcher can substitute the bf16 norm in its place. + + The trailing FP8 group-quant is matched via ``MatcherQuantFP8`` (consistent + with the sibling patterns above), which traces both ``QuantFP8.forward_hip`` + and ``forward_native`` paths and so matches whichever op the call site + lowers to (``vllm.triton_per_token_group_quant_fp8`` or + ``vllm.rocm_aiter_group_fp8_quant``). + """ + + def __init__( + self, + epsilon: float, + dtype: torch.dtype, + device: str | None, + group_size: int = 128, + ) -> None: + super().__init__(dtype, device) + self.epsilon = epsilon + self.dtype = dtype + self.group_size = group_size + self.FUSED_AR_RMS_QUANT_BF16_OP = ( + rocm_aiter_ops.get_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_op() # noqa: E501 + ) + self.quant_dtype = current_platform.fp8_dtype() + self.quant_matcher = MatcherQuantFP8( + QuantKey( + dtype=self.quant_dtype, + scale=ScaleDesc(torch.float32, False, GroupShape(1, group_size)), + symmetric=True, + ), + match_rocm_aiter=True, + ) + + def get_inputs(self) -> list[torch.Tensor]: + h = self.group_size + indexer_out = 8 + return [ + self.empty(5, h), + self.empty(5, h), + self.empty(h), + self.empty(indexer_out, h), + ] + + @property + def pattern(self): + eps = self.epsilon + + def _pattern( + residual: torch.Tensor, + input_: torch.Tensor, + norm_weight: torch.Tensor, + indexer_weight: torch.Tensor, + ) -> tuple[ + torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor + ]: + ar_out = tensor_model_parallel_all_reduce(input_) + rms, res_out = vllm.ir.ops.fused_add_rms_norm( + ar_out, residual, norm_weight, eps + ) + q, s = self.quant_matcher(rms) + idx = torch.ops.vllm.rocm_unquantized_gemm(rms, indexer_weight) + return q, s, res_out, idx, rms + + return _pattern + + @property + def replacement(self): + gs = self.group_size + eps = self.epsilon + + def _replacement( + residual: torch.Tensor, + input_: torch.Tensor, + norm_weight: torch.Tensor, + indexer_weight: torch.Tensor, + ) -> tuple[ + torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor + ]: + fused = self.FUSED_AR_RMS_QUANT_BF16_OP( + input_=input_, + residual=residual, + weight=norm_weight.to(input_.dtype), + epsilon=eps, + group_size=gs, + ) + quant_out, residual_out, scale_out, bf16_norm = ( + fused[0], + fused[1], + fused[2], + fused[3], + ) + idx = torch.ops.vllm.rocm_unquantized_gemm(bf16_norm, indexer_weight) + return quant_out, scale_out, residual_out, idx, bf16_norm + + return _replacement + + class RocmAiterAllReduceFusionPass(VllmFusionPatternMatcherPass): def __init__(self, config: VllmConfig) -> None: super().__init__(config, "rocm_aiter_allreduce_fusion_pass") @@ -1232,7 +1508,52 @@ class RocmAiterAllReduceFusionPass(VllmFusionPatternMatcherPass): config.scheduler_config.max_num_batched_tokens, ) + # Only register the AR+RMS+per-group-FP8-quant patterns when the + # running aiter exposes the kernel. Older aiter builds (pre PR #2823) + # fall back to the AR+RMS-only fusion paired with PR #41825's + # standalone RMS+quant fusion -- still correct, just leaves the + # post-AR quant as a standalone kernel. + supports_per_group_quant = ( + rocm_aiter_ops.has_fused_allreduce_rmsnorm_quant_per_group() + ) + if not supports_per_group_quant: + logger.warning_once( + "AITER AR+RMS+per-group-FP8-quant fusion disabled: aiter " + "build is missing 'fused_ar_rms_per_group_quant'. Upgrade " + "aiter past PR #2823 to enable the trailing per-group " + "FP8 quant fusion." + ) + for epsilon in [1e-5, 1e-6]: + # Quant-fused variants must register first so the pattern matcher + # tries them before the AR+RMS-only variants. Otherwise the + # AR+RMS-only fusion runs first and consumes the all_reduce node, + # leaving the trailing quant op stranded as an unfused kernel. + # Register larger subgraphs first (DeepSeek indexer fan-out, then + # quant-only AR+RMS+quant, then AR+RMS-only). + if supports_per_group_quant: + self.register( + AiterAllreduceFusedAddRMSNormGroupQuantWithIndexerPattern( + epsilon, + self.model_dtype, + self.device, + ) + ) + self.register( + AiterAllreduceFusedRMSNormGroupQuantFP8Pattern( + epsilon, + self.model_dtype, + self.device, + ) + ) + self.register( + AiterAllreduceFusedAddRMSNormGroupQuantFP8Pattern( + epsilon, + self.model_dtype, + self.device, + ) + ) + self.register( AiterAllreduceFusedRMSNormPattern( epsilon,