[ROCm][Compile] Fuse AR + RMSNorm + per-group FP8 quant (+ DSv3.2 indexer fan-out) (#42864)

Signed-off-by: Markus Hartikainen <markus.hartikainen@amd.com>
Co-authored-by: Frida Andersson <fanderss@amd.com>
This commit is contained in:
Markus Hartikainen
2026-06-09 12:06:56 +00:00
committed by GitHub
co-authored by Frida Andersson
parent 5b3807e862
commit 80e2c4462d
3 changed files with 795 additions and 2 deletions
@@ -189,6 +189,118 @@ class TestAllReduceRMSNormStaticQuantFP8Model(torch.nn.Module):
]
class TestAiterAllReduceRMSNormGroupQuantFP8Model(torch.nn.Module):
"""Exercises the new ROCm AITER AR+RMS+per-group-FP8-quant patterns.
Four ``rms_norm`` sites that together hit every pattern registered by
``RocmAiterAllReduceFusionPass`` for the per-group FP8 quant path:
* ``norm[0]``: ``all_reduce -> rms_norm -> group_fp8_quant`` (no residual)
-> ``AiterAllreduceFusedRMSNormGroupQuantFP8Pattern``
* ``norm[1]``: ``all_reduce -> fused_add_rms_norm -> group_fp8_quant``
(single ``rms`` consumer)
-> ``AiterAllreduceFusedAddRMSNormGroupQuantFP8Pattern``
* ``norm[2..3]``: ``all_reduce -> fused_add_rms_norm
-> (group_fp8_quant + rocm_unquantized_gemm)`` (two ``rms`` consumers,
modeling the DSv3.2 indexer fan-out)
-> ``AiterAllreduceFusedAddRMSNormGroupQuantWithIndexerPattern``
The chain feeds the next AllReduce by dequantizing the FP8 output (FP8
cast back to bf16 multiplied by the per-group scale), which is enough to
keep the matmul chain bf16 without depending on a real FP8 block-scaled
GEMM kernel.
"""
quant_group_size = 128
indexer_out_dim = 8
def __init__(
self,
hidden_size=128,
token_num=16,
eps=1e-6,
dtype: torch.dtype = torch.bfloat16,
use_triton_quant: bool = False,
):
super().__init__()
self.hidden_size = hidden_size
self.eps = eps
self.use_triton_quant = use_triton_quant
assert hidden_size % self.quant_group_size == 0, (
f"hidden_size ({hidden_size}) must be a multiple of "
f"quant_group_size ({self.quant_group_size}) for per-group FP8 quant"
)
self.norm = [RMSNorm(hidden_size, eps) for _ in range(4)]
self.w = [torch.rand(hidden_size, hidden_size, dtype=dtype) for _ in range(3)]
self.indexer_w = [
torch.rand(self.indexer_out_dim, hidden_size, dtype=dtype) for _ in range(2)
]
def _group_quant(self, rms: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
if self.use_triton_quant:
return torch.ops.vllm.triton_per_token_group_quant_fp8(
rms, self.quant_group_size
)
return torch.ops.vllm.rocm_aiter_group_fp8_quant.default(
rms, self.quant_group_size
)
def _dequantize_to_bf16(
self, q: torch.Tensor, s: torch.Tensor, ref: torch.Tensor
) -> torch.Tensor:
# Broadcast the per-group scale across each group of `quant_group_size`
# so we can chain the FP8 output back into a bf16 matmul. This avoids
# depending on a real FP8 block-scaled GEMM kernel in the test.
s_full = s.repeat_interleave(self.quant_group_size, dim=-1).to(ref.dtype)
return q.to(ref.dtype) * s_full
def forward(self, hidden_states):
z = torch.relu(hidden_states)
x = resid = tensor_model_parallel_all_reduce(z)
rms = self.norm[0](x)
q0, s0 = self._group_quant(rms)
y = self._dequantize_to_bf16(q0, s0, rms)
z2 = torch.mm(y, self.w[0])
x2 = tensor_model_parallel_all_reduce(z2)
rms2, resid = self.norm[1](x2, resid)
q1, s1 = self._group_quant(rms2)
y2 = self._dequantize_to_bf16(q1, s1, rms2)
z3 = torch.mm(y2, self.w[1])
x3 = tensor_model_parallel_all_reduce(z3)
rms3, resid = self.norm[2](x3, resid)
q2, s2 = self._group_quant(rms3)
# Second consumer of ``rms3``: forces the with-indexer pattern.
idx2 = torch.ops.vllm.rocm_unquantized_gemm(rms3, self.indexer_w[0], None)
y3 = self._dequantize_to_bf16(q2, s2, rms3)
z4 = torch.mm(y3, self.w[2])
x4 = tensor_model_parallel_all_reduce(z4)
rms4, resid = self.norm[3](x4, resid)
q3, s3 = self._group_quant(rms4)
# Second consumer of ``rms4``: forces the with-indexer pattern.
idx3 = torch.ops.vllm.rocm_unquantized_gemm(rms4, self.indexer_w[1], None)
y4 = self._dequantize_to_bf16(q3, s3, rms4)
return y4, idx2, idx3
def ops_in_model_before(self):
return [
torch.ops.vllm.all_reduce.default,
(
torch.ops.vllm.triton_per_token_group_quant_fp8.default
if self.use_triton_quant
else torch.ops.vllm.rocm_aiter_group_fp8_quant.default
),
]
def ops_in_model_after(self):
return [
rocm_aiter_ops.get_fused_allreduce_rmsnorm_quant_per_group_op(),
rocm_aiter_ops.get_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_op(), # noqa: E501
]
class TestAllReduceFusedAddRMSNormStaticQuantFP4Model(torch.nn.Module):
def __init__(
self, hidden_size=16, token_num=16, eps=1e-6, dtype: torch.dtype = torch.float16
@@ -463,3 +575,167 @@ def all_reduce_fusion_pass_on_test_model(
assert fused_nodes
assert all(n.kwargs.get("weight_bias") == 1.0 for n in fused_nodes)
del all_reduce_fusion_pass
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize("use_triton_quant", [True, False])
@pytest.mark.parametrize("batch_size", [8])
@pytest.mark.parametrize("seq_len", [8])
@pytest.mark.parametrize("hidden_size", [128])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("enable_rms_norm_custom_op", [True, False])
@pytest.mark.skipif(
not current_platform.is_rocm(),
reason="ROCm AITER AR+RMS+per-group-FP8-quant fusion is ROCm-only",
)
@pytest.mark.skipif(not IS_AITER_FOUND, reason="aiter is not found")
def test_rocm_aiter_all_reduce_rmsnorm_group_quant_fp8_fusion_pass_replace(
batch_size: int,
seq_len: int,
hidden_size: int,
dtype: torch.dtype,
enable_rms_norm_custom_op: bool,
use_triton_quant: bool,
monkeypatch: pytest.MonkeyPatch,
):
"""Sibling of ``test_all_reduce_fusion_pass_replace`` for the new
ROCm AITER AR+RMS+per-group-FP8-quant fusion patterns.
Validates the three new ``VllmPatternReplacement`` patterns added to
``RocmAiterAllReduceFusionPass``:
* ``AiterAllreduceFusedRMSNormGroupQuantFP8Pattern`` (no-residual)
* ``AiterAllreduceFusedAddRMSNormGroupQuantFP8Pattern`` (with-residual,
single ``rms`` consumer)
* ``AiterAllreduceFusedAddRMSNormGroupQuantWithIndexerPattern`` (with-
residual, DSv3.2 indexer fan-out; parametrized over both
``triton_per_token_group_quant_fp8`` and ``rocm_aiter_group_fp8_quant``
producers).
"""
with monkeypatch.context() as m:
m.setenv("VLLM_ROCM_USE_AITER", "1")
rocm_aiter_ops.refresh_env_variables()
if not rocm_aiter_ops.has_fused_allreduce_rmsnorm_quant_per_group():
pytest.skip(
"aiter build is missing 'fused_ar_rms_per_group_quant' (needs "
"ROCm/aiter PR #2823); the new patterns aren't registered."
)
num_processes = 2
def run_torch_spawn(fn, nprocs):
torch.multiprocessing.spawn(
fn,
args=(
num_processes,
TestAiterAllReduceRMSNormGroupQuantFP8Model,
batch_size,
seq_len,
hidden_size,
dtype,
enable_rms_norm_custom_op,
use_triton_quant,
monkeypatch,
),
nprocs=nprocs,
)
run_torch_spawn(rocm_aiter_group_quant_fusion_pass_on_test_model, num_processes)
def rocm_aiter_group_quant_fusion_pass_on_test_model(
local_rank: int,
world_size: int,
test_model_cls: torch.nn.Module,
batch_size: int,
seq_len: int,
hidden_size: int,
dtype: torch.dtype,
enable_rms_norm_custom_op: bool,
use_triton_quant: bool,
monkeypatch: pytest.MonkeyPatch,
):
set_random_seed(0)
device = torch.device(f"{DEVICE_TYPE}:{local_rank}")
torch.accelerator.set_device_index(device)
torch.set_default_device(device)
torch.set_default_dtype(dtype)
update_environment_variables(
{
"RANK": str(local_rank),
"LOCAL_RANK": str(local_rank),
"WORLD_SIZE": str(world_size),
"MASTER_ADDR": "localhost",
"MASTER_PORT": "12345",
"VLLM_ROCM_USE_AITER": "1",
}
)
rocm_aiter_ops.refresh_env_variables()
init_distributed_environment()
custom_ops = []
if enable_rms_norm_custom_op:
custom_ops.append("+rms_norm")
# ``triton_per_token_group_quant_fp8`` is emitted by ``QuantFP8.forward_hip``
# only when QuantFP8 is enabled as a custom op (and ``use_triton=True`` at
# the call site). The patterns in this PR are robust to both Triton and
# rocm_aiter forms; we always enable +quant_fp8 so the matcher's example
# trace finds the same form the test model uses.
custom_ops.append("+quant_fp8")
vllm_config = VllmConfig(
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE, custom_ops=custom_ops
)
)
vllm_config.compilation_config.pass_config = PassConfig(
fuse_allreduce_rms=True, eliminate_noops=True
)
vllm_config.device_config = DeviceConfig(device=torch.device(DEVICE_TYPE))
vllm_config.parallel_config.rank = local_rank
model_name = "RedHatAI/Llama-3.2-1B-Instruct-FP8"
vllm_config.model_config = ModelConfig(
model=model_name, trust_remote_code=True, dtype=dtype, seed=42
)
with set_current_vllm_config(vllm_config):
initialize_model_parallel(tensor_model_parallel_size=world_size)
all_reduce_fusion_pass = RocmAiterAllReduceFusionPass(vllm_config)
noop_pass = NoOpEliminationPass(vllm_config)
func_pass = FixFunctionalizationPass(vllm_config)
cleanup_pass = PostCleanupPass(vllm_config)
backend = TestBackend(
noop_pass, all_reduce_fusion_pass, func_pass, cleanup_pass
)
token_num = batch_size * seq_len
model = test_model_cls(
hidden_size, token_num, dtype=dtype, use_triton_quant=use_triton_quant
)
hidden_states = torch.randn((token_num, hidden_size), requires_grad=False)
compiled_model = torch.compile(model, backend=backend)
compiled_model(hidden_states)
results_unfused = model(hidden_states)
results_fused = compiled_model(hidden_states)
# The fused per-group AR+RMS+QUANT op is bit-equivalent to the unfused
# chain modulo the small AllReduce + RMSNorm reordering inside aiter.
# Per-group FP8 quant introduces step noise <=1 per group; use the
# same tolerance as the sibling FP8 static test.
torch.testing.assert_close(results_unfused, results_fused, atol=1e-2, rtol=1e-2)
# Four pattern firings: norm[0] (no-add quant), norm[1] (add quant,
# single ``rms`` consumer), norm[2..3] (add quant + indexer fan-out).
assert all_reduce_fusion_pass.matched_count == 4, (
f"{all_reduce_fusion_pass.matched_count=}"
)
backend.check_before_ops(model.ops_in_model_before(), fully_replaced=False)
backend.check_after_ops(model.ops_in_model_after())
del all_reduce_fusion_pass
+196
View File
@@ -70,6 +70,21 @@ class AiterCustomAllreduceProto(Protocol):
registered: bool = False,
use_1stage: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]: ...
def fused_ar_rms_per_group_quant(
self,
inp: torch.Tensor,
res_inp: torch.Tensor,
*,
w: torch.Tensor,
eps: float,
group_size: int = 128,
registered: bool = False,
use_1stage: bool = False,
emit_bf16: bool = False,
) -> (
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]
): ...
def should_custom_ar(self, inp: torch.Tensor) -> bool: ...
@@ -850,6 +865,152 @@ def _rocm_aiter_fused_allreduce_rmsnorm_fake(
return torch.empty_like(input_), torch.empty_like(residual)
def _rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_impl(
input_: torch.Tensor,
residual: torch.Tensor,
weight: torch.Tensor,
epsilon: float,
group_size: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Fused AllReduce + RMSNorm + per-group FP8 quant.
Mirrors the eligibility logic of ``_rocm_aiter_fused_allreduce_rmsnorm_impl``
for the 1-stage vs 2-stage AITER kernel dispatch (both variants run inside
AITER, the only choice we make here is the launcher to call into).
"""
aiter_ar = rocm_aiter_ops.get_aiter_allreduce()
assert aiter_ar is not None, "aiter allreduce must be initialized"
total_bytes = input_.numel() * input_.element_size()
hidden_dim = input_.shape[-1]
token_num = input_.shape[0]
if input_.dtype in (torch.bfloat16, torch.float16):
pack_size = 16 // input_.element_size()
hidden_ok = hidden_dim % pack_size == 0 and hidden_dim // pack_size <= 1024
else:
hidden_ok = False
token_ok = token_num <= 80
world_size = aiter_ar.world_size
full_nvlink = aiter_ar.fully_connected
if world_size == 2:
size_ok = True
elif full_nvlink and world_size <= 4:
size_ok = total_bytes < 256 * 1024
elif full_nvlink and world_size <= 8:
size_ok = total_bytes < 128 * 1024
else:
size_ok = False
use_1stage = hidden_ok and token_ok and size_ok
result = aiter_ar.fused_ar_rms_per_group_quant(
input_,
residual,
w=weight,
eps=epsilon,
group_size=group_size,
registered=torch.cuda.is_current_stream_capturing(),
use_1stage=use_1stage,
)
assert result is not None
return result[0], result[1], result[2]
def _rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_fake(
input_: torch.Tensor,
residual: torch.Tensor,
weight: torch.Tensor,
epsilon: float,
group_size: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
hidden_dim = input_.shape[-1]
num_groups = hidden_dim // group_size
quant_out = torch.empty(input_.shape, dtype=FP8_DTYPE, device=input_.device)
residual_out = torch.empty_like(residual)
scale_out = torch.empty(
input_.shape[:-1] + (num_groups,),
dtype=torch.float32,
device=input_.device,
)
return quant_out, residual_out, scale_out
def _rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_impl(
input_: torch.Tensor,
residual: torch.Tensor,
weight: torch.Tensor,
epsilon: float,
group_size: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Fused AllReduce + add-RMSNorm + per-group FP8 quant + bf16 normed act.
Wraps the same AITER launcher as ``_rocm_aiter_fused_allreduce_rmsnorm_
quant_per_group_impl`` with ``emit_bf16=True``, emitting the pre-quant
bf16/fp16 normed activation for a parallel consumer (DeepSeek V3.2 sparse
indexer ``wk_weights_proj``).
"""
aiter_ar = rocm_aiter_ops.get_aiter_allreduce()
assert aiter_ar is not None, "aiter allreduce must be initialized"
total_bytes = input_.numel() * input_.element_size()
hidden_dim = input_.shape[-1]
token_num = input_.shape[0]
if input_.dtype in (torch.bfloat16, torch.float16):
pack_size = 16 // input_.element_size()
hidden_ok = hidden_dim % pack_size == 0 and hidden_dim // pack_size <= 1024
else:
hidden_ok = False
token_ok = token_num <= 80
world_size = aiter_ar.world_size
full_nvlink = aiter_ar.fully_connected
if world_size == 2:
size_ok = True
elif full_nvlink and world_size <= 4:
size_ok = total_bytes < 256 * 1024
elif full_nvlink and world_size <= 8:
size_ok = total_bytes < 128 * 1024
else:
size_ok = False
use_1stage = hidden_ok and token_ok and size_ok
result = aiter_ar.fused_ar_rms_per_group_quant(
input_,
residual,
w=weight,
eps=epsilon,
group_size=group_size,
registered=torch.cuda.is_current_stream_capturing(),
use_1stage=use_1stage,
emit_bf16=True,
)
assert result is not None
assert len(result) == 4, "emit_bf16=True must return four tensors from aiter"
return result[0], result[1], result[2], result[3]
def _rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_fake(
input_: torch.Tensor,
residual: torch.Tensor,
weight: torch.Tensor,
epsilon: float,
group_size: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
hidden_dim = input_.shape[-1]
num_groups = hidden_dim // group_size
quant_out = torch.empty(input_.shape, dtype=FP8_DTYPE, device=input_.device)
residual_out = torch.empty_like(residual)
scale_out = torch.empty(
input_.shape[:-1] + (num_groups,),
dtype=torch.float32,
device=input_.device,
)
bf16_norm_out = torch.empty_like(input_)
return quant_out, residual_out, scale_out, bf16_norm_out
def _rocm_aiter_per_tensor_quant_impl(
x: torch.Tensor,
quant_dtype: torch.dtype,
@@ -1838,6 +1999,18 @@ class rocm_aiter_ops:
fake_impl=_rocm_aiter_fused_allreduce_rmsnorm_fake,
)
direct_register_custom_op(
op_name="rocm_aiter_fused_allreduce_rmsnorm_quant_per_group",
op_func=(_rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_impl),
fake_impl=(_rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_fake),
)
direct_register_custom_op(
op_name="rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm", # noqa: E501
op_func=_rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_impl, # noqa: E501
fake_impl=_rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_fake, # noqa: E501
)
direct_register_custom_op(
op_name="fused_mla_dual_rms_norm",
op_func=_fused_mla_dual_rms_norm_impl,
@@ -1892,6 +2065,29 @@ class rocm_aiter_ops:
def get_fused_allreduce_rmsnorm_op() -> OpOverload:
return torch.ops.vllm.rocm_aiter_fused_allreduce_rmsnorm.default
@staticmethod
def get_fused_allreduce_rmsnorm_quant_per_group_op() -> OpOverload:
return torch.ops.vllm.rocm_aiter_fused_allreduce_rmsnorm_quant_per_group.default
@staticmethod
def get_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_op() -> OpOverload: # noqa: E501
return torch.ops.vllm.rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm.default # noqa: E501
# TODO(frida-andersson): drop once vLLM pins AITER >= 0.1.14 (ROCm/aiter#2823).
@classmethod
def has_fused_allreduce_rmsnorm_quant_per_group(cls) -> bool:
"""True if the running AITER build exposes the per-group AR+RMS+quant
kernel (added in ROCm/aiter PR #2823).
The pattern registration in ``RocmAiterAllReduceFusionPass`` keys off
this so vLLM degrades to the AR+RMS-only fusion when run against an
older aiter that lacks the per-group launcher.
"""
aiter_ar = cls.get_aiter_allreduce()
return aiter_ar is not None and hasattr(
aiter_ar, "fused_ar_rms_per_group_quant"
)
@staticmethod
def get_fused_mla_dual_rms_norm_op() -> OpOverload:
return torch.ops.vllm.fused_mla_dual_rms_norm.default
@@ -26,6 +26,9 @@ from vllm.distributed.parallel_state import (
)
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
QuantKey,
ScaleDesc,
kFp8StaticTensorSym,
)
from vllm.platforms import current_platform
@@ -1109,7 +1112,7 @@ class AiterAllreduceFusedRMSNormPattern(BasePattern, VllmPatternReplacement):
allreduce = self.FUSED_AR_RMSNORM_OP(
input_=input,
residual=residual,
weight=weight,
weight=weight.to(input.dtype),
epsilon=self.epsilon,
)
return allreduce[0], allreduce[1]
@@ -1155,7 +1158,7 @@ class AiterAllreduceFusedAddRMSNormPattern(BasePattern, VllmPatternReplacement):
allreduce = self.FUSED_AR_RMSNORM_OP(
input_=input,
residual=residual,
weight=weight,
weight=weight.to(input.dtype),
epsilon=self.epsilon,
)
return allreduce[0], allreduce[1]
@@ -1163,6 +1166,279 @@ class AiterAllreduceFusedAddRMSNormPattern(BasePattern, VllmPatternReplacement):
return _replacement
class AiterAllreduceFusedRMSNormGroupQuantFP8Pattern(
BasePattern, VllmPatternReplacement
):
"""Fuse AllReduce + RMSNorm + per-group FP8 quant into a single AITER
custom op.
Matches the AR-side analogue of ``AiterRMSFp8GroupQuantPattern`` in
``rocm_aiter_fusion.py``: ``all_reduce -> rms_norm -> group_fp8_quant``
fans out into ``rocm_aiter_fused_allreduce_rmsnorm_quant_per_group``.
Without this pattern, ``RocmAiterAllReduceFusionPass`` would fuse the
``all_reduce + rms_norm`` half (PR #41825 wires that), but the trailing
``rocm_aiter_group_fp8_quant`` would still launch as a separate kernel.
That standalone quant accounts for ~535us / decode step on DSv3.2 MI355X
TP4 -- this pattern eliminates it by absorbing the quant into the AR
epilogue. Group size 128 matches the FP8 block-scaled MM kernel used by
DSv3.2's linear weights.
"""
def __init__(
self,
epsilon: float,
dtype: torch.dtype,
device: str | None,
group_size: int = 128,
) -> None:
super().__init__(dtype, device)
self.epsilon = epsilon
self.dtype = dtype
self.group_size = group_size
self.FUSED_AR_RMS_QUANT_OP = (
rocm_aiter_ops.get_fused_allreduce_rmsnorm_quant_per_group_op()
)
self.quant_dtype = current_platform.fp8_dtype()
self.quant_matcher = MatcherQuantFP8(
QuantKey(
dtype=self.quant_dtype,
scale=ScaleDesc(torch.float32, False, GroupShape(1, group_size)),
symmetric=True,
),
match_rocm_aiter=True,
)
def get_inputs(self) -> list[torch.Tensor]:
# input, weight; hidden dim must be a group_size multiple so the
# group quant matcher's example trace is well-defined.
return [self.empty(5, self.group_size), self.empty(self.group_size)]
@property
def pattern(self):
def _pattern(
input: torch.Tensor, weight: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
allreduce_output = tensor_model_parallel_all_reduce(input)
rms = vllm.ir.ops.rms_norm(allreduce_output, weight, self.epsilon)
quant, scale = self.quant_matcher(rms)
return quant, scale
return _pattern
@property
def replacement(self):
def _replacement(
input: torch.Tensor, weight: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
residual = torch.zeros_like(input)
result = self.FUSED_AR_RMS_QUANT_OP(
input_=input,
residual=residual,
weight=weight.to(input.dtype),
epsilon=self.epsilon,
group_size=self.group_size,
)
# quant_out, scale_out (residual is unused on the no-add path,
# mirroring how AiterAllreduceFusedRMSNormPattern drops the
# residual output)
return result[0], result[2]
return _replacement
class AiterAllreduceFusedAddRMSNormGroupQuantFP8Pattern(
BasePattern, VllmPatternReplacement
):
"""``fused_add`` variant of ``AiterAllreduceFusedRMSNormGroupQuantFP8Pattern``.
Targets the dominant DSv3.2-style post-attention / post-MLP path:
``all_reduce -> fused_add_rms_norm -> group_fp8_quant``. Returns the
FP8 quant output, the residual carry-over, and the per-group scale.
"""
def __init__(
self,
epsilon: float,
dtype: torch.dtype,
device: str | None,
group_size: int = 128,
) -> None:
super().__init__(dtype, device)
self.epsilon = epsilon
self.dtype = dtype
self.group_size = group_size
self.FUSED_AR_RMS_QUANT_OP = (
rocm_aiter_ops.get_fused_allreduce_rmsnorm_quant_per_group_op()
)
self.quant_dtype = current_platform.fp8_dtype()
self.quant_matcher = MatcherQuantFP8(
QuantKey(
dtype=self.quant_dtype,
scale=ScaleDesc(torch.float32, False, GroupShape(1, group_size)),
symmetric=True,
),
match_rocm_aiter=True,
)
def get_inputs(self) -> list[torch.Tensor]:
# residual, input, weight
return [
self.empty(5, self.group_size),
self.empty(5, self.group_size),
self.empty(self.group_size),
]
@property
def pattern(self):
def _pattern(
residual: torch.Tensor,
input: torch.Tensor,
weight: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
allreduce_output = tensor_model_parallel_all_reduce(input)
rms, residual_out = vllm.ir.ops.fused_add_rms_norm(
allreduce_output, residual, weight, self.epsilon
)
quant, scale = self.quant_matcher(rms)
return quant, scale, residual_out
return _pattern
@property
def replacement(self):
def _replacement(
residual: torch.Tensor,
input: torch.Tensor,
weight: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
result = self.FUSED_AR_RMS_QUANT_OP(
input_=input,
residual=residual,
weight=weight.to(input.dtype),
epsilon=self.epsilon,
group_size=self.group_size,
)
# quant_out, scale_out, residual_out
return result[0], result[2], result[1]
return _replacement
class AiterAllreduceFusedAddRMSNormGroupQuantWithIndexerPattern(
BasePattern, VllmPatternReplacement
):
"""Indexer-fan-out variant of ``AiterAllreduceFusedAddRMSNormGroupQuantFP8Pattern``.
Targets the DSv3.2 post-attention / post-MLP path where the post-AR normed
activation has two consumers: a per-group FP8 quant for ``fused_qkv_a_proj``
*and* a bf16 ``rocm_unquantized_gemm`` for the indexer ``wk_weights_proj``.
The single-consumer pattern above cannot fire when this fan-out is present,
so without this pattern the standalone FP8 quant kernel survives unfused
(~535us / decode step on DSv3.2 MI355X TP4).
Lowers to ``rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm``
(the ``emit_bf16=True`` variant of the AR+RMS+QUANT launcher, which returns
FP8 quant + scales + bf16 normed activations in one kernel) and rewires the
indexer GEMM onto the emitted bf16 norm output. The RMS output is also a
graph output in DSv3.2's residual carry; it is returned as a pattern output
so the matcher can substitute the bf16 norm in its place.
The trailing FP8 group-quant is matched via ``MatcherQuantFP8`` (consistent
with the sibling patterns above), which traces both ``QuantFP8.forward_hip``
and ``forward_native`` paths and so matches whichever op the call site
lowers to (``vllm.triton_per_token_group_quant_fp8`` or
``vllm.rocm_aiter_group_fp8_quant``).
"""
def __init__(
self,
epsilon: float,
dtype: torch.dtype,
device: str | None,
group_size: int = 128,
) -> None:
super().__init__(dtype, device)
self.epsilon = epsilon
self.dtype = dtype
self.group_size = group_size
self.FUSED_AR_RMS_QUANT_BF16_OP = (
rocm_aiter_ops.get_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_op() # noqa: E501
)
self.quant_dtype = current_platform.fp8_dtype()
self.quant_matcher = MatcherQuantFP8(
QuantKey(
dtype=self.quant_dtype,
scale=ScaleDesc(torch.float32, False, GroupShape(1, group_size)),
symmetric=True,
),
match_rocm_aiter=True,
)
def get_inputs(self) -> list[torch.Tensor]:
h = self.group_size
indexer_out = 8
return [
self.empty(5, h),
self.empty(5, h),
self.empty(h),
self.empty(indexer_out, h),
]
@property
def pattern(self):
eps = self.epsilon
def _pattern(
residual: torch.Tensor,
input_: torch.Tensor,
norm_weight: torch.Tensor,
indexer_weight: torch.Tensor,
) -> tuple[
torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor
]:
ar_out = tensor_model_parallel_all_reduce(input_)
rms, res_out = vllm.ir.ops.fused_add_rms_norm(
ar_out, residual, norm_weight, eps
)
q, s = self.quant_matcher(rms)
idx = torch.ops.vllm.rocm_unquantized_gemm(rms, indexer_weight)
return q, s, res_out, idx, rms
return _pattern
@property
def replacement(self):
gs = self.group_size
eps = self.epsilon
def _replacement(
residual: torch.Tensor,
input_: torch.Tensor,
norm_weight: torch.Tensor,
indexer_weight: torch.Tensor,
) -> tuple[
torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor
]:
fused = self.FUSED_AR_RMS_QUANT_BF16_OP(
input_=input_,
residual=residual,
weight=norm_weight.to(input_.dtype),
epsilon=eps,
group_size=gs,
)
quant_out, residual_out, scale_out, bf16_norm = (
fused[0],
fused[1],
fused[2],
fused[3],
)
idx = torch.ops.vllm.rocm_unquantized_gemm(bf16_norm, indexer_weight)
return quant_out, scale_out, residual_out, idx, bf16_norm
return _replacement
class RocmAiterAllReduceFusionPass(VllmFusionPatternMatcherPass):
def __init__(self, config: VllmConfig) -> None:
super().__init__(config, "rocm_aiter_allreduce_fusion_pass")
@@ -1232,7 +1508,52 @@ class RocmAiterAllReduceFusionPass(VllmFusionPatternMatcherPass):
config.scheduler_config.max_num_batched_tokens,
)
# Only register the AR+RMS+per-group-FP8-quant patterns when the
# running aiter exposes the kernel. Older aiter builds (pre PR #2823)
# fall back to the AR+RMS-only fusion paired with PR #41825's
# standalone RMS+quant fusion -- still correct, just leaves the
# post-AR quant as a standalone kernel.
supports_per_group_quant = (
rocm_aiter_ops.has_fused_allreduce_rmsnorm_quant_per_group()
)
if not supports_per_group_quant:
logger.warning_once(
"AITER AR+RMS+per-group-FP8-quant fusion disabled: aiter "
"build is missing 'fused_ar_rms_per_group_quant'. Upgrade "
"aiter past PR #2823 to enable the trailing per-group "
"FP8 quant fusion."
)
for epsilon in [1e-5, 1e-6]:
# Quant-fused variants must register first so the pattern matcher
# tries them before the AR+RMS-only variants. Otherwise the
# AR+RMS-only fusion runs first and consumes the all_reduce node,
# leaving the trailing quant op stranded as an unfused kernel.
# Register larger subgraphs first (DeepSeek indexer fan-out, then
# quant-only AR+RMS+quant, then AR+RMS-only).
if supports_per_group_quant:
self.register(
AiterAllreduceFusedAddRMSNormGroupQuantWithIndexerPattern(
epsilon,
self.model_dtype,
self.device,
)
)
self.register(
AiterAllreduceFusedRMSNormGroupQuantFP8Pattern(
epsilon,
self.model_dtype,
self.device,
)
)
self.register(
AiterAllreduceFusedAddRMSNormGroupQuantFP8Pattern(
epsilon,
self.model_dtype,
self.device,
)
)
self.register(
AiterAllreduceFusedRMSNormPattern(
epsilon,