[ROCm][Compile] Fuse AR + RMSNorm + per-group FP8 quant (+ DSv3.2 indexer fan-out) (#42864)
Signed-off-by: Markus Hartikainen <markus.hartikainen@amd.com> Co-authored-by: Frida Andersson <fanderss@amd.com>
This commit is contained in:
co-authored by
Frida Andersson
parent
5b3807e862
commit
80e2c4462d
@@ -189,6 +189,118 @@ class TestAllReduceRMSNormStaticQuantFP8Model(torch.nn.Module):
|
||||
]
|
||||
|
||||
|
||||
class TestAiterAllReduceRMSNormGroupQuantFP8Model(torch.nn.Module):
|
||||
"""Exercises the new ROCm AITER AR+RMS+per-group-FP8-quant patterns.
|
||||
|
||||
Four ``rms_norm`` sites that together hit every pattern registered by
|
||||
``RocmAiterAllReduceFusionPass`` for the per-group FP8 quant path:
|
||||
|
||||
* ``norm[0]``: ``all_reduce -> rms_norm -> group_fp8_quant`` (no residual)
|
||||
-> ``AiterAllreduceFusedRMSNormGroupQuantFP8Pattern``
|
||||
* ``norm[1]``: ``all_reduce -> fused_add_rms_norm -> group_fp8_quant``
|
||||
(single ``rms`` consumer)
|
||||
-> ``AiterAllreduceFusedAddRMSNormGroupQuantFP8Pattern``
|
||||
* ``norm[2..3]``: ``all_reduce -> fused_add_rms_norm
|
||||
-> (group_fp8_quant + rocm_unquantized_gemm)`` (two ``rms`` consumers,
|
||||
modeling the DSv3.2 indexer fan-out)
|
||||
-> ``AiterAllreduceFusedAddRMSNormGroupQuantWithIndexerPattern``
|
||||
|
||||
The chain feeds the next AllReduce by dequantizing the FP8 output (FP8
|
||||
cast back to bf16 multiplied by the per-group scale), which is enough to
|
||||
keep the matmul chain bf16 without depending on a real FP8 block-scaled
|
||||
GEMM kernel.
|
||||
"""
|
||||
|
||||
quant_group_size = 128
|
||||
indexer_out_dim = 8
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size=128,
|
||||
token_num=16,
|
||||
eps=1e-6,
|
||||
dtype: torch.dtype = torch.bfloat16,
|
||||
use_triton_quant: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
self.eps = eps
|
||||
self.use_triton_quant = use_triton_quant
|
||||
assert hidden_size % self.quant_group_size == 0, (
|
||||
f"hidden_size ({hidden_size}) must be a multiple of "
|
||||
f"quant_group_size ({self.quant_group_size}) for per-group FP8 quant"
|
||||
)
|
||||
self.norm = [RMSNorm(hidden_size, eps) for _ in range(4)]
|
||||
self.w = [torch.rand(hidden_size, hidden_size, dtype=dtype) for _ in range(3)]
|
||||
self.indexer_w = [
|
||||
torch.rand(self.indexer_out_dim, hidden_size, dtype=dtype) for _ in range(2)
|
||||
]
|
||||
|
||||
def _group_quant(self, rms: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if self.use_triton_quant:
|
||||
return torch.ops.vllm.triton_per_token_group_quant_fp8(
|
||||
rms, self.quant_group_size
|
||||
)
|
||||
return torch.ops.vllm.rocm_aiter_group_fp8_quant.default(
|
||||
rms, self.quant_group_size
|
||||
)
|
||||
|
||||
def _dequantize_to_bf16(
|
||||
self, q: torch.Tensor, s: torch.Tensor, ref: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
# Broadcast the per-group scale across each group of `quant_group_size`
|
||||
# so we can chain the FP8 output back into a bf16 matmul. This avoids
|
||||
# depending on a real FP8 block-scaled GEMM kernel in the test.
|
||||
s_full = s.repeat_interleave(self.quant_group_size, dim=-1).to(ref.dtype)
|
||||
return q.to(ref.dtype) * s_full
|
||||
|
||||
def forward(self, hidden_states):
|
||||
z = torch.relu(hidden_states)
|
||||
x = resid = tensor_model_parallel_all_reduce(z)
|
||||
rms = self.norm[0](x)
|
||||
q0, s0 = self._group_quant(rms)
|
||||
y = self._dequantize_to_bf16(q0, s0, rms)
|
||||
|
||||
z2 = torch.mm(y, self.w[0])
|
||||
x2 = tensor_model_parallel_all_reduce(z2)
|
||||
rms2, resid = self.norm[1](x2, resid)
|
||||
q1, s1 = self._group_quant(rms2)
|
||||
y2 = self._dequantize_to_bf16(q1, s1, rms2)
|
||||
|
||||
z3 = torch.mm(y2, self.w[1])
|
||||
x3 = tensor_model_parallel_all_reduce(z3)
|
||||
rms3, resid = self.norm[2](x3, resid)
|
||||
q2, s2 = self._group_quant(rms3)
|
||||
# Second consumer of ``rms3``: forces the with-indexer pattern.
|
||||
idx2 = torch.ops.vllm.rocm_unquantized_gemm(rms3, self.indexer_w[0], None)
|
||||
y3 = self._dequantize_to_bf16(q2, s2, rms3)
|
||||
|
||||
z4 = torch.mm(y3, self.w[2])
|
||||
x4 = tensor_model_parallel_all_reduce(z4)
|
||||
rms4, resid = self.norm[3](x4, resid)
|
||||
q3, s3 = self._group_quant(rms4)
|
||||
# Second consumer of ``rms4``: forces the with-indexer pattern.
|
||||
idx3 = torch.ops.vllm.rocm_unquantized_gemm(rms4, self.indexer_w[1], None)
|
||||
y4 = self._dequantize_to_bf16(q3, s3, rms4)
|
||||
return y4, idx2, idx3
|
||||
|
||||
def ops_in_model_before(self):
|
||||
return [
|
||||
torch.ops.vllm.all_reduce.default,
|
||||
(
|
||||
torch.ops.vllm.triton_per_token_group_quant_fp8.default
|
||||
if self.use_triton_quant
|
||||
else torch.ops.vllm.rocm_aiter_group_fp8_quant.default
|
||||
),
|
||||
]
|
||||
|
||||
def ops_in_model_after(self):
|
||||
return [
|
||||
rocm_aiter_ops.get_fused_allreduce_rmsnorm_quant_per_group_op(),
|
||||
rocm_aiter_ops.get_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_op(), # noqa: E501
|
||||
]
|
||||
|
||||
|
||||
class TestAllReduceFusedAddRMSNormStaticQuantFP4Model(torch.nn.Module):
|
||||
def __init__(
|
||||
self, hidden_size=16, token_num=16, eps=1e-6, dtype: torch.dtype = torch.float16
|
||||
@@ -463,3 +575,167 @@ def all_reduce_fusion_pass_on_test_model(
|
||||
assert fused_nodes
|
||||
assert all(n.kwargs.get("weight_bias") == 1.0 for n in fused_nodes)
|
||||
del all_reduce_fusion_pass
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize("use_triton_quant", [True, False])
|
||||
@pytest.mark.parametrize("batch_size", [8])
|
||||
@pytest.mark.parametrize("seq_len", [8])
|
||||
@pytest.mark.parametrize("hidden_size", [128])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("enable_rms_norm_custom_op", [True, False])
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="ROCm AITER AR+RMS+per-group-FP8-quant fusion is ROCm-only",
|
||||
)
|
||||
@pytest.mark.skipif(not IS_AITER_FOUND, reason="aiter is not found")
|
||||
def test_rocm_aiter_all_reduce_rmsnorm_group_quant_fp8_fusion_pass_replace(
|
||||
batch_size: int,
|
||||
seq_len: int,
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
enable_rms_norm_custom_op: bool,
|
||||
use_triton_quant: bool,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""Sibling of ``test_all_reduce_fusion_pass_replace`` for the new
|
||||
ROCm AITER AR+RMS+per-group-FP8-quant fusion patterns.
|
||||
|
||||
Validates the three new ``VllmPatternReplacement`` patterns added to
|
||||
``RocmAiterAllReduceFusionPass``:
|
||||
|
||||
* ``AiterAllreduceFusedRMSNormGroupQuantFP8Pattern`` (no-residual)
|
||||
* ``AiterAllreduceFusedAddRMSNormGroupQuantFP8Pattern`` (with-residual,
|
||||
single ``rms`` consumer)
|
||||
* ``AiterAllreduceFusedAddRMSNormGroupQuantWithIndexerPattern`` (with-
|
||||
residual, DSv3.2 indexer fan-out; parametrized over both
|
||||
``triton_per_token_group_quant_fp8`` and ``rocm_aiter_group_fp8_quant``
|
||||
producers).
|
||||
"""
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
if not rocm_aiter_ops.has_fused_allreduce_rmsnorm_quant_per_group():
|
||||
pytest.skip(
|
||||
"aiter build is missing 'fused_ar_rms_per_group_quant' (needs "
|
||||
"ROCm/aiter PR #2823); the new patterns aren't registered."
|
||||
)
|
||||
|
||||
num_processes = 2
|
||||
|
||||
def run_torch_spawn(fn, nprocs):
|
||||
torch.multiprocessing.spawn(
|
||||
fn,
|
||||
args=(
|
||||
num_processes,
|
||||
TestAiterAllReduceRMSNormGroupQuantFP8Model,
|
||||
batch_size,
|
||||
seq_len,
|
||||
hidden_size,
|
||||
dtype,
|
||||
enable_rms_norm_custom_op,
|
||||
use_triton_quant,
|
||||
monkeypatch,
|
||||
),
|
||||
nprocs=nprocs,
|
||||
)
|
||||
|
||||
run_torch_spawn(rocm_aiter_group_quant_fusion_pass_on_test_model, num_processes)
|
||||
|
||||
|
||||
def rocm_aiter_group_quant_fusion_pass_on_test_model(
|
||||
local_rank: int,
|
||||
world_size: int,
|
||||
test_model_cls: torch.nn.Module,
|
||||
batch_size: int,
|
||||
seq_len: int,
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
enable_rms_norm_custom_op: bool,
|
||||
use_triton_quant: bool,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
set_random_seed(0)
|
||||
|
||||
device = torch.device(f"{DEVICE_TYPE}:{local_rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
torch.set_default_device(device)
|
||||
torch.set_default_dtype(dtype)
|
||||
|
||||
update_environment_variables(
|
||||
{
|
||||
"RANK": str(local_rank),
|
||||
"LOCAL_RANK": str(local_rank),
|
||||
"WORLD_SIZE": str(world_size),
|
||||
"MASTER_ADDR": "localhost",
|
||||
"MASTER_PORT": "12345",
|
||||
"VLLM_ROCM_USE_AITER": "1",
|
||||
}
|
||||
)
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
init_distributed_environment()
|
||||
|
||||
custom_ops = []
|
||||
if enable_rms_norm_custom_op:
|
||||
custom_ops.append("+rms_norm")
|
||||
# ``triton_per_token_group_quant_fp8`` is emitted by ``QuantFP8.forward_hip``
|
||||
# only when QuantFP8 is enabled as a custom op (and ``use_triton=True`` at
|
||||
# the call site). The patterns in this PR are robust to both Triton and
|
||||
# rocm_aiter forms; we always enable +quant_fp8 so the matcher's example
|
||||
# trace finds the same form the test model uses.
|
||||
custom_ops.append("+quant_fp8")
|
||||
|
||||
vllm_config = VllmConfig(
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE, custom_ops=custom_ops
|
||||
)
|
||||
)
|
||||
vllm_config.compilation_config.pass_config = PassConfig(
|
||||
fuse_allreduce_rms=True, eliminate_noops=True
|
||||
)
|
||||
vllm_config.device_config = DeviceConfig(device=torch.device(DEVICE_TYPE))
|
||||
vllm_config.parallel_config.rank = local_rank
|
||||
|
||||
model_name = "RedHatAI/Llama-3.2-1B-Instruct-FP8"
|
||||
vllm_config.model_config = ModelConfig(
|
||||
model=model_name, trust_remote_code=True, dtype=dtype, seed=42
|
||||
)
|
||||
with set_current_vllm_config(vllm_config):
|
||||
initialize_model_parallel(tensor_model_parallel_size=world_size)
|
||||
all_reduce_fusion_pass = RocmAiterAllReduceFusionPass(vllm_config)
|
||||
noop_pass = NoOpEliminationPass(vllm_config)
|
||||
func_pass = FixFunctionalizationPass(vllm_config)
|
||||
cleanup_pass = PostCleanupPass(vllm_config)
|
||||
|
||||
backend = TestBackend(
|
||||
noop_pass, all_reduce_fusion_pass, func_pass, cleanup_pass
|
||||
)
|
||||
|
||||
token_num = batch_size * seq_len
|
||||
model = test_model_cls(
|
||||
hidden_size, token_num, dtype=dtype, use_triton_quant=use_triton_quant
|
||||
)
|
||||
|
||||
hidden_states = torch.randn((token_num, hidden_size), requires_grad=False)
|
||||
|
||||
compiled_model = torch.compile(model, backend=backend)
|
||||
compiled_model(hidden_states)
|
||||
|
||||
results_unfused = model(hidden_states)
|
||||
results_fused = compiled_model(hidden_states)
|
||||
# The fused per-group AR+RMS+QUANT op is bit-equivalent to the unfused
|
||||
# chain modulo the small AllReduce + RMSNorm reordering inside aiter.
|
||||
# Per-group FP8 quant introduces step noise <=1 per group; use the
|
||||
# same tolerance as the sibling FP8 static test.
|
||||
torch.testing.assert_close(results_unfused, results_fused, atol=1e-2, rtol=1e-2)
|
||||
|
||||
# Four pattern firings: norm[0] (no-add quant), norm[1] (add quant,
|
||||
# single ``rms`` consumer), norm[2..3] (add quant + indexer fan-out).
|
||||
assert all_reduce_fusion_pass.matched_count == 4, (
|
||||
f"{all_reduce_fusion_pass.matched_count=}"
|
||||
)
|
||||
backend.check_before_ops(model.ops_in_model_before(), fully_replaced=False)
|
||||
backend.check_after_ops(model.ops_in_model_after())
|
||||
del all_reduce_fusion_pass
|
||||
|
||||
@@ -70,6 +70,21 @@ class AiterCustomAllreduceProto(Protocol):
|
||||
registered: bool = False,
|
||||
use_1stage: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]: ...
|
||||
def fused_ar_rms_per_group_quant(
|
||||
self,
|
||||
inp: torch.Tensor,
|
||||
res_inp: torch.Tensor,
|
||||
*,
|
||||
w: torch.Tensor,
|
||||
eps: float,
|
||||
group_size: int = 128,
|
||||
registered: bool = False,
|
||||
use_1stage: bool = False,
|
||||
emit_bf16: bool = False,
|
||||
) -> (
|
||||
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]
|
||||
): ...
|
||||
def should_custom_ar(self, inp: torch.Tensor) -> bool: ...
|
||||
|
||||
|
||||
@@ -850,6 +865,152 @@ def _rocm_aiter_fused_allreduce_rmsnorm_fake(
|
||||
return torch.empty_like(input_), torch.empty_like(residual)
|
||||
|
||||
|
||||
def _rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_impl(
|
||||
input_: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
epsilon: float,
|
||||
group_size: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Fused AllReduce + RMSNorm + per-group FP8 quant.
|
||||
|
||||
Mirrors the eligibility logic of ``_rocm_aiter_fused_allreduce_rmsnorm_impl``
|
||||
for the 1-stage vs 2-stage AITER kernel dispatch (both variants run inside
|
||||
AITER, the only choice we make here is the launcher to call into).
|
||||
"""
|
||||
aiter_ar = rocm_aiter_ops.get_aiter_allreduce()
|
||||
assert aiter_ar is not None, "aiter allreduce must be initialized"
|
||||
|
||||
total_bytes = input_.numel() * input_.element_size()
|
||||
hidden_dim = input_.shape[-1]
|
||||
token_num = input_.shape[0]
|
||||
if input_.dtype in (torch.bfloat16, torch.float16):
|
||||
pack_size = 16 // input_.element_size()
|
||||
hidden_ok = hidden_dim % pack_size == 0 and hidden_dim // pack_size <= 1024
|
||||
else:
|
||||
hidden_ok = False
|
||||
token_ok = token_num <= 80
|
||||
world_size = aiter_ar.world_size
|
||||
full_nvlink = aiter_ar.fully_connected
|
||||
|
||||
if world_size == 2:
|
||||
size_ok = True
|
||||
elif full_nvlink and world_size <= 4:
|
||||
size_ok = total_bytes < 256 * 1024
|
||||
elif full_nvlink and world_size <= 8:
|
||||
size_ok = total_bytes < 128 * 1024
|
||||
else:
|
||||
size_ok = False
|
||||
|
||||
use_1stage = hidden_ok and token_ok and size_ok
|
||||
|
||||
result = aiter_ar.fused_ar_rms_per_group_quant(
|
||||
input_,
|
||||
residual,
|
||||
w=weight,
|
||||
eps=epsilon,
|
||||
group_size=group_size,
|
||||
registered=torch.cuda.is_current_stream_capturing(),
|
||||
use_1stage=use_1stage,
|
||||
)
|
||||
assert result is not None
|
||||
return result[0], result[1], result[2]
|
||||
|
||||
|
||||
def _rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_fake(
|
||||
input_: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
epsilon: float,
|
||||
group_size: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
hidden_dim = input_.shape[-1]
|
||||
num_groups = hidden_dim // group_size
|
||||
quant_out = torch.empty(input_.shape, dtype=FP8_DTYPE, device=input_.device)
|
||||
residual_out = torch.empty_like(residual)
|
||||
scale_out = torch.empty(
|
||||
input_.shape[:-1] + (num_groups,),
|
||||
dtype=torch.float32,
|
||||
device=input_.device,
|
||||
)
|
||||
return quant_out, residual_out, scale_out
|
||||
|
||||
|
||||
def _rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_impl(
|
||||
input_: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
epsilon: float,
|
||||
group_size: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Fused AllReduce + add-RMSNorm + per-group FP8 quant + bf16 normed act.
|
||||
|
||||
Wraps the same AITER launcher as ``_rocm_aiter_fused_allreduce_rmsnorm_
|
||||
quant_per_group_impl`` with ``emit_bf16=True``, emitting the pre-quant
|
||||
bf16/fp16 normed activation for a parallel consumer (DeepSeek V3.2 sparse
|
||||
indexer ``wk_weights_proj``).
|
||||
"""
|
||||
aiter_ar = rocm_aiter_ops.get_aiter_allreduce()
|
||||
assert aiter_ar is not None, "aiter allreduce must be initialized"
|
||||
|
||||
total_bytes = input_.numel() * input_.element_size()
|
||||
hidden_dim = input_.shape[-1]
|
||||
token_num = input_.shape[0]
|
||||
if input_.dtype in (torch.bfloat16, torch.float16):
|
||||
pack_size = 16 // input_.element_size()
|
||||
hidden_ok = hidden_dim % pack_size == 0 and hidden_dim // pack_size <= 1024
|
||||
else:
|
||||
hidden_ok = False
|
||||
token_ok = token_num <= 80
|
||||
world_size = aiter_ar.world_size
|
||||
full_nvlink = aiter_ar.fully_connected
|
||||
|
||||
if world_size == 2:
|
||||
size_ok = True
|
||||
elif full_nvlink and world_size <= 4:
|
||||
size_ok = total_bytes < 256 * 1024
|
||||
elif full_nvlink and world_size <= 8:
|
||||
size_ok = total_bytes < 128 * 1024
|
||||
else:
|
||||
size_ok = False
|
||||
|
||||
use_1stage = hidden_ok and token_ok and size_ok
|
||||
|
||||
result = aiter_ar.fused_ar_rms_per_group_quant(
|
||||
input_,
|
||||
residual,
|
||||
w=weight,
|
||||
eps=epsilon,
|
||||
group_size=group_size,
|
||||
registered=torch.cuda.is_current_stream_capturing(),
|
||||
use_1stage=use_1stage,
|
||||
emit_bf16=True,
|
||||
)
|
||||
assert result is not None
|
||||
assert len(result) == 4, "emit_bf16=True must return four tensors from aiter"
|
||||
return result[0], result[1], result[2], result[3]
|
||||
|
||||
|
||||
def _rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_fake(
|
||||
input_: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
epsilon: float,
|
||||
group_size: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
hidden_dim = input_.shape[-1]
|
||||
num_groups = hidden_dim // group_size
|
||||
quant_out = torch.empty(input_.shape, dtype=FP8_DTYPE, device=input_.device)
|
||||
residual_out = torch.empty_like(residual)
|
||||
scale_out = torch.empty(
|
||||
input_.shape[:-1] + (num_groups,),
|
||||
dtype=torch.float32,
|
||||
device=input_.device,
|
||||
)
|
||||
bf16_norm_out = torch.empty_like(input_)
|
||||
return quant_out, residual_out, scale_out, bf16_norm_out
|
||||
|
||||
|
||||
def _rocm_aiter_per_tensor_quant_impl(
|
||||
x: torch.Tensor,
|
||||
quant_dtype: torch.dtype,
|
||||
@@ -1838,6 +1999,18 @@ class rocm_aiter_ops:
|
||||
fake_impl=_rocm_aiter_fused_allreduce_rmsnorm_fake,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="rocm_aiter_fused_allreduce_rmsnorm_quant_per_group",
|
||||
op_func=(_rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_impl),
|
||||
fake_impl=(_rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_fake),
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm", # noqa: E501
|
||||
op_func=_rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_impl, # noqa: E501
|
||||
fake_impl=_rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_fake, # noqa: E501
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="fused_mla_dual_rms_norm",
|
||||
op_func=_fused_mla_dual_rms_norm_impl,
|
||||
@@ -1892,6 +2065,29 @@ class rocm_aiter_ops:
|
||||
def get_fused_allreduce_rmsnorm_op() -> OpOverload:
|
||||
return torch.ops.vllm.rocm_aiter_fused_allreduce_rmsnorm.default
|
||||
|
||||
@staticmethod
|
||||
def get_fused_allreduce_rmsnorm_quant_per_group_op() -> OpOverload:
|
||||
return torch.ops.vllm.rocm_aiter_fused_allreduce_rmsnorm_quant_per_group.default
|
||||
|
||||
@staticmethod
|
||||
def get_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_op() -> OpOverload: # noqa: E501
|
||||
return torch.ops.vllm.rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm.default # noqa: E501
|
||||
|
||||
# TODO(frida-andersson): drop once vLLM pins AITER >= 0.1.14 (ROCm/aiter#2823).
|
||||
@classmethod
|
||||
def has_fused_allreduce_rmsnorm_quant_per_group(cls) -> bool:
|
||||
"""True if the running AITER build exposes the per-group AR+RMS+quant
|
||||
kernel (added in ROCm/aiter PR #2823).
|
||||
|
||||
The pattern registration in ``RocmAiterAllReduceFusionPass`` keys off
|
||||
this so vLLM degrades to the AR+RMS-only fusion when run against an
|
||||
older aiter that lacks the per-group launcher.
|
||||
"""
|
||||
aiter_ar = cls.get_aiter_allreduce()
|
||||
return aiter_ar is not None and hasattr(
|
||||
aiter_ar, "fused_ar_rms_per_group_quant"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_fused_mla_dual_rms_norm_op() -> OpOverload:
|
||||
return torch.ops.vllm.fused_mla_dual_rms_norm.default
|
||||
|
||||
@@ -26,6 +26,9 @@ from vllm.distributed.parallel_state import (
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
QuantKey,
|
||||
ScaleDesc,
|
||||
kFp8StaticTensorSym,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
@@ -1109,7 +1112,7 @@ class AiterAllreduceFusedRMSNormPattern(BasePattern, VllmPatternReplacement):
|
||||
allreduce = self.FUSED_AR_RMSNORM_OP(
|
||||
input_=input,
|
||||
residual=residual,
|
||||
weight=weight,
|
||||
weight=weight.to(input.dtype),
|
||||
epsilon=self.epsilon,
|
||||
)
|
||||
return allreduce[0], allreduce[1]
|
||||
@@ -1155,7 +1158,7 @@ class AiterAllreduceFusedAddRMSNormPattern(BasePattern, VllmPatternReplacement):
|
||||
allreduce = self.FUSED_AR_RMSNORM_OP(
|
||||
input_=input,
|
||||
residual=residual,
|
||||
weight=weight,
|
||||
weight=weight.to(input.dtype),
|
||||
epsilon=self.epsilon,
|
||||
)
|
||||
return allreduce[0], allreduce[1]
|
||||
@@ -1163,6 +1166,279 @@ class AiterAllreduceFusedAddRMSNormPattern(BasePattern, VllmPatternReplacement):
|
||||
return _replacement
|
||||
|
||||
|
||||
class AiterAllreduceFusedRMSNormGroupQuantFP8Pattern(
|
||||
BasePattern, VllmPatternReplacement
|
||||
):
|
||||
"""Fuse AllReduce + RMSNorm + per-group FP8 quant into a single AITER
|
||||
custom op.
|
||||
|
||||
Matches the AR-side analogue of ``AiterRMSFp8GroupQuantPattern`` in
|
||||
``rocm_aiter_fusion.py``: ``all_reduce -> rms_norm -> group_fp8_quant``
|
||||
fans out into ``rocm_aiter_fused_allreduce_rmsnorm_quant_per_group``.
|
||||
|
||||
Without this pattern, ``RocmAiterAllReduceFusionPass`` would fuse the
|
||||
``all_reduce + rms_norm`` half (PR #41825 wires that), but the trailing
|
||||
``rocm_aiter_group_fp8_quant`` would still launch as a separate kernel.
|
||||
That standalone quant accounts for ~535us / decode step on DSv3.2 MI355X
|
||||
TP4 -- this pattern eliminates it by absorbing the quant into the AR
|
||||
epilogue. Group size 128 matches the FP8 block-scaled MM kernel used by
|
||||
DSv3.2's linear weights.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
epsilon: float,
|
||||
dtype: torch.dtype,
|
||||
device: str | None,
|
||||
group_size: int = 128,
|
||||
) -> None:
|
||||
super().__init__(dtype, device)
|
||||
self.epsilon = epsilon
|
||||
self.dtype = dtype
|
||||
self.group_size = group_size
|
||||
self.FUSED_AR_RMS_QUANT_OP = (
|
||||
rocm_aiter_ops.get_fused_allreduce_rmsnorm_quant_per_group_op()
|
||||
)
|
||||
self.quant_dtype = current_platform.fp8_dtype()
|
||||
self.quant_matcher = MatcherQuantFP8(
|
||||
QuantKey(
|
||||
dtype=self.quant_dtype,
|
||||
scale=ScaleDesc(torch.float32, False, GroupShape(1, group_size)),
|
||||
symmetric=True,
|
||||
),
|
||||
match_rocm_aiter=True,
|
||||
)
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
# input, weight; hidden dim must be a group_size multiple so the
|
||||
# group quant matcher's example trace is well-defined.
|
||||
return [self.empty(5, self.group_size), self.empty(self.group_size)]
|
||||
|
||||
@property
|
||||
def pattern(self):
|
||||
def _pattern(
|
||||
input: torch.Tensor, weight: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
allreduce_output = tensor_model_parallel_all_reduce(input)
|
||||
rms = vllm.ir.ops.rms_norm(allreduce_output, weight, self.epsilon)
|
||||
quant, scale = self.quant_matcher(rms)
|
||||
return quant, scale
|
||||
|
||||
return _pattern
|
||||
|
||||
@property
|
||||
def replacement(self):
|
||||
def _replacement(
|
||||
input: torch.Tensor, weight: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
residual = torch.zeros_like(input)
|
||||
result = self.FUSED_AR_RMS_QUANT_OP(
|
||||
input_=input,
|
||||
residual=residual,
|
||||
weight=weight.to(input.dtype),
|
||||
epsilon=self.epsilon,
|
||||
group_size=self.group_size,
|
||||
)
|
||||
# quant_out, scale_out (residual is unused on the no-add path,
|
||||
# mirroring how AiterAllreduceFusedRMSNormPattern drops the
|
||||
# residual output)
|
||||
return result[0], result[2]
|
||||
|
||||
return _replacement
|
||||
|
||||
|
||||
class AiterAllreduceFusedAddRMSNormGroupQuantFP8Pattern(
|
||||
BasePattern, VllmPatternReplacement
|
||||
):
|
||||
"""``fused_add`` variant of ``AiterAllreduceFusedRMSNormGroupQuantFP8Pattern``.
|
||||
|
||||
Targets the dominant DSv3.2-style post-attention / post-MLP path:
|
||||
``all_reduce -> fused_add_rms_norm -> group_fp8_quant``. Returns the
|
||||
FP8 quant output, the residual carry-over, and the per-group scale.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
epsilon: float,
|
||||
dtype: torch.dtype,
|
||||
device: str | None,
|
||||
group_size: int = 128,
|
||||
) -> None:
|
||||
super().__init__(dtype, device)
|
||||
self.epsilon = epsilon
|
||||
self.dtype = dtype
|
||||
self.group_size = group_size
|
||||
self.FUSED_AR_RMS_QUANT_OP = (
|
||||
rocm_aiter_ops.get_fused_allreduce_rmsnorm_quant_per_group_op()
|
||||
)
|
||||
self.quant_dtype = current_platform.fp8_dtype()
|
||||
self.quant_matcher = MatcherQuantFP8(
|
||||
QuantKey(
|
||||
dtype=self.quant_dtype,
|
||||
scale=ScaleDesc(torch.float32, False, GroupShape(1, group_size)),
|
||||
symmetric=True,
|
||||
),
|
||||
match_rocm_aiter=True,
|
||||
)
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
# residual, input, weight
|
||||
return [
|
||||
self.empty(5, self.group_size),
|
||||
self.empty(5, self.group_size),
|
||||
self.empty(self.group_size),
|
||||
]
|
||||
|
||||
@property
|
||||
def pattern(self):
|
||||
def _pattern(
|
||||
residual: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
allreduce_output = tensor_model_parallel_all_reduce(input)
|
||||
rms, residual_out = vllm.ir.ops.fused_add_rms_norm(
|
||||
allreduce_output, residual, weight, self.epsilon
|
||||
)
|
||||
quant, scale = self.quant_matcher(rms)
|
||||
return quant, scale, residual_out
|
||||
|
||||
return _pattern
|
||||
|
||||
@property
|
||||
def replacement(self):
|
||||
def _replacement(
|
||||
residual: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
result = self.FUSED_AR_RMS_QUANT_OP(
|
||||
input_=input,
|
||||
residual=residual,
|
||||
weight=weight.to(input.dtype),
|
||||
epsilon=self.epsilon,
|
||||
group_size=self.group_size,
|
||||
)
|
||||
# quant_out, scale_out, residual_out
|
||||
return result[0], result[2], result[1]
|
||||
|
||||
return _replacement
|
||||
|
||||
|
||||
class AiterAllreduceFusedAddRMSNormGroupQuantWithIndexerPattern(
|
||||
BasePattern, VllmPatternReplacement
|
||||
):
|
||||
"""Indexer-fan-out variant of ``AiterAllreduceFusedAddRMSNormGroupQuantFP8Pattern``.
|
||||
|
||||
Targets the DSv3.2 post-attention / post-MLP path where the post-AR normed
|
||||
activation has two consumers: a per-group FP8 quant for ``fused_qkv_a_proj``
|
||||
*and* a bf16 ``rocm_unquantized_gemm`` for the indexer ``wk_weights_proj``.
|
||||
The single-consumer pattern above cannot fire when this fan-out is present,
|
||||
so without this pattern the standalone FP8 quant kernel survives unfused
|
||||
(~535us / decode step on DSv3.2 MI355X TP4).
|
||||
|
||||
Lowers to ``rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm``
|
||||
(the ``emit_bf16=True`` variant of the AR+RMS+QUANT launcher, which returns
|
||||
FP8 quant + scales + bf16 normed activations in one kernel) and rewires the
|
||||
indexer GEMM onto the emitted bf16 norm output. The RMS output is also a
|
||||
graph output in DSv3.2's residual carry; it is returned as a pattern output
|
||||
so the matcher can substitute the bf16 norm in its place.
|
||||
|
||||
The trailing FP8 group-quant is matched via ``MatcherQuantFP8`` (consistent
|
||||
with the sibling patterns above), which traces both ``QuantFP8.forward_hip``
|
||||
and ``forward_native`` paths and so matches whichever op the call site
|
||||
lowers to (``vllm.triton_per_token_group_quant_fp8`` or
|
||||
``vllm.rocm_aiter_group_fp8_quant``).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
epsilon: float,
|
||||
dtype: torch.dtype,
|
||||
device: str | None,
|
||||
group_size: int = 128,
|
||||
) -> None:
|
||||
super().__init__(dtype, device)
|
||||
self.epsilon = epsilon
|
||||
self.dtype = dtype
|
||||
self.group_size = group_size
|
||||
self.FUSED_AR_RMS_QUANT_BF16_OP = (
|
||||
rocm_aiter_ops.get_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_op() # noqa: E501
|
||||
)
|
||||
self.quant_dtype = current_platform.fp8_dtype()
|
||||
self.quant_matcher = MatcherQuantFP8(
|
||||
QuantKey(
|
||||
dtype=self.quant_dtype,
|
||||
scale=ScaleDesc(torch.float32, False, GroupShape(1, group_size)),
|
||||
symmetric=True,
|
||||
),
|
||||
match_rocm_aiter=True,
|
||||
)
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
h = self.group_size
|
||||
indexer_out = 8
|
||||
return [
|
||||
self.empty(5, h),
|
||||
self.empty(5, h),
|
||||
self.empty(h),
|
||||
self.empty(indexer_out, h),
|
||||
]
|
||||
|
||||
@property
|
||||
def pattern(self):
|
||||
eps = self.epsilon
|
||||
|
||||
def _pattern(
|
||||
residual: torch.Tensor,
|
||||
input_: torch.Tensor,
|
||||
norm_weight: torch.Tensor,
|
||||
indexer_weight: torch.Tensor,
|
||||
) -> tuple[
|
||||
torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor
|
||||
]:
|
||||
ar_out = tensor_model_parallel_all_reduce(input_)
|
||||
rms, res_out = vllm.ir.ops.fused_add_rms_norm(
|
||||
ar_out, residual, norm_weight, eps
|
||||
)
|
||||
q, s = self.quant_matcher(rms)
|
||||
idx = torch.ops.vllm.rocm_unquantized_gemm(rms, indexer_weight)
|
||||
return q, s, res_out, idx, rms
|
||||
|
||||
return _pattern
|
||||
|
||||
@property
|
||||
def replacement(self):
|
||||
gs = self.group_size
|
||||
eps = self.epsilon
|
||||
|
||||
def _replacement(
|
||||
residual: torch.Tensor,
|
||||
input_: torch.Tensor,
|
||||
norm_weight: torch.Tensor,
|
||||
indexer_weight: torch.Tensor,
|
||||
) -> tuple[
|
||||
torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor
|
||||
]:
|
||||
fused = self.FUSED_AR_RMS_QUANT_BF16_OP(
|
||||
input_=input_,
|
||||
residual=residual,
|
||||
weight=norm_weight.to(input_.dtype),
|
||||
epsilon=eps,
|
||||
group_size=gs,
|
||||
)
|
||||
quant_out, residual_out, scale_out, bf16_norm = (
|
||||
fused[0],
|
||||
fused[1],
|
||||
fused[2],
|
||||
fused[3],
|
||||
)
|
||||
idx = torch.ops.vllm.rocm_unquantized_gemm(bf16_norm, indexer_weight)
|
||||
return quant_out, scale_out, residual_out, idx, bf16_norm
|
||||
|
||||
return _replacement
|
||||
|
||||
|
||||
class RocmAiterAllReduceFusionPass(VllmFusionPatternMatcherPass):
|
||||
def __init__(self, config: VllmConfig) -> None:
|
||||
super().__init__(config, "rocm_aiter_allreduce_fusion_pass")
|
||||
@@ -1232,7 +1508,52 @@ class RocmAiterAllReduceFusionPass(VllmFusionPatternMatcherPass):
|
||||
config.scheduler_config.max_num_batched_tokens,
|
||||
)
|
||||
|
||||
# Only register the AR+RMS+per-group-FP8-quant patterns when the
|
||||
# running aiter exposes the kernel. Older aiter builds (pre PR #2823)
|
||||
# fall back to the AR+RMS-only fusion paired with PR #41825's
|
||||
# standalone RMS+quant fusion -- still correct, just leaves the
|
||||
# post-AR quant as a standalone kernel.
|
||||
supports_per_group_quant = (
|
||||
rocm_aiter_ops.has_fused_allreduce_rmsnorm_quant_per_group()
|
||||
)
|
||||
if not supports_per_group_quant:
|
||||
logger.warning_once(
|
||||
"AITER AR+RMS+per-group-FP8-quant fusion disabled: aiter "
|
||||
"build is missing 'fused_ar_rms_per_group_quant'. Upgrade "
|
||||
"aiter past PR #2823 to enable the trailing per-group "
|
||||
"FP8 quant fusion."
|
||||
)
|
||||
|
||||
for epsilon in [1e-5, 1e-6]:
|
||||
# Quant-fused variants must register first so the pattern matcher
|
||||
# tries them before the AR+RMS-only variants. Otherwise the
|
||||
# AR+RMS-only fusion runs first and consumes the all_reduce node,
|
||||
# leaving the trailing quant op stranded as an unfused kernel.
|
||||
# Register larger subgraphs first (DeepSeek indexer fan-out, then
|
||||
# quant-only AR+RMS+quant, then AR+RMS-only).
|
||||
if supports_per_group_quant:
|
||||
self.register(
|
||||
AiterAllreduceFusedAddRMSNormGroupQuantWithIndexerPattern(
|
||||
epsilon,
|
||||
self.model_dtype,
|
||||
self.device,
|
||||
)
|
||||
)
|
||||
self.register(
|
||||
AiterAllreduceFusedRMSNormGroupQuantFP8Pattern(
|
||||
epsilon,
|
||||
self.model_dtype,
|
||||
self.device,
|
||||
)
|
||||
)
|
||||
self.register(
|
||||
AiterAllreduceFusedAddRMSNormGroupQuantFP8Pattern(
|
||||
epsilon,
|
||||
self.model_dtype,
|
||||
self.device,
|
||||
)
|
||||
)
|
||||
|
||||
self.register(
|
||||
AiterAllreduceFusedRMSNormPattern(
|
||||
epsilon,
|
||||
|
||||
Reference in New Issue
Block a user