[ROCm][Compile] Fuse AR + RMSNorm + per-group FP8 quant (+ DSv3.2 indexer fan-out) (#42864)
Signed-off-by: Markus Hartikainen <markus.hartikainen@amd.com> Co-authored-by: Frida Andersson <fanderss@amd.com>
This commit is contained in:
co-authored by
Frida Andersson
parent
5b3807e862
commit
80e2c4462d
@@ -189,6 +189,118 @@ class TestAllReduceRMSNormStaticQuantFP8Model(torch.nn.Module):
|
||||
]
|
||||
|
||||
|
||||
class TestAiterAllReduceRMSNormGroupQuantFP8Model(torch.nn.Module):
|
||||
"""Exercises the new ROCm AITER AR+RMS+per-group-FP8-quant patterns.
|
||||
|
||||
Four ``rms_norm`` sites that together hit every pattern registered by
|
||||
``RocmAiterAllReduceFusionPass`` for the per-group FP8 quant path:
|
||||
|
||||
* ``norm[0]``: ``all_reduce -> rms_norm -> group_fp8_quant`` (no residual)
|
||||
-> ``AiterAllreduceFusedRMSNormGroupQuantFP8Pattern``
|
||||
* ``norm[1]``: ``all_reduce -> fused_add_rms_norm -> group_fp8_quant``
|
||||
(single ``rms`` consumer)
|
||||
-> ``AiterAllreduceFusedAddRMSNormGroupQuantFP8Pattern``
|
||||
* ``norm[2..3]``: ``all_reduce -> fused_add_rms_norm
|
||||
-> (group_fp8_quant + rocm_unquantized_gemm)`` (two ``rms`` consumers,
|
||||
modeling the DSv3.2 indexer fan-out)
|
||||
-> ``AiterAllreduceFusedAddRMSNormGroupQuantWithIndexerPattern``
|
||||
|
||||
The chain feeds the next AllReduce by dequantizing the FP8 output (FP8
|
||||
cast back to bf16 multiplied by the per-group scale), which is enough to
|
||||
keep the matmul chain bf16 without depending on a real FP8 block-scaled
|
||||
GEMM kernel.
|
||||
"""
|
||||
|
||||
quant_group_size = 128
|
||||
indexer_out_dim = 8
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size=128,
|
||||
token_num=16,
|
||||
eps=1e-6,
|
||||
dtype: torch.dtype = torch.bfloat16,
|
||||
use_triton_quant: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
self.eps = eps
|
||||
self.use_triton_quant = use_triton_quant
|
||||
assert hidden_size % self.quant_group_size == 0, (
|
||||
f"hidden_size ({hidden_size}) must be a multiple of "
|
||||
f"quant_group_size ({self.quant_group_size}) for per-group FP8 quant"
|
||||
)
|
||||
self.norm = [RMSNorm(hidden_size, eps) for _ in range(4)]
|
||||
self.w = [torch.rand(hidden_size, hidden_size, dtype=dtype) for _ in range(3)]
|
||||
self.indexer_w = [
|
||||
torch.rand(self.indexer_out_dim, hidden_size, dtype=dtype) for _ in range(2)
|
||||
]
|
||||
|
||||
def _group_quant(self, rms: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if self.use_triton_quant:
|
||||
return torch.ops.vllm.triton_per_token_group_quant_fp8(
|
||||
rms, self.quant_group_size
|
||||
)
|
||||
return torch.ops.vllm.rocm_aiter_group_fp8_quant.default(
|
||||
rms, self.quant_group_size
|
||||
)
|
||||
|
||||
def _dequantize_to_bf16(
|
||||
self, q: torch.Tensor, s: torch.Tensor, ref: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
# Broadcast the per-group scale across each group of `quant_group_size`
|
||||
# so we can chain the FP8 output back into a bf16 matmul. This avoids
|
||||
# depending on a real FP8 block-scaled GEMM kernel in the test.
|
||||
s_full = s.repeat_interleave(self.quant_group_size, dim=-1).to(ref.dtype)
|
||||
return q.to(ref.dtype) * s_full
|
||||
|
||||
def forward(self, hidden_states):
|
||||
z = torch.relu(hidden_states)
|
||||
x = resid = tensor_model_parallel_all_reduce(z)
|
||||
rms = self.norm[0](x)
|
||||
q0, s0 = self._group_quant(rms)
|
||||
y = self._dequantize_to_bf16(q0, s0, rms)
|
||||
|
||||
z2 = torch.mm(y, self.w[0])
|
||||
x2 = tensor_model_parallel_all_reduce(z2)
|
||||
rms2, resid = self.norm[1](x2, resid)
|
||||
q1, s1 = self._group_quant(rms2)
|
||||
y2 = self._dequantize_to_bf16(q1, s1, rms2)
|
||||
|
||||
z3 = torch.mm(y2, self.w[1])
|
||||
x3 = tensor_model_parallel_all_reduce(z3)
|
||||
rms3, resid = self.norm[2](x3, resid)
|
||||
q2, s2 = self._group_quant(rms3)
|
||||
# Second consumer of ``rms3``: forces the with-indexer pattern.
|
||||
idx2 = torch.ops.vllm.rocm_unquantized_gemm(rms3, self.indexer_w[0], None)
|
||||
y3 = self._dequantize_to_bf16(q2, s2, rms3)
|
||||
|
||||
z4 = torch.mm(y3, self.w[2])
|
||||
x4 = tensor_model_parallel_all_reduce(z4)
|
||||
rms4, resid = self.norm[3](x4, resid)
|
||||
q3, s3 = self._group_quant(rms4)
|
||||
# Second consumer of ``rms4``: forces the with-indexer pattern.
|
||||
idx3 = torch.ops.vllm.rocm_unquantized_gemm(rms4, self.indexer_w[1], None)
|
||||
y4 = self._dequantize_to_bf16(q3, s3, rms4)
|
||||
return y4, idx2, idx3
|
||||
|
||||
def ops_in_model_before(self):
|
||||
return [
|
||||
torch.ops.vllm.all_reduce.default,
|
||||
(
|
||||
torch.ops.vllm.triton_per_token_group_quant_fp8.default
|
||||
if self.use_triton_quant
|
||||
else torch.ops.vllm.rocm_aiter_group_fp8_quant.default
|
||||
),
|
||||
]
|
||||
|
||||
def ops_in_model_after(self):
|
||||
return [
|
||||
rocm_aiter_ops.get_fused_allreduce_rmsnorm_quant_per_group_op(),
|
||||
rocm_aiter_ops.get_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_op(), # noqa: E501
|
||||
]
|
||||
|
||||
|
||||
class TestAllReduceFusedAddRMSNormStaticQuantFP4Model(torch.nn.Module):
|
||||
def __init__(
|
||||
self, hidden_size=16, token_num=16, eps=1e-6, dtype: torch.dtype = torch.float16
|
||||
@@ -463,3 +575,167 @@ def all_reduce_fusion_pass_on_test_model(
|
||||
assert fused_nodes
|
||||
assert all(n.kwargs.get("weight_bias") == 1.0 for n in fused_nodes)
|
||||
del all_reduce_fusion_pass
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize("use_triton_quant", [True, False])
|
||||
@pytest.mark.parametrize("batch_size", [8])
|
||||
@pytest.mark.parametrize("seq_len", [8])
|
||||
@pytest.mark.parametrize("hidden_size", [128])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("enable_rms_norm_custom_op", [True, False])
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="ROCm AITER AR+RMS+per-group-FP8-quant fusion is ROCm-only",
|
||||
)
|
||||
@pytest.mark.skipif(not IS_AITER_FOUND, reason="aiter is not found")
|
||||
def test_rocm_aiter_all_reduce_rmsnorm_group_quant_fp8_fusion_pass_replace(
|
||||
batch_size: int,
|
||||
seq_len: int,
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
enable_rms_norm_custom_op: bool,
|
||||
use_triton_quant: bool,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""Sibling of ``test_all_reduce_fusion_pass_replace`` for the new
|
||||
ROCm AITER AR+RMS+per-group-FP8-quant fusion patterns.
|
||||
|
||||
Validates the three new ``VllmPatternReplacement`` patterns added to
|
||||
``RocmAiterAllReduceFusionPass``:
|
||||
|
||||
* ``AiterAllreduceFusedRMSNormGroupQuantFP8Pattern`` (no-residual)
|
||||
* ``AiterAllreduceFusedAddRMSNormGroupQuantFP8Pattern`` (with-residual,
|
||||
single ``rms`` consumer)
|
||||
* ``AiterAllreduceFusedAddRMSNormGroupQuantWithIndexerPattern`` (with-
|
||||
residual, DSv3.2 indexer fan-out; parametrized over both
|
||||
``triton_per_token_group_quant_fp8`` and ``rocm_aiter_group_fp8_quant``
|
||||
producers).
|
||||
"""
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
if not rocm_aiter_ops.has_fused_allreduce_rmsnorm_quant_per_group():
|
||||
pytest.skip(
|
||||
"aiter build is missing 'fused_ar_rms_per_group_quant' (needs "
|
||||
"ROCm/aiter PR #2823); the new patterns aren't registered."
|
||||
)
|
||||
|
||||
num_processes = 2
|
||||
|
||||
def run_torch_spawn(fn, nprocs):
|
||||
torch.multiprocessing.spawn(
|
||||
fn,
|
||||
args=(
|
||||
num_processes,
|
||||
TestAiterAllReduceRMSNormGroupQuantFP8Model,
|
||||
batch_size,
|
||||
seq_len,
|
||||
hidden_size,
|
||||
dtype,
|
||||
enable_rms_norm_custom_op,
|
||||
use_triton_quant,
|
||||
monkeypatch,
|
||||
),
|
||||
nprocs=nprocs,
|
||||
)
|
||||
|
||||
run_torch_spawn(rocm_aiter_group_quant_fusion_pass_on_test_model, num_processes)
|
||||
|
||||
|
||||
def rocm_aiter_group_quant_fusion_pass_on_test_model(
|
||||
local_rank: int,
|
||||
world_size: int,
|
||||
test_model_cls: torch.nn.Module,
|
||||
batch_size: int,
|
||||
seq_len: int,
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
enable_rms_norm_custom_op: bool,
|
||||
use_triton_quant: bool,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
set_random_seed(0)
|
||||
|
||||
device = torch.device(f"{DEVICE_TYPE}:{local_rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
torch.set_default_device(device)
|
||||
torch.set_default_dtype(dtype)
|
||||
|
||||
update_environment_variables(
|
||||
{
|
||||
"RANK": str(local_rank),
|
||||
"LOCAL_RANK": str(local_rank),
|
||||
"WORLD_SIZE": str(world_size),
|
||||
"MASTER_ADDR": "localhost",
|
||||
"MASTER_PORT": "12345",
|
||||
"VLLM_ROCM_USE_AITER": "1",
|
||||
}
|
||||
)
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
init_distributed_environment()
|
||||
|
||||
custom_ops = []
|
||||
if enable_rms_norm_custom_op:
|
||||
custom_ops.append("+rms_norm")
|
||||
# ``triton_per_token_group_quant_fp8`` is emitted by ``QuantFP8.forward_hip``
|
||||
# only when QuantFP8 is enabled as a custom op (and ``use_triton=True`` at
|
||||
# the call site). The patterns in this PR are robust to both Triton and
|
||||
# rocm_aiter forms; we always enable +quant_fp8 so the matcher's example
|
||||
# trace finds the same form the test model uses.
|
||||
custom_ops.append("+quant_fp8")
|
||||
|
||||
vllm_config = VllmConfig(
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE, custom_ops=custom_ops
|
||||
)
|
||||
)
|
||||
vllm_config.compilation_config.pass_config = PassConfig(
|
||||
fuse_allreduce_rms=True, eliminate_noops=True
|
||||
)
|
||||
vllm_config.device_config = DeviceConfig(device=torch.device(DEVICE_TYPE))
|
||||
vllm_config.parallel_config.rank = local_rank
|
||||
|
||||
model_name = "RedHatAI/Llama-3.2-1B-Instruct-FP8"
|
||||
vllm_config.model_config = ModelConfig(
|
||||
model=model_name, trust_remote_code=True, dtype=dtype, seed=42
|
||||
)
|
||||
with set_current_vllm_config(vllm_config):
|
||||
initialize_model_parallel(tensor_model_parallel_size=world_size)
|
||||
all_reduce_fusion_pass = RocmAiterAllReduceFusionPass(vllm_config)
|
||||
noop_pass = NoOpEliminationPass(vllm_config)
|
||||
func_pass = FixFunctionalizationPass(vllm_config)
|
||||
cleanup_pass = PostCleanupPass(vllm_config)
|
||||
|
||||
backend = TestBackend(
|
||||
noop_pass, all_reduce_fusion_pass, func_pass, cleanup_pass
|
||||
)
|
||||
|
||||
token_num = batch_size * seq_len
|
||||
model = test_model_cls(
|
||||
hidden_size, token_num, dtype=dtype, use_triton_quant=use_triton_quant
|
||||
)
|
||||
|
||||
hidden_states = torch.randn((token_num, hidden_size), requires_grad=False)
|
||||
|
||||
compiled_model = torch.compile(model, backend=backend)
|
||||
compiled_model(hidden_states)
|
||||
|
||||
results_unfused = model(hidden_states)
|
||||
results_fused = compiled_model(hidden_states)
|
||||
# The fused per-group AR+RMS+QUANT op is bit-equivalent to the unfused
|
||||
# chain modulo the small AllReduce + RMSNorm reordering inside aiter.
|
||||
# Per-group FP8 quant introduces step noise <=1 per group; use the
|
||||
# same tolerance as the sibling FP8 static test.
|
||||
torch.testing.assert_close(results_unfused, results_fused, atol=1e-2, rtol=1e-2)
|
||||
|
||||
# Four pattern firings: norm[0] (no-add quant), norm[1] (add quant,
|
||||
# single ``rms`` consumer), norm[2..3] (add quant + indexer fan-out).
|
||||
assert all_reduce_fusion_pass.matched_count == 4, (
|
||||
f"{all_reduce_fusion_pass.matched_count=}"
|
||||
)
|
||||
backend.check_before_ops(model.ops_in_model_before(), fully_replaced=False)
|
||||
backend.check_after_ops(model.ops_in_model_after())
|
||||
del all_reduce_fusion_pass
|
||||
|
||||
Reference in New Issue
Block a user