[ROCm] Fix AITER all-reduce fusion coverage (#50802)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
@@ -43,6 +43,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
|||||||
kFp8StaticTensorSym,
|
kFp8StaticTensorSym,
|
||||||
)
|
)
|
||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
|
from vllm.utils.network_utils import get_open_port
|
||||||
from vllm.utils.system_utils import update_environment_variables
|
from vllm.utils.system_utils import update_environment_variables
|
||||||
from vllm.utils.torch_utils import set_random_seed
|
from vllm.utils.torch_utils import set_random_seed
|
||||||
|
|
||||||
@@ -257,10 +258,9 @@ class TestAiterAllReduceRMSNormGroupQuantFP8Model(torch.nn.Module):
|
|||||||
modeling the DSv3.2 indexer fan-out)
|
modeling the DSv3.2 indexer fan-out)
|
||||||
-> ``AiterAllreduceFusedAddRMSNormGroupQuantWithIndexerPattern``
|
-> ``AiterAllreduceFusedAddRMSNormGroupQuantWithIndexerPattern``
|
||||||
|
|
||||||
The chain feeds the next AllReduce by dequantizing the FP8 output (FP8
|
Each site starts from the same bf16 activation and is returned separately.
|
||||||
cast back to bf16 multiplied by the per-group scale), which is enough to
|
This avoids compounding four quantize/dequantize errors while still testing
|
||||||
keep the matmul chain bf16 without depending on a real FP8 block-scaled
|
all four pattern firings without depending on a real FP8 block-scaled GEMM.
|
||||||
GEMM kernel.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
quant_group_size = 128
|
quant_group_size = 128
|
||||||
@@ -276,6 +276,7 @@ class TestAiterAllReduceRMSNormGroupQuantFP8Model(torch.nn.Module):
|
|||||||
super().__init__()
|
super().__init__()
|
||||||
self.hidden_size = hidden_size
|
self.hidden_size = hidden_size
|
||||||
self.eps = eps
|
self.eps = eps
|
||||||
|
self.dtype = dtype
|
||||||
assert hidden_size % self.quant_group_size == 0, (
|
assert hidden_size % self.quant_group_size == 0, (
|
||||||
f"hidden_size ({hidden_size}) must be a multiple of "
|
f"hidden_size ({hidden_size}) must be a multiple of "
|
||||||
f"quant_group_size ({self.quant_group_size}) for per-group FP8 quant"
|
f"quant_group_size ({self.quant_group_size}) for per-group FP8 quant"
|
||||||
@@ -291,44 +292,39 @@ class TestAiterAllReduceRMSNormGroupQuantFP8Model(torch.nn.Module):
|
|||||||
rms, self.quant_group_size
|
rms, self.quant_group_size
|
||||||
)
|
)
|
||||||
|
|
||||||
def _dequantize_to_bf16(
|
def _dequantize_to_bf16(self, q: torch.Tensor, s: torch.Tensor) -> torch.Tensor:
|
||||||
self, q: torch.Tensor, s: torch.Tensor, ref: torch.Tensor
|
|
||||||
) -> torch.Tensor:
|
|
||||||
# Broadcast the per-group scale across each group of `quant_group_size`
|
# Broadcast the per-group scale across each group of `quant_group_size`
|
||||||
# so we can chain the FP8 output back into a bf16 matmul. This avoids
|
# to validate the quantized result without depending on a real FP8
|
||||||
# depending on a real FP8 block-scaled GEMM kernel in the test.
|
# block-scaled GEMM kernel in the test.
|
||||||
s_full = s.repeat_interleave(self.quant_group_size, dim=-1).to(ref.dtype)
|
s_full = s.repeat_interleave(self.quant_group_size, dim=-1).to(self.dtype)
|
||||||
return q.to(ref.dtype) * s_full
|
return q.to(self.dtype) * s_full
|
||||||
|
|
||||||
def forward(self, hidden_states):
|
def forward(self, hidden_states):
|
||||||
z = torch.relu(hidden_states)
|
z = torch.relu(hidden_states)
|
||||||
x = resid = tensor_model_parallel_all_reduce(z)
|
x0 = tensor_model_parallel_all_reduce(z)
|
||||||
rms = self.norm[0](x)
|
rms0 = self.norm[0](x0)
|
||||||
q0, s0 = self._group_quant(rms)
|
q0, s0 = self._group_quant(rms0)
|
||||||
y = self._dequantize_to_bf16(q0, s0, rms)
|
y0 = self._dequantize_to_bf16(q0, s0)
|
||||||
|
|
||||||
z2 = torch.mm(y, self.w[0])
|
x1 = tensor_model_parallel_all_reduce(torch.mm(z, self.w[0]))
|
||||||
x2 = tensor_model_parallel_all_reduce(z2)
|
rms1, resid1 = self.norm[1](x1, z.clone())
|
||||||
rms2, resid = self.norm[1](x2, resid)
|
q1, s1 = self._group_quant(rms1)
|
||||||
q1, s1 = self._group_quant(rms2)
|
y1 = self._dequantize_to_bf16(q1, s1)
|
||||||
y2 = self._dequantize_to_bf16(q1, s1, rms2)
|
|
||||||
|
|
||||||
z3 = torch.mm(y2, self.w[1])
|
x2 = tensor_model_parallel_all_reduce(torch.mm(z, self.w[1]))
|
||||||
x3 = tensor_model_parallel_all_reduce(z3)
|
rms2, resid2 = self.norm[2](x2, z.clone())
|
||||||
rms3, resid = self.norm[2](x3, resid)
|
q2, s2 = self._group_quant(rms2)
|
||||||
q2, s2 = self._group_quant(rms3)
|
# Second consumer of ``rms2``: forces the with-indexer pattern.
|
||||||
|
idx2 = torch.ops.vllm.rocm_unquantized_gemm(rms2, self.indexer_w[0], None)
|
||||||
|
y2 = self._dequantize_to_bf16(q2, s2)
|
||||||
|
|
||||||
|
x3 = tensor_model_parallel_all_reduce(torch.mm(z, self.w[2]))
|
||||||
|
rms3, resid3 = self.norm[3](x3, z.clone())
|
||||||
|
q3, s3 = self._group_quant(rms3)
|
||||||
# Second consumer of ``rms3``: forces the with-indexer pattern.
|
# Second consumer of ``rms3``: forces the with-indexer pattern.
|
||||||
idx2 = torch.ops.vllm.rocm_unquantized_gemm(rms3, self.indexer_w[0], None)
|
idx3 = torch.ops.vllm.rocm_unquantized_gemm(rms3, self.indexer_w[1], None)
|
||||||
y3 = self._dequantize_to_bf16(q2, s2, rms3)
|
y3 = self._dequantize_to_bf16(q3, s3)
|
||||||
|
return y0, y1, y2, y3, resid1, resid2, resid3, idx2, idx3
|
||||||
z4 = torch.mm(y3, self.w[2])
|
|
||||||
x4 = tensor_model_parallel_all_reduce(z4)
|
|
||||||
rms4, resid = self.norm[3](x4, resid)
|
|
||||||
q3, s3 = self._group_quant(rms4)
|
|
||||||
# Second consumer of ``rms4``: forces the with-indexer pattern.
|
|
||||||
idx3 = torch.ops.vllm.rocm_unquantized_gemm(rms4, self.indexer_w[1], None)
|
|
||||||
y4 = self._dequantize_to_bf16(q3, s3, rms4)
|
|
||||||
return y4, idx2, idx3
|
|
||||||
|
|
||||||
def ops_in_model_before(self):
|
def ops_in_model_before(self):
|
||||||
return [
|
return [
|
||||||
@@ -505,10 +501,12 @@ def test_all_reduce_fusion_pass_replace(
|
|||||||
)
|
)
|
||||||
|
|
||||||
def run_torch_spawn(fn, nprocs):
|
def run_torch_spawn(fn, nprocs):
|
||||||
|
master_port = get_open_port()
|
||||||
torch.multiprocessing.spawn(
|
torch.multiprocessing.spawn(
|
||||||
fn,
|
fn,
|
||||||
args=(
|
args=(
|
||||||
num_processes,
|
num_processes,
|
||||||
|
master_port,
|
||||||
test_model,
|
test_model,
|
||||||
batch_size,
|
batch_size,
|
||||||
seq_len,
|
seq_len,
|
||||||
@@ -529,6 +527,7 @@ def test_all_reduce_fusion_pass_replace(
|
|||||||
def all_reduce_fusion_pass_on_test_model(
|
def all_reduce_fusion_pass_on_test_model(
|
||||||
local_rank: int,
|
local_rank: int,
|
||||||
world_size: int,
|
world_size: int,
|
||||||
|
master_port: int,
|
||||||
test_model_cls: torch.nn.Module,
|
test_model_cls: torch.nn.Module,
|
||||||
batch_size: int,
|
batch_size: int,
|
||||||
seq_len: int,
|
seq_len: int,
|
||||||
@@ -553,7 +552,7 @@ def all_reduce_fusion_pass_on_test_model(
|
|||||||
"LOCAL_RANK": str(local_rank),
|
"LOCAL_RANK": str(local_rank),
|
||||||
"WORLD_SIZE": str(world_size),
|
"WORLD_SIZE": str(world_size),
|
||||||
"MASTER_ADDR": "localhost",
|
"MASTER_ADDR": "localhost",
|
||||||
"MASTER_PORT": "12345",
|
"MASTER_PORT": str(master_port),
|
||||||
"VLLM_FLASHINFER_ALLREDUCE_BACKEND": flashinfer_allreduce_backend,
|
"VLLM_FLASHINFER_ALLREDUCE_BACKEND": flashinfer_allreduce_backend,
|
||||||
"VLLM_ROCM_USE_AITER": str(int(use_aiter)),
|
"VLLM_ROCM_USE_AITER": str(int(use_aiter)),
|
||||||
"VLLM_ROCM_USE_AITER_CUSTOM_AR": str(int(use_aiter)),
|
"VLLM_ROCM_USE_AITER_CUSTOM_AR": str(int(use_aiter)),
|
||||||
@@ -681,10 +680,12 @@ def test_rocm_aiter_all_reduce_rmsnorm_group_quant_fp8_fusion_pass_replace(
|
|||||||
num_processes = 2
|
num_processes = 2
|
||||||
|
|
||||||
def run_torch_spawn(fn, nprocs):
|
def run_torch_spawn(fn, nprocs):
|
||||||
|
master_port = get_open_port()
|
||||||
torch.multiprocessing.spawn(
|
torch.multiprocessing.spawn(
|
||||||
fn,
|
fn,
|
||||||
args=(
|
args=(
|
||||||
num_processes,
|
num_processes,
|
||||||
|
master_port,
|
||||||
TestAiterAllReduceRMSNormGroupQuantFP8Model,
|
TestAiterAllReduceRMSNormGroupQuantFP8Model,
|
||||||
batch_size,
|
batch_size,
|
||||||
seq_len,
|
seq_len,
|
||||||
@@ -702,6 +703,7 @@ def test_rocm_aiter_all_reduce_rmsnorm_group_quant_fp8_fusion_pass_replace(
|
|||||||
def rocm_aiter_group_quant_fusion_pass_on_test_model(
|
def rocm_aiter_group_quant_fusion_pass_on_test_model(
|
||||||
local_rank: int,
|
local_rank: int,
|
||||||
world_size: int,
|
world_size: int,
|
||||||
|
master_port: int,
|
||||||
test_model_cls: torch.nn.Module,
|
test_model_cls: torch.nn.Module,
|
||||||
batch_size: int,
|
batch_size: int,
|
||||||
seq_len: int,
|
seq_len: int,
|
||||||
@@ -723,7 +725,7 @@ def rocm_aiter_group_quant_fusion_pass_on_test_model(
|
|||||||
"LOCAL_RANK": str(local_rank),
|
"LOCAL_RANK": str(local_rank),
|
||||||
"WORLD_SIZE": str(world_size),
|
"WORLD_SIZE": str(world_size),
|
||||||
"MASTER_ADDR": "localhost",
|
"MASTER_ADDR": "localhost",
|
||||||
"MASTER_PORT": "12345",
|
"MASTER_PORT": str(master_port),
|
||||||
"VLLM_ROCM_USE_AITER": "1",
|
"VLLM_ROCM_USE_AITER": "1",
|
||||||
"VLLM_ROCM_USE_AITER_CUSTOM_AR": "1",
|
"VLLM_ROCM_USE_AITER_CUSTOM_AR": "1",
|
||||||
}
|
}
|
||||||
@@ -775,17 +777,29 @@ def rocm_aiter_group_quant_fusion_pass_on_test_model(
|
|||||||
|
|
||||||
results_unfused = model(hidden_states)
|
results_unfused = model(hidden_states)
|
||||||
results_fused = compiled_model(hidden_states)
|
results_fused = compiled_model(hidden_states)
|
||||||
# The fused per-group AR+RMS+QUANT op is bit-equivalent to the unfused
|
# AITER's kernel-level oracle permits up to 5% isolated FP8 values to
|
||||||
# chain modulo the small AllReduce + RMSNorm reordering inside aiter.
|
# fall outside a 5% allclose envelope after collective reordering.
|
||||||
# Per-group FP8 quant introduces step noise <=1 per group; use the
|
for site, (reference, actual) in enumerate(
|
||||||
# same tolerance as the sibling FP8 static test.
|
zip(results_unfused[:4], results_fused[:4])
|
||||||
torch.testing.assert_close(results_unfused, results_fused, atol=1e-2, rtol=1e-2)
|
):
|
||||||
|
close = torch.isclose(reference, actual, atol=5e-2, rtol=5e-2)
|
||||||
|
mismatch_ratio = float((~close).float().mean().item())
|
||||||
|
assert mismatch_ratio <= 0.05, (
|
||||||
|
f"FP8 site {site} mismatch ratio {mismatch_ratio:.2%} exceeds 5%"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Residual and bf16 indexer outputs do not incur FP8 quantization noise.
|
||||||
|
torch.testing.assert_close(
|
||||||
|
results_unfused[4:], results_fused[4:], atol=1e-2, rtol=1e-2
|
||||||
|
)
|
||||||
|
|
||||||
# Four pattern firings: norm[0] (no-add quant), norm[1] (add quant,
|
# Four pattern firings: norm[0] (no-add quant), norm[1] (add quant,
|
||||||
# single ``rms`` consumer), norm[2..3] (add quant + indexer fan-out).
|
# single ``rms`` consumer), norm[2..3] (add quant + indexer fan-out).
|
||||||
assert all_reduce_fusion_pass.matched_count == 4, (
|
assert all_reduce_fusion_pass.matched_count == 4, (
|
||||||
f"{all_reduce_fusion_pass.matched_count=}"
|
f"{all_reduce_fusion_pass.matched_count=}"
|
||||||
)
|
)
|
||||||
backend.check_before_ops(model.ops_in_model_before(), fully_replaced=False)
|
backend.check_before_ops(model.ops_in_model_before(), fully_replaced=True)
|
||||||
backend.check_after_ops(model.ops_in_model_after())
|
fused_quant_op, fused_indexer_op = model.ops_in_model_after()
|
||||||
|
assert backend.op_count(fused_quant_op) == 2
|
||||||
|
assert backend.op_count(fused_indexer_op) == 2
|
||||||
del all_reduce_fusion_pass
|
del all_reduce_fusion_pass
|
||||||
|
|||||||
@@ -1245,6 +1245,34 @@ class AiterAllreduceFusedAddRMSNormPattern(BasePattern, VllmPatternReplacement):
|
|||||||
return _replacement
|
return _replacement
|
||||||
|
|
||||||
|
|
||||||
|
class AiterAllreduceFusedAddRMSNormOutputOnlyPattern(
|
||||||
|
AiterAllreduceFusedAddRMSNormPattern
|
||||||
|
):
|
||||||
|
"""Match the add-RMSNorm form when its residual output is dead."""
|
||||||
|
|
||||||
|
@property
|
||||||
|
def pattern(self):
|
||||||
|
pattern = super().pattern
|
||||||
|
|
||||||
|
def _pattern(
|
||||||
|
residual: torch.Tensor, input: torch.Tensor, weight: torch.Tensor
|
||||||
|
) -> torch.Tensor:
|
||||||
|
return pattern(residual, input, weight)[0]
|
||||||
|
|
||||||
|
return _pattern
|
||||||
|
|
||||||
|
@property
|
||||||
|
def replacement(self):
|
||||||
|
replacement = super().replacement
|
||||||
|
|
||||||
|
def _replacement(
|
||||||
|
residual: torch.Tensor, input: torch.Tensor, weight: torch.Tensor
|
||||||
|
) -> torch.Tensor:
|
||||||
|
return replacement(residual, input, weight)[0]
|
||||||
|
|
||||||
|
return _replacement
|
||||||
|
|
||||||
|
|
||||||
class AiterAllreduceFusedRMSNormGroupQuantFP8Pattern(
|
class AiterAllreduceFusedRMSNormGroupQuantFP8Pattern(
|
||||||
BasePattern, VllmPatternReplacement
|
BasePattern, VllmPatternReplacement
|
||||||
):
|
):
|
||||||
@@ -1624,6 +1652,13 @@ class RocmAiterAllReduceFusionPass(VllmFusionPatternMatcherPass):
|
|||||||
self.device,
|
self.device,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
self.register(
|
||||||
|
AiterAllreduceFusedAddRMSNormOutputOnlyPattern(
|
||||||
|
epsilon,
|
||||||
|
self.model_dtype,
|
||||||
|
self.device,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
# WARNING: This is a hack to clear the pattern matcher cache
|
# WARNING: This is a hack to clear the pattern matcher cache
|
||||||
# and allow multiple values of epsilon.
|
# and allow multiple values of epsilon.
|
||||||
|
|||||||
Reference in New Issue
Block a user