[ROCm] Fix CI runtime and tests for MI355 DPX (#58244)
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: Mahesh Kunreddi <mahesh.kunreddi@amd.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Mahesh Kunreddi
Claude
Andreas Karatzas
Codex
Andreas Karatzas
parent
44257c2e35
commit
4013f3ad6c
@@ -12,6 +12,9 @@ from tests.kernels.quant_utils import (
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.fused_moe import fused_experts, fused_topk
|
||||
from vllm.model_executor.layers.quantization.utils.int8_utils import (
|
||||
per_token_group_quant_int8,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.get_device_capability() < (7, 0):
|
||||
@@ -64,7 +67,7 @@ def torch_w8a8_block_int8_moe(a, w1, w2, w1_s, w2_s, score, topk, block_shape):
|
||||
topk_ids = topk_ids.view(-1)
|
||||
|
||||
_, block_k = block_shape[0], block_shape[1]
|
||||
a_q, a_s = native_per_token_group_quant_int8(a, block_k)
|
||||
a_q, a_s = native_per_token_group_quant_int8(a, block_k, round_to_nearest=False)
|
||||
for i in range(w1.shape[0]):
|
||||
mask = topk_ids == i
|
||||
if mask.sum():
|
||||
@@ -72,7 +75,9 @@ def torch_w8a8_block_int8_moe(a, w1, w2, w1_s, w2_s, score, topk, block_shape):
|
||||
a_q[mask], w1[i], a_s[mask], w1_s[i], block_shape, output_dtype=a.dtype
|
||||
)
|
||||
act_out = SiluAndMul().forward_native(inter_out)
|
||||
act_out_q, act_out_s = native_per_token_group_quant_int8(act_out, block_k)
|
||||
act_out_q, act_out_s = native_per_token_group_quant_int8(
|
||||
act_out, block_k, round_to_nearest=False
|
||||
)
|
||||
act_out = act_out.to(torch.float32)
|
||||
out[mask] = native_w8a8_block_matmul(
|
||||
act_out_q, w2[i], act_out_s, w2_s[i], block_shape, output_dtype=a.dtype
|
||||
@@ -132,3 +137,27 @@ def test_w8a8_block_int8_fused_moe(M, N, K, E, topk, block_size, dtype, seed):
|
||||
|
||||
# Check results
|
||||
torch.testing.assert_close(out, ref_out, atol=0.065, rtol=0.065)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
|
||||
def test_block_int8_quantization_truncates_fractional_values(dtype):
|
||||
"""Block INT8 quantization truncates, unlike per-token round-to-nearest."""
|
||||
device = current_platform.device_type
|
||||
x = torch.tensor(
|
||||
[[-127, -1.75, -0.75, 0.75, 1.75, 127] + [0] * 122],
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
)
|
||||
expected = torch.tensor(
|
||||
[[-127, -1, 0, 0, 1, 127] + [0] * 122], dtype=torch.int8, device=device
|
||||
)
|
||||
|
||||
out_q, scale = per_token_group_quant_int8(x, group_size=128)
|
||||
ref_q, ref_s = native_per_token_group_quant_int8(
|
||||
x, group_size=128, round_to_nearest=False
|
||||
)
|
||||
|
||||
torch.testing.assert_close(out_q, expected, atol=0, rtol=0)
|
||||
torch.testing.assert_close(ref_q, expected, atol=0, rtol=0)
|
||||
torch.testing.assert_close(scale, torch.ones_like(scale), atol=0, rtol=0)
|
||||
torch.testing.assert_close(ref_s, scale, atol=0, rtol=0)
|
||||
|
||||
@@ -181,12 +181,15 @@ def native_per_token_group_quant_fp8(
|
||||
return x_q, x_s
|
||||
|
||||
|
||||
def native_per_token_group_quant_int8(x, group_size, eps=1e-10, dtype=torch.int8):
|
||||
def native_per_token_group_quant_int8(
|
||||
x, group_size, eps=1e-10, dtype=torch.int8, *, round_to_nearest=True
|
||||
):
|
||||
"""Function to perform per-token-group quantization on an input tensor
|
||||
`x` using native torch.
|
||||
|
||||
It converts the tensor values into int8 values and returns the
|
||||
quantized tensor along with the scaling factor used for quantization.
|
||||
Per-token quantization rounds to nearest; block quantization truncates.
|
||||
"""
|
||||
assert x.shape[-1] % group_size == 0, (
|
||||
"the last dimension of `x` must be divisible by `group_size`"
|
||||
@@ -201,9 +204,10 @@ def native_per_token_group_quant_int8(x, group_size, eps=1e-10, dtype=torch.int8
|
||||
# Use float32 for scale calculation for stability
|
||||
amax = x_.abs().max(dim=-1, keepdim=True)[0].clamp(min=eps).to(torch.float32)
|
||||
x_s = amax / int8_max
|
||||
x_q = (
|
||||
(x_.to(torch.float32) / x_s).round().clamp(min=int8_min, max=int8_max).to(dtype)
|
||||
) # Round before clamping
|
||||
x_q = x_.to(torch.float32) / x_s
|
||||
if round_to_nearest:
|
||||
x_q = x_q.round()
|
||||
x_q = x_q.clamp(min=int8_min, max=int8_max).to(dtype)
|
||||
x_q = x_q.reshape(x.shape)
|
||||
x_s = x_s.reshape(x.shape[:-1] + (x.shape[-1] // group_size,))
|
||||
|
||||
|
||||
@@ -614,11 +614,13 @@ def test_aiter_fp4_gemm_skinny_shapes(M, N, K):
|
||||
out = gemm_afp4wfp4(A_fp4, B_fp4, A_scale, B_scale)
|
||||
|
||||
assert out.shape == (M, N)
|
||||
assert out.dtype == torch.bfloat16
|
||||
assert not torch.any(torch.isnan(out))
|
||||
|
||||
# Accumulate in FP32, then match the GEMM's BF16 output rounding.
|
||||
A_dq = quant_dequant_mxfp4(A)
|
||||
B_dq = quant_dequant_mxfp4(B)
|
||||
ref = torch.matmul(A_dq.float(), B_dq.t().float())
|
||||
ref = torch.matmul(A_dq.float(), B_dq.t().float()).to(torch.bfloat16).float()
|
||||
|
||||
_print_close_stats(
|
||||
f"skinny_gemm M={M} N={N} K={K}",
|
||||
|
||||
@@ -132,7 +132,8 @@ M_FACTORS_WVSPLITKRC = [128, 128 + 16, 256, 256 + 16, 640, 640 + 16]
|
||||
NKM_FACTORS_WVSPLITKRC_LARGE_K = [
|
||||
(128, 6144, 128),
|
||||
(96, 8192, 128),
|
||||
(128, 12288, 128),
|
||||
# Keep 24 K-shards (batched readback) within the DPX budget of 128 CUs.
|
||||
(128, 12288, 64),
|
||||
(32, 12288, 128),
|
||||
]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user