From 88a419649e36f3412d2650c7162135a3b12bc5a1 Mon Sep 17 00:00:00 2001 From: Rohan Potdar Date: Thu, 8 Oct 2026 23:45:13 -0500 Subject: [PATCH] [ROCm][Kimi-K3] Use vLLM's FP8 max in the fused AttnRes quantization (#60717) Signed-off-by: Rohan138 Co-authored-by: Claude Opus 5.5 --- tests/models/kimi_k3/test_amd_attn_res.py | 12 ++++++++---- vllm/models/kimi_k3/amd/ops/attn_res.py | 3 ++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/models/kimi_k3/test_amd_attn_res.py b/tests/models/kimi_k3/test_amd_attn_res.py index f4efdbd5b2..871d53faae 100644 --- a/tests/models/kimi_k3/test_amd_attn_res.py +++ b/tests/models/kimi_k3/test_amd_attn_res.py @@ -247,12 +247,16 @@ def test_amd_attn_res_fp8_preserves_prefix_and_quantized_output( ) ref_output, ref_scale = quant(reference) torch.testing.assert_close(scale, ref_scale, atol=1e-7, rtol=1e-6) + # Floating-point fusion can move values at FP8 rounding midpoints. # Bound each change to the adjacent code and bound how often it occurs. - code_delta = ( - output.view(torch.uint8).to(torch.int16) - - ref_output.view(torch.uint8).to(torch.int16) - ).abs() + # Codes are sign-magnitude, so order them by value to keep a step + # across zero adjacent. + def ordinal(x: torch.Tensor) -> torch.Tensor: + code = x.view(torch.uint8).to(torch.int16) + return torch.where(code >= 0x80, 0x80 - code, code) + + code_delta = (ordinal(output) - ordinal(ref_output)).abs() assert code_delta.max().item() <= 1 assert (code_delta != 0).float().mean().item() < 1e-4 torch.testing.assert_close(prefix, ref_prefix, atol=0, rtol=0) diff --git a/vllm/models/kimi_k3/amd/ops/attn_res.py b/vllm/models/kimi_k3/amd/ops/attn_res.py index 5c4fe7a0b0..95f748e72d 100644 --- a/vllm/models/kimi_k3/amd/ops/attn_res.py +++ b/vllm/models/kimi_k3/amd/ops/attn_res.py @@ -10,6 +10,7 @@ import torch +from vllm.model_executor.layers.quantization.utils.quant_utils import get_fp8_min_max from vllm.triton_utils import tl, triton @@ -207,7 +208,7 @@ def attn_res( HAS_DELTA=delta is not None, WRITE_BLOCK=block_write_idx >= 0, APPLY_OUTPUT_NORM=output_norm_weight is not None, - QUANT_MAX=0.0 if quant_dtype is None else torch.finfo(quant_dtype).max, + QUANT_MAX=0.0 if quant_dtype is None else get_fp8_min_max()[1], BLOCK_L=block_l, BLOCK_D=triton.next_power_of_2(hidden_size), num_warps=num_warps,