[ROCm][Kimi-K3] Use vLLM's FP8 max in the fused AttnRes quantization (#60717)
Signed-off-by: Rohan138 <rohanpotdar138@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
fa312090cb
commit
88a419649e
@@ -247,12 +247,16 @@ def test_amd_attn_res_fp8_preserves_prefix_and_quantized_output(
|
||||
)
|
||||
ref_output, ref_scale = quant(reference)
|
||||
torch.testing.assert_close(scale, ref_scale, atol=1e-7, rtol=1e-6)
|
||||
|
||||
# Floating-point fusion can move values at FP8 rounding midpoints.
|
||||
# Bound each change to the adjacent code and bound how often it occurs.
|
||||
code_delta = (
|
||||
output.view(torch.uint8).to(torch.int16)
|
||||
- ref_output.view(torch.uint8).to(torch.int16)
|
||||
).abs()
|
||||
# Codes are sign-magnitude, so order them by value to keep a step
|
||||
# across zero adjacent.
|
||||
def ordinal(x: torch.Tensor) -> torch.Tensor:
|
||||
code = x.view(torch.uint8).to(torch.int16)
|
||||
return torch.where(code >= 0x80, 0x80 - code, code)
|
||||
|
||||
code_delta = (ordinal(output) - ordinal(ref_output)).abs()
|
||||
assert code_delta.max().item() <= 1
|
||||
assert (code_delta != 0).float().mean().item() < 1e-4
|
||||
torch.testing.assert_close(prefix, ref_prefix, atol=0, rtol=0)
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import get_fp8_min_max
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@@ -207,7 +208,7 @@ def attn_res(
|
||||
HAS_DELTA=delta is not None,
|
||||
WRITE_BLOCK=block_write_idx >= 0,
|
||||
APPLY_OUTPUT_NORM=output_norm_weight is not None,
|
||||
QUANT_MAX=0.0 if quant_dtype is None else torch.finfo(quant_dtype).max,
|
||||
QUANT_MAX=0.0 if quant_dtype is None else get_fp8_min_max()[1],
|
||||
BLOCK_L=block_l,
|
||||
BLOCK_D=triton.next_power_of_2(hidden_size),
|
||||
num_warps=num_warps,
|
||||
|
||||
Reference in New Issue
Block a user