[ROCm][Kimi-K3] Fuse AttnRes output with per-token FP8 input quantization (#59069)

Signed-off-by: whx-sjtu <xiaowang990929@gmail.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Fangzhou Ai <31551580+Fangzhou-Ai@users.noreply.github.com>
This commit is contained in:
Hexiang Wang
2026-10-08 12:01:47 +00:00
committed by GitHub
co-authored by OpenAI Codex Fangzhou Ai
parent 73c742b09c
commit 458ba2edf8
7 changed files with 455 additions and 15 deletions
+99
View File
@@ -191,3 +191,102 @@ def test_amd_attn_res_fused_contract(
original_blocks[:, block_write_idx].copy_(expected_prefix)
torch.testing.assert_close(blocks, original_blocks, atol=0, rtol=0)
assert actual.is_contiguous()
@pytest.mark.parametrize(
"num_tokens,num_blocks,has_delta,write_block",
[
(0, 0, False, False),
(1, 0, True, True),
(17, 4, True, True),
(320, 8, False, False),
],
)
def test_amd_attn_res_fp8_preserves_prefix_and_quantized_output(
num_tokens,
num_blocks,
has_delta,
write_block,
default_vllm_config,
):
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
torch.manual_seed(41)
hidden_size = 7168
prefix = _randn_with_row_padding(num_tokens, hidden_size, padding=7)
delta = _randn_with_row_padding(num_tokens, hidden_size) if has_delta else None
blocks = _randn_with_row_padding(num_tokens, 9, hidden_size, padding=13)
norm = 1 + torch.randn(hidden_size, device="cuda", dtype=torch.bfloat16) * 0.1
score = torch.randn_like(norm) / hidden_size**0.5
out_norm = 1 + torch.randn_like(norm) * 0.1
ref_prefix, ref_blocks = prefix.clone(), blocks.clone()
kwargs = dict(
num_blocks=num_blocks,
block_write_idx=num_blocks if write_block else -1,
eps=1e-5,
output_norm_eps=1e-5,
)
reference = attn_res(ref_prefix, delta, ref_blocks, norm, score, out_norm, **kwargs)
output, scale = attn_res(
prefix,
delta,
blocks,
norm,
score,
out_norm,
quant_dtype=current_platform.fp8_dtype(),
**kwargs,
)
assert output.dtype == current_platform.fp8_dtype()
assert scale.shape == (num_tokens, 1)
if num_tokens:
# Keep reference rounding independent of Inductor's fused quantization.
quant = QuantFP8(
static=False, group_shape=GroupShape.PER_TOKEN, compile_native=False
)
ref_output, ref_scale = quant(reference)
torch.testing.assert_close(scale, ref_scale, atol=1e-7, rtol=1e-6)
# Floating-point fusion can move values at FP8 rounding midpoints.
# Bound each change to the adjacent code and bound how often it occurs.
code_delta = (
output.view(torch.uint8).to(torch.int16)
- ref_output.view(torch.uint8).to(torch.int16)
).abs()
assert code_delta.max().item() <= 1
assert (code_delta != 0).float().mean().item() < 1e-4
torch.testing.assert_close(prefix, ref_prefix, atol=0, rtol=0)
torch.testing.assert_close(blocks, ref_blocks, atol=0, rtol=0)
@pytest.mark.parametrize(
"matching,requires_unquantized", [(True, False), (False, False), (True, True)]
)
def test_mla_input_quant_requires_all_consumers(matching, requires_unquantized):
from types import SimpleNamespace
from torch import nn
from vllm.model_executor.layers.fusion.quant_activation import (
expose_input_quant_key,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8DynamicTokenSym,
)
from vllm.models.kimi_k3.amd.mla import KimiK3MultiHeadLatentAttentionWrapper
attention = object.__new__(KimiK3MultiHeadLatentAttentionWrapper)
nn.Module.__init__(attention)
attention.indexer = None
attention.q_lora_rank = 1536
attention.fused_qkv_a_proj = nn.Identity()
attention.g_proj = nn.Identity()
kernel = SimpleNamespace(input_quant_key=lambda: kFp8DynamicTokenSym)
expose_input_quant_key(attention.fused_qkv_a_proj, kernel)
if matching:
expose_input_quant_key(attention.g_proj, kernel)
attention.g_proj.requires_unquantized_input = requires_unquantized
actual = attention.get_input_quant_key()
assert actual == (
kFp8DynamicTokenSym if matching and not requires_unquantized else None
)
@@ -1,6 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from torch import nn
@@ -36,3 +39,81 @@ def test_mla_self_attention_returns_projection_storage_directly() -> None:
assert output is projected
assert output.data_ptr() == projected.data_ptr()
@pytest.mark.parametrize("quantized", [False, True])
@pytest.mark.parametrize("fused_decode", [False, True])
def test_mla_forward_preserves_output_shape_with_fp8_input(
monkeypatch, quantized: bool, fused_decode: bool
) -> None:
"""Both MLA paths accept producer-quantized input after decode fusion."""
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8DynamicTokenSym,
)
from vllm.models.kimi_k3.amd.mla import KimiK3MultiHeadLatentAttentionWrapper
hidden_states = torch.randn(4, 8, dtype=torch.bfloat16)
if quantized:
hidden_states = QuantizedActivation(
hidden_states.to(kFp8DynamicTokenSym.dtype),
torch.ones(4, 1),
hidden_states.dtype,
hidden_states.shape,
kFp8DynamicTokenSym,
)
projected = torch.randn(4, 8, dtype=torch.bfloat16)
paths = []
def input_projection(value):
assert value is hidden_states
return torch.zeros(4, 8, dtype=torch.bfloat16), None
def attention_output(path, output_shape):
assert output_shape == projected.shape
paths.append(path)
return projected
attention = SimpleNamespace(
q_lora_rank=2,
kv_lora_rank=4,
qk_rope_head_dim=2,
qk_head_dim=4,
num_heads=2,
v_head_dim=4,
fused_qkv_a_proj=input_projection,
q_a_layernorm=nn.Identity(),
_normalize_q_kv=lambda q, kv: (q, kv),
q_b_proj=lambda q: (torch.zeros(4, 8, dtype=q.dtype), None),
rotary_emb=None,
indexer=None,
dcp_q_replicate=False,
_fused_qk_prep=fused_decode,
_fused_decode=lambda q, kv, pe, pos, shape, *args: attention_output(
"fused", shape
),
g_proj=None,
o_proj=lambda value: (value, None),
)
if fused_decode:
attention.mla_attn = SimpleNamespace(layer_name="test")
monkeypatch.setattr(
"vllm.models.kimi_k3.amd.mla.get_attention_context",
lambda _: (
SimpleNamespace(num_actual_tokens=4, num_decode_tokens=4),
None,
torch.zeros(1),
torch.arange(4),
),
)
else:
attention.mla_attn = lambda q, kv, pe, **kwargs: attention_output(
"regular", kwargs["output_shape"]
)
output = KimiK3MultiHeadLatentAttentionWrapper.forward(
attention, torch.arange(4), hidden_states
)
assert output is projected
assert paths == ["fused" if fused_decode else "regular"]