[ROCm][Kimi-K3] Fuse AttnRes output with per-token FP8 input quantization (#59069)

Signed-off-by: whx-sjtu <xiaowang990929@gmail.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Fangzhou Ai <31551580+Fangzhou-Ai@users.noreply.github.com>
This commit is contained in:
Hexiang Wang
2026-10-08 12:01:47 +00:00
committed by GitHub
co-authored by OpenAI Codex Fangzhou Ai
parent 73c742b09c
commit 458ba2edf8
7 changed files with 455 additions and 15 deletions
+12 -4
View File
@@ -13,6 +13,10 @@ from vllm.config import VllmConfig
from vllm.distributed import divide
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.layers.fusion.quant_activation import (
QuantizedActivation,
get_input_quant_key,
)
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
RowParallelLinear,
@@ -37,6 +41,7 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
causal_conv1d_fn,
causal_conv1d_update,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
from vllm.model_executor.utils import set_weight_attrs
from vllm.models.kimi_k3.amd.kda_metadata import KimiK3ROCmKDABackend
@@ -300,13 +305,16 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
qkv = qkv.permute(1, 0, 2, 3).contiguous().unsqueeze(1)
return qkv.unbind(0)
def get_input_quant_key(self) -> QuantKey | None:
return get_input_quant_key(self.in_proj_qkvgfab)
def forward(
self,
hidden_states: torch.Tensor,
hidden_states: torch.Tensor | QuantizedActivation,
positions: torch.Tensor,
) -> torch.Tensor:
num_tokens = hidden_states.size(0)
projected_qkvgfab = self.in_proj_qkvgfab(hidden_states)[0]
num_tokens = projected_qkvgfab.shape[0]
split_sizes = [
3 * self.local_projection_size,
@@ -326,8 +334,8 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
core_attn_out = torch.empty(
(1, num_tokens, self.local_num_heads, self.head_dim),
dtype=hidden_states.dtype,
device=hidden_states.device,
dtype=projected_qkvgfab.dtype,
device=projected_qkvgfab.device,
)
self._forward(
+44 -5
View File
@@ -2,11 +2,12 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
from typing import Any
from typing import Any, cast
import torch
from torch import nn
from vllm import envs
from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import (
get_pp_group,
@@ -19,6 +20,10 @@ from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
from vllm.model_executor.layers.fusion.quant_activation import (
QuantizedActivation,
get_input_quant_key,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -38,6 +43,10 @@ from vllm.model_executor.layers.mamba.mamba_utils import (
)
from vllm.model_executor.layers.mla import MLAModules
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kFp8DynamicTokenSym,
)
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
@@ -114,6 +123,9 @@ class KimiMLP(nn.Module):
"Only silu and situ are supported."
)
def get_input_quant_key(self) -> QuantKey | None:
return get_input_quant_key(self.gate_up_proj)
def forward(self, x):
gate_up, _ = self.gate_up_proj(x)
x = self.act_fn(gate_up)
@@ -148,8 +160,10 @@ def _apply_attn_res(
delta: torch.Tensor | None = None,
output_norm: RMSNorm | None = None,
block_write_idx: int = -1,
) -> torch.Tensor:
return attn_res(
quant_key: QuantKey | None = None,
) -> torch.Tensor | QuantizedActivation:
fuse_quant = quant_key == kFp8DynamicTokenSym and not envs.VLLM_BATCH_INVARIANT
result = attn_res(
prefix_sum,
delta,
block_residual,
@@ -160,7 +174,18 @@ def _apply_attn_res(
block_write_idx,
norm.variance_epsilon,
0.0 if output_norm is None else output_norm.variance_epsilon,
quant_dtype=cast(torch.dtype, kFp8DynamicTokenSym.dtype)
if fuse_quant
else None,
)
if fuse_quant:
assert isinstance(result, tuple)
data, scale = result
return QuantizedActivation(
data, scale, prefix_sum.dtype, prefix_sum.shape, kFp8DynamicTokenSym
)
assert isinstance(result, torch.Tensor)
return result
class KimiMoE(nn.Module):
@@ -452,10 +477,13 @@ class KimiMLAAttention(nn.Module):
prefix,
)
def get_input_quant_key(self) -> QuantKey | None:
return self.mla_attn.get_input_quant_key()
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
hidden_states: torch.Tensor | QuantizedActivation,
) -> torch.Tensor:
return self.mla_attn(positions, hidden_states)
@@ -579,7 +607,7 @@ class KimiDecoderLayer(nn.Module):
def _run_self_attn(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
hidden_states: torch.Tensor | QuantizedActivation,
) -> torch.Tensor:
return self.self_attn(
hidden_states=hidden_states,
@@ -626,6 +654,11 @@ class KimiDecoderLayer(nn.Module):
prefix_delta: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
prefix_sum = hidden_states
attention_quant_key = (
self.self_attn.get_input_quant_key()
if isinstance(self.self_attn, (KimiK3DeltaAttention, KimiMLAAttention))
else None
)
hidden_states = _apply_attn_res(
prefix_sum,
block_residual,
@@ -635,6 +668,7 @@ class KimiDecoderLayer(nn.Module):
delta=prefix_delta,
output_norm=self.input_layernorm,
block_write_idx=(self.block_write_idx if self.is_block_write_layer else -1),
quant_key=attention_quant_key,
)
if self.is_block_write_layer:
@@ -659,6 +693,11 @@ class KimiDecoderLayer(nn.Module):
mlp_valid_blocks,
delta=prefix_delta,
output_norm=self.post_attention_layernorm,
quant_key=(
self.mlp.get_input_quant_key()
if isinstance(self.mlp, KimiMLP)
else None
),
)
hidden_states = self.mlp(hidden_states)
+27 -2
View File
@@ -8,8 +8,13 @@ import torch
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.layers.attention.attention import get_attention_context
from vllm.model_executor.layers.fusion.quant_activation import (
QuantizedActivation,
get_input_quant_key,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.mla import MultiHeadLatentAttentionWrapper
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
from vllm.platforms import current_platform
_OPT_KV_LORA_RANK = 512
@@ -49,6 +54,21 @@ class KimiK3MultiHeadLatentAttentionWrapper(MultiHeadLatentAttentionWrapper):
not attn.is_aiter_triton_fp8_bmm_enabled and attn.W_UK_T is not None
)
def get_input_quant_key(self) -> QuantKey | None:
if self.indexer is not None:
return None
consumers = (
[self.fused_qkv_a_proj]
if self.q_lora_rank is not None
else [self.kv_a_proj_with_mqa, self.q_proj]
)
if self.g_proj is not None:
consumers.append(self.g_proj)
if any(layer is None for layer in consumers):
return None
keys = [get_input_quant_key(layer) for layer in consumers if layer is not None]
return keys[0] if all(key == keys[0] for key in keys) else None
def _normalize_q_kv(
self,
q_c: torch.Tensor,
@@ -173,10 +193,15 @@ class KimiK3MultiHeadLatentAttentionWrapper(MultiHeadLatentAttentionWrapper):
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
hidden_states: torch.Tensor | QuantizedActivation,
llama_4_scaling: torch.Tensor | None = None,
) -> torch.Tensor:
q_c = None
num_tokens = (
hidden_states.orig_shape[0]
if isinstance(hidden_states, QuantizedActivation)
else hidden_states.shape[0]
)
if self.q_lora_rank is not None:
assert self.fused_qkv_a_proj is not None, (
@@ -238,7 +263,7 @@ class KimiK3MultiHeadLatentAttentionWrapper(MultiHeadLatentAttentionWrapper):
if self.dcp_q_replicate:
q_dcp_replicated, q = q, q_proj_layer._local_view(q)
output_shape = (hidden_states.shape[0], self.num_heads * self.v_head_dim)
output_shape = (num_tokens, self.num_heads * self.v_head_dim)
# The fused decode path covers only the MQA slice, so it is taken for
# decode-only batches; anything else falls through to MLAAttention.
+27 -4
View File
@@ -22,6 +22,7 @@ def _attn_res_kernel(
qk_weight_ptr,
output_norm_weight_ptr,
output_ptr,
output_scale_ptr,
stride_prefix_m: tl.constexpr,
stride_delta_m: tl.constexpr,
stride_block_m: tl.constexpr,
@@ -35,6 +36,7 @@ def _attn_res_kernel(
HAS_DELTA: tl.constexpr,
WRITE_BLOCK: tl.constexpr,
APPLY_OUTPUT_NORM: tl.constexpr,
QUANT_MAX: tl.constexpr,
BLOCK_L: tl.constexpr,
BLOCK_D: tl.constexpr,
):
@@ -129,6 +131,14 @@ def _attn_res_kernel(
output_norm_weight_ptr + d_offsets, mask=d_mask, other=0.0
).to(tl.float32)
output = mixed * output_reciprocal_std * output_norm_weight
if QUANT_MAX > 0:
# Preserve the rounding of the original BF16 output before quantizing.
output = output.to(prefix_ptr.dtype.element_ty).to(tl.float32)
amax = tl.max(tl.where(d_mask, tl.abs(output), 0.0), axis=0)
scale = tl.maximum(amax, 1e-10) * (1.0 / QUANT_MAX)
inv_scale = 1.0 / scale
output = tl.minimum(tl.maximum(output * inv_scale, -QUANT_MAX), QUANT_MAX)
tl.store(output_scale_ptr + row_idx, scale)
tl.store(
output_ptr + row_idx * stride_output_m + d_offsets,
output,
@@ -147,7 +157,9 @@ def attn_res(
block_write_idx: int,
eps: float,
output_norm_eps: float,
) -> torch.Tensor:
*,
quant_dtype: torch.dtype | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
num_tokens, hidden_size = prefix.shape
assert prefix.stride(-1) == 1
assert delta is None or delta.stride(-1) == 1
@@ -155,9 +167,18 @@ def attn_res(
assert norm_weight.stride(-1) == 1
assert qk_weight.stride(-1) == 1
assert output_norm_weight is None or output_norm_weight.stride(-1) == 1
output = prefix.new_empty(prefix.shape)
if quant_dtype is not None:
assert quant_dtype in (torch.float8_e4m3fn, torch.float8_e4m3fnuz)
output = torch.empty_like(
prefix, dtype=quant_dtype or prefix.dtype, memory_format=torch.contiguous_format
)
scale = (
torch.empty((num_tokens, 1), device=prefix.device, dtype=torch.float32)
if quant_dtype is not None
else None
)
if num_tokens == 0:
return output
return output if scale is None else (output, scale)
# Source tiling helps decode, while one-source tiles scale better for prefill.
if num_tokens >= 256 or num_blocks <= 1:
@@ -172,6 +193,7 @@ def attn_res(
qk_weight,
output_norm_weight,
output,
scale,
prefix.stride(0),
0 if delta is None else delta.stride(0),
blocks.stride(0),
@@ -185,9 +207,10 @@ def attn_res(
HAS_DELTA=delta is not None,
WRITE_BLOCK=block_write_idx >= 0,
APPLY_OUTPUT_NORM=output_norm_weight is not None,
QUANT_MAX=0.0 if quant_dtype is None else torch.finfo(quant_dtype).max,
BLOCK_L=block_l,
BLOCK_D=triton.next_power_of_2(hidden_size),
num_warps=num_warps,
num_stages=2,
)
return output
return output if scale is None else (output, scale)