[ROCm][Kimi-K3] Fuse AttnRes output with per-token FP8 input quantization (#59069)
Signed-off-by: whx-sjtu <xiaowang990929@gmail.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Fangzhou Ai <31551580+Fangzhou-Ai@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Fangzhou Ai
parent
73c742b09c
commit
458ba2edf8
@@ -13,6 +13,10 @@ from vllm.config import VllmConfig
|
||||
from vllm.distributed import divide
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fusion.quant_activation import (
|
||||
QuantizedActivation,
|
||||
get_input_quant_key,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
@@ -37,6 +41,7 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
|
||||
causal_conv1d_fn,
|
||||
causal_conv1d_update,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
|
||||
from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.models.kimi_k3.amd.kda_metadata import KimiK3ROCmKDABackend
|
||||
@@ -300,13 +305,16 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
qkv = qkv.permute(1, 0, 2, 3).contiguous().unsqueeze(1)
|
||||
return qkv.unbind(0)
|
||||
|
||||
def get_input_quant_key(self) -> QuantKey | None:
|
||||
return get_input_quant_key(self.in_proj_qkvgfab)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
hidden_states: torch.Tensor | QuantizedActivation,
|
||||
positions: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
num_tokens = hidden_states.size(0)
|
||||
projected_qkvgfab = self.in_proj_qkvgfab(hidden_states)[0]
|
||||
num_tokens = projected_qkvgfab.shape[0]
|
||||
|
||||
split_sizes = [
|
||||
3 * self.local_projection_size,
|
||||
@@ -326,8 +334,8 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
|
||||
core_attn_out = torch.empty(
|
||||
(1, num_tokens, self.local_num_heads, self.head_dim),
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
dtype=projected_qkvgfab.dtype,
|
||||
device=projected_qkvgfab.device,
|
||||
)
|
||||
|
||||
self._forward(
|
||||
|
||||
@@ -2,11 +2,12 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
from typing import Any, cast
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm import envs
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_pp_group,
|
||||
@@ -19,6 +20,10 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
|
||||
from vllm.model_executor.layers.fusion.quant_activation import (
|
||||
QuantizedActivation,
|
||||
get_input_quant_key,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -38,6 +43,10 @@ from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
)
|
||||
from vllm.model_executor.layers.mla import MLAModules
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kFp8DynamicTokenSym,
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
@@ -114,6 +123,9 @@ class KimiMLP(nn.Module):
|
||||
"Only silu and situ are supported."
|
||||
)
|
||||
|
||||
def get_input_quant_key(self) -> QuantKey | None:
|
||||
return get_input_quant_key(self.gate_up_proj)
|
||||
|
||||
def forward(self, x):
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
x = self.act_fn(gate_up)
|
||||
@@ -148,8 +160,10 @@ def _apply_attn_res(
|
||||
delta: torch.Tensor | None = None,
|
||||
output_norm: RMSNorm | None = None,
|
||||
block_write_idx: int = -1,
|
||||
) -> torch.Tensor:
|
||||
return attn_res(
|
||||
quant_key: QuantKey | None = None,
|
||||
) -> torch.Tensor | QuantizedActivation:
|
||||
fuse_quant = quant_key == kFp8DynamicTokenSym and not envs.VLLM_BATCH_INVARIANT
|
||||
result = attn_res(
|
||||
prefix_sum,
|
||||
delta,
|
||||
block_residual,
|
||||
@@ -160,7 +174,18 @@ def _apply_attn_res(
|
||||
block_write_idx,
|
||||
norm.variance_epsilon,
|
||||
0.0 if output_norm is None else output_norm.variance_epsilon,
|
||||
quant_dtype=cast(torch.dtype, kFp8DynamicTokenSym.dtype)
|
||||
if fuse_quant
|
||||
else None,
|
||||
)
|
||||
if fuse_quant:
|
||||
assert isinstance(result, tuple)
|
||||
data, scale = result
|
||||
return QuantizedActivation(
|
||||
data, scale, prefix_sum.dtype, prefix_sum.shape, kFp8DynamicTokenSym
|
||||
)
|
||||
assert isinstance(result, torch.Tensor)
|
||||
return result
|
||||
|
||||
|
||||
class KimiMoE(nn.Module):
|
||||
@@ -452,10 +477,13 @@ class KimiMLAAttention(nn.Module):
|
||||
prefix,
|
||||
)
|
||||
|
||||
def get_input_quant_key(self) -> QuantKey | None:
|
||||
return self.mla_attn.get_input_quant_key()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
hidden_states: torch.Tensor | QuantizedActivation,
|
||||
) -> torch.Tensor:
|
||||
return self.mla_attn(positions, hidden_states)
|
||||
|
||||
@@ -579,7 +607,7 @@ class KimiDecoderLayer(nn.Module):
|
||||
def _run_self_attn(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
hidden_states: torch.Tensor | QuantizedActivation,
|
||||
) -> torch.Tensor:
|
||||
return self.self_attn(
|
||||
hidden_states=hidden_states,
|
||||
@@ -626,6 +654,11 @@ class KimiDecoderLayer(nn.Module):
|
||||
prefix_delta: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
prefix_sum = hidden_states
|
||||
attention_quant_key = (
|
||||
self.self_attn.get_input_quant_key()
|
||||
if isinstance(self.self_attn, (KimiK3DeltaAttention, KimiMLAAttention))
|
||||
else None
|
||||
)
|
||||
hidden_states = _apply_attn_res(
|
||||
prefix_sum,
|
||||
block_residual,
|
||||
@@ -635,6 +668,7 @@ class KimiDecoderLayer(nn.Module):
|
||||
delta=prefix_delta,
|
||||
output_norm=self.input_layernorm,
|
||||
block_write_idx=(self.block_write_idx if self.is_block_write_layer else -1),
|
||||
quant_key=attention_quant_key,
|
||||
)
|
||||
|
||||
if self.is_block_write_layer:
|
||||
@@ -659,6 +693,11 @@ class KimiDecoderLayer(nn.Module):
|
||||
mlp_valid_blocks,
|
||||
delta=prefix_delta,
|
||||
output_norm=self.post_attention_layernorm,
|
||||
quant_key=(
|
||||
self.mlp.get_input_quant_key()
|
||||
if isinstance(self.mlp, KimiMLP)
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
|
||||
@@ -8,8 +8,13 @@ import torch
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.layers.attention.attention import get_attention_context
|
||||
from vllm.model_executor.layers.fusion.quant_activation import (
|
||||
QuantizedActivation,
|
||||
get_input_quant_key,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.mla import MultiHeadLatentAttentionWrapper
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
_OPT_KV_LORA_RANK = 512
|
||||
@@ -49,6 +54,21 @@ class KimiK3MultiHeadLatentAttentionWrapper(MultiHeadLatentAttentionWrapper):
|
||||
not attn.is_aiter_triton_fp8_bmm_enabled and attn.W_UK_T is not None
|
||||
)
|
||||
|
||||
def get_input_quant_key(self) -> QuantKey | None:
|
||||
if self.indexer is not None:
|
||||
return None
|
||||
consumers = (
|
||||
[self.fused_qkv_a_proj]
|
||||
if self.q_lora_rank is not None
|
||||
else [self.kv_a_proj_with_mqa, self.q_proj]
|
||||
)
|
||||
if self.g_proj is not None:
|
||||
consumers.append(self.g_proj)
|
||||
if any(layer is None for layer in consumers):
|
||||
return None
|
||||
keys = [get_input_quant_key(layer) for layer in consumers if layer is not None]
|
||||
return keys[0] if all(key == keys[0] for key in keys) else None
|
||||
|
||||
def _normalize_q_kv(
|
||||
self,
|
||||
q_c: torch.Tensor,
|
||||
@@ -173,10 +193,15 @@ class KimiK3MultiHeadLatentAttentionWrapper(MultiHeadLatentAttentionWrapper):
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
hidden_states: torch.Tensor | QuantizedActivation,
|
||||
llama_4_scaling: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
q_c = None
|
||||
num_tokens = (
|
||||
hidden_states.orig_shape[0]
|
||||
if isinstance(hidden_states, QuantizedActivation)
|
||||
else hidden_states.shape[0]
|
||||
)
|
||||
|
||||
if self.q_lora_rank is not None:
|
||||
assert self.fused_qkv_a_proj is not None, (
|
||||
@@ -238,7 +263,7 @@ class KimiK3MultiHeadLatentAttentionWrapper(MultiHeadLatentAttentionWrapper):
|
||||
if self.dcp_q_replicate:
|
||||
q_dcp_replicated, q = q, q_proj_layer._local_view(q)
|
||||
|
||||
output_shape = (hidden_states.shape[0], self.num_heads * self.v_head_dim)
|
||||
output_shape = (num_tokens, self.num_heads * self.v_head_dim)
|
||||
|
||||
# The fused decode path covers only the MQA slice, so it is taken for
|
||||
# decode-only batches; anything else falls through to MLAAttention.
|
||||
|
||||
@@ -22,6 +22,7 @@ def _attn_res_kernel(
|
||||
qk_weight_ptr,
|
||||
output_norm_weight_ptr,
|
||||
output_ptr,
|
||||
output_scale_ptr,
|
||||
stride_prefix_m: tl.constexpr,
|
||||
stride_delta_m: tl.constexpr,
|
||||
stride_block_m: tl.constexpr,
|
||||
@@ -35,6 +36,7 @@ def _attn_res_kernel(
|
||||
HAS_DELTA: tl.constexpr,
|
||||
WRITE_BLOCK: tl.constexpr,
|
||||
APPLY_OUTPUT_NORM: tl.constexpr,
|
||||
QUANT_MAX: tl.constexpr,
|
||||
BLOCK_L: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
):
|
||||
@@ -129,6 +131,14 @@ def _attn_res_kernel(
|
||||
output_norm_weight_ptr + d_offsets, mask=d_mask, other=0.0
|
||||
).to(tl.float32)
|
||||
output = mixed * output_reciprocal_std * output_norm_weight
|
||||
if QUANT_MAX > 0:
|
||||
# Preserve the rounding of the original BF16 output before quantizing.
|
||||
output = output.to(prefix_ptr.dtype.element_ty).to(tl.float32)
|
||||
amax = tl.max(tl.where(d_mask, tl.abs(output), 0.0), axis=0)
|
||||
scale = tl.maximum(amax, 1e-10) * (1.0 / QUANT_MAX)
|
||||
inv_scale = 1.0 / scale
|
||||
output = tl.minimum(tl.maximum(output * inv_scale, -QUANT_MAX), QUANT_MAX)
|
||||
tl.store(output_scale_ptr + row_idx, scale)
|
||||
tl.store(
|
||||
output_ptr + row_idx * stride_output_m + d_offsets,
|
||||
output,
|
||||
@@ -147,7 +157,9 @@ def attn_res(
|
||||
block_write_idx: int,
|
||||
eps: float,
|
||||
output_norm_eps: float,
|
||||
) -> torch.Tensor:
|
||||
*,
|
||||
quant_dtype: torch.dtype | None = None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
num_tokens, hidden_size = prefix.shape
|
||||
assert prefix.stride(-1) == 1
|
||||
assert delta is None or delta.stride(-1) == 1
|
||||
@@ -155,9 +167,18 @@ def attn_res(
|
||||
assert norm_weight.stride(-1) == 1
|
||||
assert qk_weight.stride(-1) == 1
|
||||
assert output_norm_weight is None or output_norm_weight.stride(-1) == 1
|
||||
output = prefix.new_empty(prefix.shape)
|
||||
if quant_dtype is not None:
|
||||
assert quant_dtype in (torch.float8_e4m3fn, torch.float8_e4m3fnuz)
|
||||
output = torch.empty_like(
|
||||
prefix, dtype=quant_dtype or prefix.dtype, memory_format=torch.contiguous_format
|
||||
)
|
||||
scale = (
|
||||
torch.empty((num_tokens, 1), device=prefix.device, dtype=torch.float32)
|
||||
if quant_dtype is not None
|
||||
else None
|
||||
)
|
||||
if num_tokens == 0:
|
||||
return output
|
||||
return output if scale is None else (output, scale)
|
||||
|
||||
# Source tiling helps decode, while one-source tiles scale better for prefill.
|
||||
if num_tokens >= 256 or num_blocks <= 1:
|
||||
@@ -172,6 +193,7 @@ def attn_res(
|
||||
qk_weight,
|
||||
output_norm_weight,
|
||||
output,
|
||||
scale,
|
||||
prefix.stride(0),
|
||||
0 if delta is None else delta.stride(0),
|
||||
blocks.stride(0),
|
||||
@@ -185,9 +207,10 @@ def attn_res(
|
||||
HAS_DELTA=delta is not None,
|
||||
WRITE_BLOCK=block_write_idx >= 0,
|
||||
APPLY_OUTPUT_NORM=output_norm_weight is not None,
|
||||
QUANT_MAX=0.0 if quant_dtype is None else torch.finfo(quant_dtype).max,
|
||||
BLOCK_L=block_l,
|
||||
BLOCK_D=triton.next_power_of_2(hidden_size),
|
||||
num_warps=num_warps,
|
||||
num_stages=2,
|
||||
)
|
||||
return output
|
||||
return output if scale is None else (output, scale)
|
||||
|
||||
Reference in New Issue
Block a user