[ROCm][Kimi-K3] Fuse AttnRes output with per-token FP8 input quantization (#59069)
Signed-off-by: whx-sjtu <xiaowang990929@gmail.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Fangzhou Ai <31551580+Fangzhou-Ai@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Fangzhou Ai
parent
73c742b09c
commit
458ba2edf8
@@ -0,0 +1,165 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Eight-GPU AttnRes+FP8 projection comparison with rotated graph timing."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import statistics
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.models.kimi_k3.amd.ops.attn_res import attn_res
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def capture(fn, repeats):
|
||||
stream = torch.cuda.Stream()
|
||||
stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(stream):
|
||||
for _ in range(5):
|
||||
fn()
|
||||
torch.cuda.current_stream().wait_stream(stream)
|
||||
torch.accelerator.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
for _ in range(repeats):
|
||||
fn()
|
||||
return graph
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument(
|
||||
"--tokens", type=int, nargs="+", default=[1, 4, 16, 64, 256, 1024, 4096, 16384]
|
||||
)
|
||||
parser.add_argument("--blocks", type=int, nargs="+", default=[1, 4, 8])
|
||||
args = parser.parse_args()
|
||||
assert int(os.environ["WORLD_SIZE"]) == 8
|
||||
assert torch.accelerator.device_count() == 8
|
||||
rank = int(os.environ["LOCAL_RANK"])
|
||||
torch.accelerator.set_device_index(rank)
|
||||
torch.set_num_threads(1)
|
||||
torch.manual_seed(1729 + rank)
|
||||
dist.init_process_group("gloo")
|
||||
args.output.mkdir(parents=True, exist_ok=True)
|
||||
dtype = current_platform.fp8_dtype()
|
||||
rows = []
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
weight = torch.randn(2112, 7168, device="cuda", dtype=torch.bfloat16) * 0.02
|
||||
weight, ws = rocm_aiter_ops.per_token_quant(weight, dtype)
|
||||
weight = rocm_aiter_ops.shuffle_weight(weight)
|
||||
for m in args.tokens:
|
||||
prefix = torch.randn(m, 7168, device="cuda", dtype=torch.bfloat16)
|
||||
delta = torch.zeros_like(prefix)
|
||||
blocks = torch.randn(m, 9, 7168, device="cuda", dtype=torch.bfloat16)
|
||||
norm = torch.ones(7168, device="cuda", dtype=torch.bfloat16)
|
||||
score = torch.randn_like(norm) / 7168**0.5
|
||||
for b in args.blocks:
|
||||
|
||||
def produce(
|
||||
fused,
|
||||
prefix=prefix,
|
||||
delta=delta,
|
||||
blocks=blocks,
|
||||
norm=norm,
|
||||
score=score,
|
||||
b=b,
|
||||
):
|
||||
result = attn_res(
|
||||
prefix,
|
||||
delta,
|
||||
blocks,
|
||||
norm,
|
||||
score,
|
||||
norm,
|
||||
b,
|
||||
-1,
|
||||
1e-5,
|
||||
1e-5,
|
||||
quant_dtype=dtype if fused else None,
|
||||
)
|
||||
return (
|
||||
result
|
||||
if fused
|
||||
else rocm_aiter_ops.per_token_quant(result, dtype)
|
||||
)
|
||||
|
||||
def segment(fused, produce=produce):
|
||||
x, scale = produce(fused)
|
||||
return rocm_aiter_ops.preshuffled_per_token_w8a8_gemm(
|
||||
x, weight, scale, ws, output_dtype=torch.bfloat16
|
||||
)
|
||||
|
||||
expected, actual = segment(False), segment(True)
|
||||
relative_l2 = (
|
||||
(actual.float() - expected.float()).norm() / expected.float().norm()
|
||||
).item()
|
||||
assert relative_l2 < 0.002, relative_l2
|
||||
repeats = 20 if m < 1024 else 4
|
||||
functions = {
|
||||
"baseline_producer": lambda produce=produce: produce(False),
|
||||
"fused_producer": lambda produce=produce: produce(True),
|
||||
"baseline_segment": lambda segment=segment: segment(False),
|
||||
"fused_segment": lambda segment=segment: segment(True),
|
||||
}
|
||||
graphs = {k: capture(fn, repeats) for k, fn in functions.items()}
|
||||
samples = {k: [] for k in functions}
|
||||
names = list(functions)
|
||||
for sample in range(9):
|
||||
for name in names[sample % 4 :] + names[: sample % 4]:
|
||||
dist.barrier()
|
||||
graphs[name].replay()
|
||||
torch.accelerator.synchronize()
|
||||
start, end = [
|
||||
torch.cuda.Event(enable_timing=True) for _ in range(2)
|
||||
]
|
||||
start.record()
|
||||
graphs[name].replay()
|
||||
end.record()
|
||||
end.synchronize()
|
||||
samples[name].append(start.elapsed_time(end) * 1000 / repeats)
|
||||
row = {
|
||||
"tokens": m,
|
||||
"blocks": b,
|
||||
"relative_l2": relative_l2,
|
||||
"samples_us": samples,
|
||||
"median_us": {k: statistics.median(v) for k, v in samples.items()},
|
||||
}
|
||||
rows.append(row)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"rank": rank,
|
||||
"tokens": m,
|
||||
"blocks": b,
|
||||
"median_us": row["median_us"],
|
||||
}
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
del graphs
|
||||
(args.output / f"rank-{rank}.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"rank": rank,
|
||||
"world_size": 8,
|
||||
"torch": torch.__version__,
|
||||
"rows": rows,
|
||||
"scope": "AttnRes + FP8 quant + TP8 MLA input projection",
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
dist.barrier()
|
||||
dist.destroy_process_group()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -191,3 +191,102 @@ def test_amd_attn_res_fused_contract(
|
||||
original_blocks[:, block_write_idx].copy_(expected_prefix)
|
||||
torch.testing.assert_close(blocks, original_blocks, atol=0, rtol=0)
|
||||
assert actual.is_contiguous()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"num_tokens,num_blocks,has_delta,write_block",
|
||||
[
|
||||
(0, 0, False, False),
|
||||
(1, 0, True, True),
|
||||
(17, 4, True, True),
|
||||
(320, 8, False, False),
|
||||
],
|
||||
)
|
||||
def test_amd_attn_res_fp8_preserves_prefix_and_quantized_output(
|
||||
num_tokens,
|
||||
num_blocks,
|
||||
has_delta,
|
||||
write_block,
|
||||
default_vllm_config,
|
||||
):
|
||||
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
|
||||
|
||||
torch.manual_seed(41)
|
||||
hidden_size = 7168
|
||||
prefix = _randn_with_row_padding(num_tokens, hidden_size, padding=7)
|
||||
delta = _randn_with_row_padding(num_tokens, hidden_size) if has_delta else None
|
||||
blocks = _randn_with_row_padding(num_tokens, 9, hidden_size, padding=13)
|
||||
norm = 1 + torch.randn(hidden_size, device="cuda", dtype=torch.bfloat16) * 0.1
|
||||
score = torch.randn_like(norm) / hidden_size**0.5
|
||||
out_norm = 1 + torch.randn_like(norm) * 0.1
|
||||
ref_prefix, ref_blocks = prefix.clone(), blocks.clone()
|
||||
kwargs = dict(
|
||||
num_blocks=num_blocks,
|
||||
block_write_idx=num_blocks if write_block else -1,
|
||||
eps=1e-5,
|
||||
output_norm_eps=1e-5,
|
||||
)
|
||||
reference = attn_res(ref_prefix, delta, ref_blocks, norm, score, out_norm, **kwargs)
|
||||
output, scale = attn_res(
|
||||
prefix,
|
||||
delta,
|
||||
blocks,
|
||||
norm,
|
||||
score,
|
||||
out_norm,
|
||||
quant_dtype=current_platform.fp8_dtype(),
|
||||
**kwargs,
|
||||
)
|
||||
assert output.dtype == current_platform.fp8_dtype()
|
||||
assert scale.shape == (num_tokens, 1)
|
||||
if num_tokens:
|
||||
# Keep reference rounding independent of Inductor's fused quantization.
|
||||
quant = QuantFP8(
|
||||
static=False, group_shape=GroupShape.PER_TOKEN, compile_native=False
|
||||
)
|
||||
ref_output, ref_scale = quant(reference)
|
||||
torch.testing.assert_close(scale, ref_scale, atol=1e-7, rtol=1e-6)
|
||||
# Floating-point fusion can move values at FP8 rounding midpoints.
|
||||
# Bound each change to the adjacent code and bound how often it occurs.
|
||||
code_delta = (
|
||||
output.view(torch.uint8).to(torch.int16)
|
||||
- ref_output.view(torch.uint8).to(torch.int16)
|
||||
).abs()
|
||||
assert code_delta.max().item() <= 1
|
||||
assert (code_delta != 0).float().mean().item() < 1e-4
|
||||
torch.testing.assert_close(prefix, ref_prefix, atol=0, rtol=0)
|
||||
torch.testing.assert_close(blocks, ref_blocks, atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"matching,requires_unquantized", [(True, False), (False, False), (True, True)]
|
||||
)
|
||||
def test_mla_input_quant_requires_all_consumers(matching, requires_unquantized):
|
||||
from types import SimpleNamespace
|
||||
|
||||
from torch import nn
|
||||
|
||||
from vllm.model_executor.layers.fusion.quant_activation import (
|
||||
expose_input_quant_key,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8DynamicTokenSym,
|
||||
)
|
||||
from vllm.models.kimi_k3.amd.mla import KimiK3MultiHeadLatentAttentionWrapper
|
||||
|
||||
attention = object.__new__(KimiK3MultiHeadLatentAttentionWrapper)
|
||||
nn.Module.__init__(attention)
|
||||
attention.indexer = None
|
||||
attention.q_lora_rank = 1536
|
||||
attention.fused_qkv_a_proj = nn.Identity()
|
||||
attention.g_proj = nn.Identity()
|
||||
kernel = SimpleNamespace(input_quant_key=lambda: kFp8DynamicTokenSym)
|
||||
expose_input_quant_key(attention.fused_qkv_a_proj, kernel)
|
||||
if matching:
|
||||
expose_input_quant_key(attention.g_proj, kernel)
|
||||
attention.g_proj.requires_unquantized_input = requires_unquantized
|
||||
actual = attention.get_input_quant_key()
|
||||
assert actual == (
|
||||
kFp8DynamicTokenSym if matching and not requires_unquantized else None
|
||||
)
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
@@ -36,3 +39,81 @@ def test_mla_self_attention_returns_projection_storage_directly() -> None:
|
||||
|
||||
assert output is projected
|
||||
assert output.data_ptr() == projected.data_ptr()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("quantized", [False, True])
|
||||
@pytest.mark.parametrize("fused_decode", [False, True])
|
||||
def test_mla_forward_preserves_output_shape_with_fp8_input(
|
||||
monkeypatch, quantized: bool, fused_decode: bool
|
||||
) -> None:
|
||||
"""Both MLA paths accept producer-quantized input after decode fusion."""
|
||||
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8DynamicTokenSym,
|
||||
)
|
||||
from vllm.models.kimi_k3.amd.mla import KimiK3MultiHeadLatentAttentionWrapper
|
||||
|
||||
hidden_states = torch.randn(4, 8, dtype=torch.bfloat16)
|
||||
if quantized:
|
||||
hidden_states = QuantizedActivation(
|
||||
hidden_states.to(kFp8DynamicTokenSym.dtype),
|
||||
torch.ones(4, 1),
|
||||
hidden_states.dtype,
|
||||
hidden_states.shape,
|
||||
kFp8DynamicTokenSym,
|
||||
)
|
||||
projected = torch.randn(4, 8, dtype=torch.bfloat16)
|
||||
paths = []
|
||||
|
||||
def input_projection(value):
|
||||
assert value is hidden_states
|
||||
return torch.zeros(4, 8, dtype=torch.bfloat16), None
|
||||
|
||||
def attention_output(path, output_shape):
|
||||
assert output_shape == projected.shape
|
||||
paths.append(path)
|
||||
return projected
|
||||
|
||||
attention = SimpleNamespace(
|
||||
q_lora_rank=2,
|
||||
kv_lora_rank=4,
|
||||
qk_rope_head_dim=2,
|
||||
qk_head_dim=4,
|
||||
num_heads=2,
|
||||
v_head_dim=4,
|
||||
fused_qkv_a_proj=input_projection,
|
||||
q_a_layernorm=nn.Identity(),
|
||||
_normalize_q_kv=lambda q, kv: (q, kv),
|
||||
q_b_proj=lambda q: (torch.zeros(4, 8, dtype=q.dtype), None),
|
||||
rotary_emb=None,
|
||||
indexer=None,
|
||||
dcp_q_replicate=False,
|
||||
_fused_qk_prep=fused_decode,
|
||||
_fused_decode=lambda q, kv, pe, pos, shape, *args: attention_output(
|
||||
"fused", shape
|
||||
),
|
||||
g_proj=None,
|
||||
o_proj=lambda value: (value, None),
|
||||
)
|
||||
if fused_decode:
|
||||
attention.mla_attn = SimpleNamespace(layer_name="test")
|
||||
monkeypatch.setattr(
|
||||
"vllm.models.kimi_k3.amd.mla.get_attention_context",
|
||||
lambda _: (
|
||||
SimpleNamespace(num_actual_tokens=4, num_decode_tokens=4),
|
||||
None,
|
||||
torch.zeros(1),
|
||||
torch.arange(4),
|
||||
),
|
||||
)
|
||||
else:
|
||||
attention.mla_attn = lambda q, kv, pe, **kwargs: attention_output(
|
||||
"regular", kwargs["output_shape"]
|
||||
)
|
||||
|
||||
output = KimiK3MultiHeadLatentAttentionWrapper.forward(
|
||||
attention, torch.arange(4), hidden_states
|
||||
)
|
||||
|
||||
assert output is projected
|
||||
assert paths == ["fused" if fused_decode else "regular"]
|
||||
|
||||
@@ -13,6 +13,10 @@ from vllm.config import VllmConfig
|
||||
from vllm.distributed import divide
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fusion.quant_activation import (
|
||||
QuantizedActivation,
|
||||
get_input_quant_key,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
@@ -37,6 +41,7 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
|
||||
causal_conv1d_fn,
|
||||
causal_conv1d_update,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
|
||||
from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.models.kimi_k3.amd.kda_metadata import KimiK3ROCmKDABackend
|
||||
@@ -300,13 +305,16 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
qkv = qkv.permute(1, 0, 2, 3).contiguous().unsqueeze(1)
|
||||
return qkv.unbind(0)
|
||||
|
||||
def get_input_quant_key(self) -> QuantKey | None:
|
||||
return get_input_quant_key(self.in_proj_qkvgfab)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
hidden_states: torch.Tensor | QuantizedActivation,
|
||||
positions: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
num_tokens = hidden_states.size(0)
|
||||
projected_qkvgfab = self.in_proj_qkvgfab(hidden_states)[0]
|
||||
num_tokens = projected_qkvgfab.shape[0]
|
||||
|
||||
split_sizes = [
|
||||
3 * self.local_projection_size,
|
||||
@@ -326,8 +334,8 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
|
||||
core_attn_out = torch.empty(
|
||||
(1, num_tokens, self.local_num_heads, self.head_dim),
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
dtype=projected_qkvgfab.dtype,
|
||||
device=projected_qkvgfab.device,
|
||||
)
|
||||
|
||||
self._forward(
|
||||
|
||||
@@ -2,11 +2,12 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
from typing import Any, cast
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm import envs
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_pp_group,
|
||||
@@ -19,6 +20,10 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
|
||||
from vllm.model_executor.layers.fusion.quant_activation import (
|
||||
QuantizedActivation,
|
||||
get_input_quant_key,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -38,6 +43,10 @@ from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
)
|
||||
from vllm.model_executor.layers.mla import MLAModules
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kFp8DynamicTokenSym,
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
@@ -114,6 +123,9 @@ class KimiMLP(nn.Module):
|
||||
"Only silu and situ are supported."
|
||||
)
|
||||
|
||||
def get_input_quant_key(self) -> QuantKey | None:
|
||||
return get_input_quant_key(self.gate_up_proj)
|
||||
|
||||
def forward(self, x):
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
x = self.act_fn(gate_up)
|
||||
@@ -148,8 +160,10 @@ def _apply_attn_res(
|
||||
delta: torch.Tensor | None = None,
|
||||
output_norm: RMSNorm | None = None,
|
||||
block_write_idx: int = -1,
|
||||
) -> torch.Tensor:
|
||||
return attn_res(
|
||||
quant_key: QuantKey | None = None,
|
||||
) -> torch.Tensor | QuantizedActivation:
|
||||
fuse_quant = quant_key == kFp8DynamicTokenSym and not envs.VLLM_BATCH_INVARIANT
|
||||
result = attn_res(
|
||||
prefix_sum,
|
||||
delta,
|
||||
block_residual,
|
||||
@@ -160,7 +174,18 @@ def _apply_attn_res(
|
||||
block_write_idx,
|
||||
norm.variance_epsilon,
|
||||
0.0 if output_norm is None else output_norm.variance_epsilon,
|
||||
quant_dtype=cast(torch.dtype, kFp8DynamicTokenSym.dtype)
|
||||
if fuse_quant
|
||||
else None,
|
||||
)
|
||||
if fuse_quant:
|
||||
assert isinstance(result, tuple)
|
||||
data, scale = result
|
||||
return QuantizedActivation(
|
||||
data, scale, prefix_sum.dtype, prefix_sum.shape, kFp8DynamicTokenSym
|
||||
)
|
||||
assert isinstance(result, torch.Tensor)
|
||||
return result
|
||||
|
||||
|
||||
class KimiMoE(nn.Module):
|
||||
@@ -452,10 +477,13 @@ class KimiMLAAttention(nn.Module):
|
||||
prefix,
|
||||
)
|
||||
|
||||
def get_input_quant_key(self) -> QuantKey | None:
|
||||
return self.mla_attn.get_input_quant_key()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
hidden_states: torch.Tensor | QuantizedActivation,
|
||||
) -> torch.Tensor:
|
||||
return self.mla_attn(positions, hidden_states)
|
||||
|
||||
@@ -579,7 +607,7 @@ class KimiDecoderLayer(nn.Module):
|
||||
def _run_self_attn(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
hidden_states: torch.Tensor | QuantizedActivation,
|
||||
) -> torch.Tensor:
|
||||
return self.self_attn(
|
||||
hidden_states=hidden_states,
|
||||
@@ -626,6 +654,11 @@ class KimiDecoderLayer(nn.Module):
|
||||
prefix_delta: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
prefix_sum = hidden_states
|
||||
attention_quant_key = (
|
||||
self.self_attn.get_input_quant_key()
|
||||
if isinstance(self.self_attn, (KimiK3DeltaAttention, KimiMLAAttention))
|
||||
else None
|
||||
)
|
||||
hidden_states = _apply_attn_res(
|
||||
prefix_sum,
|
||||
block_residual,
|
||||
@@ -635,6 +668,7 @@ class KimiDecoderLayer(nn.Module):
|
||||
delta=prefix_delta,
|
||||
output_norm=self.input_layernorm,
|
||||
block_write_idx=(self.block_write_idx if self.is_block_write_layer else -1),
|
||||
quant_key=attention_quant_key,
|
||||
)
|
||||
|
||||
if self.is_block_write_layer:
|
||||
@@ -659,6 +693,11 @@ class KimiDecoderLayer(nn.Module):
|
||||
mlp_valid_blocks,
|
||||
delta=prefix_delta,
|
||||
output_norm=self.post_attention_layernorm,
|
||||
quant_key=(
|
||||
self.mlp.get_input_quant_key()
|
||||
if isinstance(self.mlp, KimiMLP)
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
|
||||
@@ -8,8 +8,13 @@ import torch
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.layers.attention.attention import get_attention_context
|
||||
from vllm.model_executor.layers.fusion.quant_activation import (
|
||||
QuantizedActivation,
|
||||
get_input_quant_key,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.mla import MultiHeadLatentAttentionWrapper
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
_OPT_KV_LORA_RANK = 512
|
||||
@@ -49,6 +54,21 @@ class KimiK3MultiHeadLatentAttentionWrapper(MultiHeadLatentAttentionWrapper):
|
||||
not attn.is_aiter_triton_fp8_bmm_enabled and attn.W_UK_T is not None
|
||||
)
|
||||
|
||||
def get_input_quant_key(self) -> QuantKey | None:
|
||||
if self.indexer is not None:
|
||||
return None
|
||||
consumers = (
|
||||
[self.fused_qkv_a_proj]
|
||||
if self.q_lora_rank is not None
|
||||
else [self.kv_a_proj_with_mqa, self.q_proj]
|
||||
)
|
||||
if self.g_proj is not None:
|
||||
consumers.append(self.g_proj)
|
||||
if any(layer is None for layer in consumers):
|
||||
return None
|
||||
keys = [get_input_quant_key(layer) for layer in consumers if layer is not None]
|
||||
return keys[0] if all(key == keys[0] for key in keys) else None
|
||||
|
||||
def _normalize_q_kv(
|
||||
self,
|
||||
q_c: torch.Tensor,
|
||||
@@ -173,10 +193,15 @@ class KimiK3MultiHeadLatentAttentionWrapper(MultiHeadLatentAttentionWrapper):
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
hidden_states: torch.Tensor | QuantizedActivation,
|
||||
llama_4_scaling: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
q_c = None
|
||||
num_tokens = (
|
||||
hidden_states.orig_shape[0]
|
||||
if isinstance(hidden_states, QuantizedActivation)
|
||||
else hidden_states.shape[0]
|
||||
)
|
||||
|
||||
if self.q_lora_rank is not None:
|
||||
assert self.fused_qkv_a_proj is not None, (
|
||||
@@ -238,7 +263,7 @@ class KimiK3MultiHeadLatentAttentionWrapper(MultiHeadLatentAttentionWrapper):
|
||||
if self.dcp_q_replicate:
|
||||
q_dcp_replicated, q = q, q_proj_layer._local_view(q)
|
||||
|
||||
output_shape = (hidden_states.shape[0], self.num_heads * self.v_head_dim)
|
||||
output_shape = (num_tokens, self.num_heads * self.v_head_dim)
|
||||
|
||||
# The fused decode path covers only the MQA slice, so it is taken for
|
||||
# decode-only batches; anything else falls through to MLAAttention.
|
||||
|
||||
@@ -22,6 +22,7 @@ def _attn_res_kernel(
|
||||
qk_weight_ptr,
|
||||
output_norm_weight_ptr,
|
||||
output_ptr,
|
||||
output_scale_ptr,
|
||||
stride_prefix_m: tl.constexpr,
|
||||
stride_delta_m: tl.constexpr,
|
||||
stride_block_m: tl.constexpr,
|
||||
@@ -35,6 +36,7 @@ def _attn_res_kernel(
|
||||
HAS_DELTA: tl.constexpr,
|
||||
WRITE_BLOCK: tl.constexpr,
|
||||
APPLY_OUTPUT_NORM: tl.constexpr,
|
||||
QUANT_MAX: tl.constexpr,
|
||||
BLOCK_L: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
):
|
||||
@@ -129,6 +131,14 @@ def _attn_res_kernel(
|
||||
output_norm_weight_ptr + d_offsets, mask=d_mask, other=0.0
|
||||
).to(tl.float32)
|
||||
output = mixed * output_reciprocal_std * output_norm_weight
|
||||
if QUANT_MAX > 0:
|
||||
# Preserve the rounding of the original BF16 output before quantizing.
|
||||
output = output.to(prefix_ptr.dtype.element_ty).to(tl.float32)
|
||||
amax = tl.max(tl.where(d_mask, tl.abs(output), 0.0), axis=0)
|
||||
scale = tl.maximum(amax, 1e-10) * (1.0 / QUANT_MAX)
|
||||
inv_scale = 1.0 / scale
|
||||
output = tl.minimum(tl.maximum(output * inv_scale, -QUANT_MAX), QUANT_MAX)
|
||||
tl.store(output_scale_ptr + row_idx, scale)
|
||||
tl.store(
|
||||
output_ptr + row_idx * stride_output_m + d_offsets,
|
||||
output,
|
||||
@@ -147,7 +157,9 @@ def attn_res(
|
||||
block_write_idx: int,
|
||||
eps: float,
|
||||
output_norm_eps: float,
|
||||
) -> torch.Tensor:
|
||||
*,
|
||||
quant_dtype: torch.dtype | None = None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
num_tokens, hidden_size = prefix.shape
|
||||
assert prefix.stride(-1) == 1
|
||||
assert delta is None or delta.stride(-1) == 1
|
||||
@@ -155,9 +167,18 @@ def attn_res(
|
||||
assert norm_weight.stride(-1) == 1
|
||||
assert qk_weight.stride(-1) == 1
|
||||
assert output_norm_weight is None or output_norm_weight.stride(-1) == 1
|
||||
output = prefix.new_empty(prefix.shape)
|
||||
if quant_dtype is not None:
|
||||
assert quant_dtype in (torch.float8_e4m3fn, torch.float8_e4m3fnuz)
|
||||
output = torch.empty_like(
|
||||
prefix, dtype=quant_dtype or prefix.dtype, memory_format=torch.contiguous_format
|
||||
)
|
||||
scale = (
|
||||
torch.empty((num_tokens, 1), device=prefix.device, dtype=torch.float32)
|
||||
if quant_dtype is not None
|
||||
else None
|
||||
)
|
||||
if num_tokens == 0:
|
||||
return output
|
||||
return output if scale is None else (output, scale)
|
||||
|
||||
# Source tiling helps decode, while one-source tiles scale better for prefill.
|
||||
if num_tokens >= 256 or num_blocks <= 1:
|
||||
@@ -172,6 +193,7 @@ def attn_res(
|
||||
qk_weight,
|
||||
output_norm_weight,
|
||||
output,
|
||||
scale,
|
||||
prefix.stride(0),
|
||||
0 if delta is None else delta.stride(0),
|
||||
blocks.stride(0),
|
||||
@@ -185,9 +207,10 @@ def attn_res(
|
||||
HAS_DELTA=delta is not None,
|
||||
WRITE_BLOCK=block_write_idx >= 0,
|
||||
APPLY_OUTPUT_NORM=output_norm_weight is not None,
|
||||
QUANT_MAX=0.0 if quant_dtype is None else torch.finfo(quant_dtype).max,
|
||||
BLOCK_L=block_l,
|
||||
BLOCK_D=triton.next_power_of_2(hidden_size),
|
||||
num_warps=num_warps,
|
||||
num_stages=2,
|
||||
)
|
||||
return output
|
||||
return output if scale is None else (output, scale)
|
||||
|
||||
Reference in New Issue
Block a user