Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
1066 lines
34 KiB
Python
1066 lines
34 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""Kimi-K3 decode GEMM selection for unquantized BF16 on SM90/SM100/SM103/SM107.
|
|
|
|
Dispatch is purely by local ``(N, K)`` shape and token count ``M`` — the module
|
|
name plays no role. Each measured shape maps to a :class:`ProjectionSpec`
|
|
holding the winning backend per token count. The static part of the decision is
|
|
resolved once per module at install time into a small ``{M: call}`` plan, so the
|
|
per-forward path is a single dict lookup.
|
|
|
|
The supported capabilities carry separate measured tables:
|
|
:data:`KIMI_K3_PROJECTIONS` was tuned on B300 (SM103),
|
|
:data:`KIMI_K3_PROJECTIONS_SM100` on B200 (SM100), and
|
|
:data:`KIMI_K3_PROJECTIONS_SM90` on H200 (SM90). The per-(shape, M) winners
|
|
genuinely differ between the parts, so the tables must not be merged. SM107
|
|
(Rubin) reuses the SM103 table: the plan was validated end-to-end on SM107
|
|
hardware, but the per-M crossovers have not been re-measured there and may
|
|
deserve their own table once retuned.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from typing import Literal
|
|
|
|
import torch
|
|
from torch import nn
|
|
|
|
import vllm.envs as envs
|
|
from vllm import _custom_ops as ops
|
|
from vllm.model_executor.kernels.linear.cute_dsl.skinny_gemm import (
|
|
SkinnyGemmConfig,
|
|
shape_dynamic_skinny_gemm,
|
|
)
|
|
from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod
|
|
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
|
ParallelLMHead,
|
|
UnquantizedEmbeddingMethod,
|
|
)
|
|
from vllm.platforms import current_platform
|
|
from vllm.utils.multi_stream_utils import maybe_execute_in_parallel
|
|
|
|
Backend = Literal["cute", "dsv3_fused_a"]
|
|
# A resolved per-token-count call: the backend plus its CuTe config (None for
|
|
# dsv3, which needs no config).
|
|
ResolvedCall = tuple[Backend, SkinnyGemmConfig | None]
|
|
|
|
# TP8 KDA projection split, measured together under CUDA graph capture on B300.
|
|
# Q/K/V/G stay on the graph's main stream while F_A/beta and F_B run on the
|
|
# model's auxiliary stream.
|
|
KDA_M1_QKVG_CONFIG = SkinnyGemmConfig(1, 64, 4, 2, 8)
|
|
KDA_M1_FAB_CONFIG = SkinnyGemmConfig(1, 224, 1, 2, 8)
|
|
KDA_QKVG_CONFIGS = {
|
|
1: KDA_M1_QKVG_CONFIG,
|
|
2: SkinnyGemmConfig(2, 64, 3, 2, 8),
|
|
}
|
|
# The captured end-to-end projection sweep wins through M=14 and regresses at
|
|
# M=15 and M=16, where the original packed projection remains selected.
|
|
KDA_PROJECTION_OVERLAP_MAX_TOKENS = 14
|
|
KDA_SKINNY_N_MAX_TOKENS = KDA_PROJECTION_OVERLAP_MAX_TOKENS
|
|
KDA_SKINNY_K_MAX_TOKENS = KDA_PROJECTION_OVERLAP_MAX_TOKENS
|
|
_KDA_QKVG_SIZE = 4 * 1536
|
|
_KDA_FAB_SIZE = 128 + 12
|
|
_KDA_PACKED_SIZE = 6288
|
|
_KDA_TP_SIZE = 8
|
|
|
|
|
|
def _kda_qkvg_flashinfer_backend() -> Literal["cute-dsl"] | None:
|
|
"""Return the tested FlashInfer backend for the low-M QKVG branch.
|
|
|
|
FlashInfer's ``cute-dsl`` BF16 GEMM supports SM100 and SM103, but rejects
|
|
SM90 during backend validation. Other capabilities use ``torch.mm`` for
|
|
QKVG while retaining the concurrent F_A/beta and F_B branch.
|
|
"""
|
|
if current_platform.is_device_capability(
|
|
(10, 0)
|
|
) or current_platform.is_device_capability((10, 3)):
|
|
return "cute-dsl"
|
|
return None
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class ProjectionSpec:
|
|
n: int
|
|
k: int
|
|
dsv3_tokens: frozenset[int] = frozenset()
|
|
cute_configs: tuple[tuple[int, SkinnyGemmConfig], ...] = ()
|
|
residual_configs: tuple[tuple[int, SkinnyGemmConfig], ...] = ()
|
|
name: str = "" # optional debug label; never used for dispatch
|
|
|
|
def cute_config(self, num_tokens: int) -> SkinnyGemmConfig | None:
|
|
return dict(self.cute_configs).get(num_tokens)
|
|
|
|
def residual_config(self, num_tokens: int) -> SkinnyGemmConfig | None:
|
|
return dict(self.residual_configs).get(num_tokens)
|
|
|
|
|
|
def _cute(
|
|
num_tokens: int,
|
|
block_size: int,
|
|
outputs_per_block: int,
|
|
k_unroll: int,
|
|
vector_width: int = 8,
|
|
) -> SkinnyGemmConfig:
|
|
return SkinnyGemmConfig(
|
|
num_tokens,
|
|
block_size,
|
|
outputs_per_block,
|
|
k_unroll,
|
|
vector_width,
|
|
)
|
|
|
|
|
|
_M1_TO_16 = frozenset(range(1, 17))
|
|
_M1 = frozenset({1})
|
|
|
|
# Keyed by local (N, K). Where two projections share a shape (only 1536x7168:
|
|
# shared_gate_up_proj and mla_g_proj) the entry is unified.
|
|
KIMI_K3_PROJECTIONS: dict[tuple[int, int], ProjectionSpec] = {
|
|
(1536, 128): ProjectionSpec(1536, 128, _M1_TO_16, name="f_b_proj"),
|
|
(3072, 128): ProjectionSpec(3072, 128, _M1_TO_16, name="f_b_proj"),
|
|
# 1536x7168 is shared by shared_gate_up_proj and mla_g_proj. dsv3 M1..16 is
|
|
# only crash-safe once the mla_g aux-stream/PDL capture fix lands (subtask
|
|
# task_7388aba1); the fallback if it cannot be fixed is dsv3_tokens=_M1.
|
|
(1536, 7168): ProjectionSpec(
|
|
1536, 7168, _M1_TO_16, name="shared_gate_up_proj/mla_g_proj"
|
|
),
|
|
(3072, 7168): ProjectionSpec(
|
|
3072,
|
|
7168,
|
|
cute_configs=(
|
|
(1, _cute(1, 224, 3, 4)),
|
|
(2, _cute(2, 128, 3, 2)),
|
|
(3, _cute(3, 128, 2, 1)),
|
|
(4, _cute(4, 64, 2, 2)),
|
|
(5, _cute(5, 128, 3, 1)),
|
|
),
|
|
name="shared_gate_up_proj",
|
|
),
|
|
(2112, 7168): ProjectionSpec(2112, 7168, _M1_TO_16, name="fused_qkv_a_proj"),
|
|
(2304, 1536): ProjectionSpec(2304, 1536, _M1_TO_16, name="q_b_proj"),
|
|
(4608, 1536): ProjectionSpec(4608, 1536, _M1_TO_16, name="q_b_proj"),
|
|
(3584, 7168): ProjectionSpec(
|
|
3584,
|
|
7168,
|
|
frozenset(range(2, 9)),
|
|
((1, _cute(1, 224, 2, 4)),),
|
|
name="routed_expert_down_proj",
|
|
),
|
|
(6288, 7168): ProjectionSpec(
|
|
6288,
|
|
7168,
|
|
cute_configs=(
|
|
(1, _cute(1, 224, 3, 4)),
|
|
(2, _cute(2, 64, 3, 2)),
|
|
(3, _cute(3, 32, 3, 4)),
|
|
(4, _cute(4, 128, 6, 1)),
|
|
),
|
|
name="in_proj_qkvgfab",
|
|
),
|
|
(12448, 7168): ProjectionSpec(
|
|
12448,
|
|
7168,
|
|
cute_configs=(
|
|
(1, _cute(1, 224, 4, 2)),
|
|
(2, _cute(2, 64, 4, 2)),
|
|
(3, _cute(3, 64, 2, 2)),
|
|
),
|
|
name="in_proj_qkvgfab",
|
|
),
|
|
(7168, 768): ProjectionSpec(7168, 768, _M1_TO_16, name="shared_down_proj"),
|
|
(7168, 1536): ProjectionSpec(
|
|
7168, 1536, cute_configs=((1, _cute(1, 96, 4, 2)),), name="o_proj"
|
|
),
|
|
(7168, 3072): ProjectionSpec(
|
|
7168,
|
|
3072,
|
|
cute_configs=(
|
|
(1, _cute(1, 96, 2, 4)),
|
|
(2, _cute(2, 32, 4, 4)),
|
|
),
|
|
name="o_proj",
|
|
),
|
|
(7168, 3584): ProjectionSpec(
|
|
7168,
|
|
3584,
|
|
cute_configs=(
|
|
(1, _cute(1, 224, 4, 2)),
|
|
(2, _cute(2, 64, 4, 2)),
|
|
),
|
|
residual_configs=(
|
|
(1, _cute(1, 64, 4, 2)),
|
|
(2, _cute(2, 64, 7, 2)),
|
|
(3, _cute(3, 64, 2, 1)),
|
|
(4, _cute(4, 64, 2, 1)),
|
|
),
|
|
name="routed_expert_up_proj",
|
|
),
|
|
(7168, 4224): ProjectionSpec(
|
|
7168,
|
|
4224,
|
|
cute_configs=((1, _cute(1, 96, 4, 2, 4)),),
|
|
name="dense_down_proj",
|
|
),
|
|
(7168, 8448): ProjectionSpec(
|
|
7168,
|
|
8448,
|
|
cute_configs=(
|
|
(1, _cute(1, 32, 4, 4)),
|
|
(2, _cute(2, 96, 4, 1)),
|
|
(3, _cute(3, 96, 4, 1)),
|
|
),
|
|
name="dense_down_proj",
|
|
),
|
|
(8448, 7168): ProjectionSpec(
|
|
8448,
|
|
7168,
|
|
cute_configs=(
|
|
(1, _cute(1, 224, 3, 4)),
|
|
(2, _cute(2, 32, 4, 4)),
|
|
),
|
|
name="dense_gate_up_proj",
|
|
),
|
|
(16896, 7168): ProjectionSpec(
|
|
16896,
|
|
7168,
|
|
cute_configs=(
|
|
(1, _cute(1, 224, 6, 4)),
|
|
(2, _cute(2, 32, 4, 4)),
|
|
),
|
|
name="dense_gate_up_proj",
|
|
),
|
|
(7168, 14336): ProjectionSpec(
|
|
7168,
|
|
14336,
|
|
cute_configs=(
|
|
(1, SkinnyGemmConfig(1, 256, 2, vector_width=4, static_k=14336)),
|
|
(2, _cute(2, 224, 4, 2)),
|
|
),
|
|
),
|
|
(20480, 7168): ProjectionSpec(
|
|
20480,
|
|
7168,
|
|
cute_configs=(
|
|
(1, _cute(1, 224, 4, 2)),
|
|
(2, _cute(2, 64, 4, 2)),
|
|
(3, _cute(3, 64, 2, 2)),
|
|
(4, _cute(4, 64, 4, 1)),
|
|
),
|
|
name="lm_head",
|
|
),
|
|
(40960, 7168): ProjectionSpec(
|
|
40960,
|
|
7168,
|
|
cute_configs=(
|
|
(1, _cute(1, 128, 4, 2)),
|
|
(2, _cute(2, 64, 4, 2)),
|
|
(3, _cute(3, 64, 2, 2)),
|
|
(4, _cute(4, 64, 4, 1)),
|
|
),
|
|
name="lm_head",
|
|
),
|
|
# TP16. Measured on B300 over M=1..16 with the same >=5% threshold as the
|
|
# entries above. The replicated projections (2112x7168, 3584x7168,
|
|
# 7168x3584) keep their shapes at TP16 and reuse the entries above, and
|
|
# o_proj lands on 7168x768, which shared_down_proj already covers.
|
|
(3216, 7168): ProjectionSpec(
|
|
3216,
|
|
7168,
|
|
# Both gaps in this range are measured, not oversights: dsv3 is only
|
|
# 4% ahead at M6..M8, and at M16 cuBLAS switches to a faster kernel
|
|
# (11.42us vs dsv3's 11.83us) after trailing it by 6-8% at M9..M15.
|
|
frozenset(range(9, 16)),
|
|
cute_configs=(
|
|
(1, _cute(1, 224, 3, 4)),
|
|
(2, _cute(2, 128, 4, 2)),
|
|
(3, _cute(3, 128, 2, 1)),
|
|
(4, _cute(4, 64, 2, 2)),
|
|
(5, _cute(5, 128, 3, 1)),
|
|
),
|
|
name="in_proj_qkvgfab",
|
|
),
|
|
(768, 7168): ProjectionSpec(
|
|
768,
|
|
7168,
|
|
frozenset(range(5, 17)),
|
|
cute_configs=(
|
|
(1, _cute(1, 224, 2, 4)),
|
|
(2, _cute(2, 224, 2, 2)),
|
|
(3, _cute(3, 224, 2, 2)),
|
|
(4, _cute(4, 224, 2, 2)),
|
|
),
|
|
name="mla_g_proj/shared_gate_up_proj",
|
|
),
|
|
(1152, 1536): ProjectionSpec(
|
|
1152,
|
|
1536,
|
|
frozenset(range(2, 17)),
|
|
((1, _cute(1, 192, 3, 4)),),
|
|
name="q_b_proj",
|
|
),
|
|
(768, 128): ProjectionSpec(768, 128, _M1_TO_16, name="f_b_proj"),
|
|
# dsv3 drops under 5% from M9 on for this shape.
|
|
(7168, 384): ProjectionSpec(
|
|
7168, 384, frozenset(range(1, 9)), name="shared_down_proj"
|
|
),
|
|
(4224, 7168): ProjectionSpec(
|
|
4224,
|
|
7168,
|
|
frozenset(range(4, 9)),
|
|
cute_configs=(
|
|
(1, _cute(1, 224, 3, 4)),
|
|
(2, _cute(2, 128, 2, 1)),
|
|
(3, _cute(3, 64, 2, 2)),
|
|
),
|
|
name="dense_gate_up_proj",
|
|
),
|
|
(10240, 7168): ProjectionSpec(
|
|
10240,
|
|
7168,
|
|
cute_configs=(
|
|
(1, _cute(1, 224, 4, 2)),
|
|
(2, _cute(2, 32, 2, 4)),
|
|
(3, _cute(3, 64, 4, 1)),
|
|
(4, _cute(4, 64, 4, 1)),
|
|
),
|
|
name="lm_head",
|
|
),
|
|
# 7168x2112 (TP16 dense down_proj) has no entry on purpose: K=2112 divides
|
|
# none of the fused-A tile_k values, and the CuTe kernel is left with
|
|
# vector_width=2, which measured slower than cuBLAS.
|
|
}
|
|
|
|
# Keyed by local (N, K), B200 (SM100) entries. Measured on B200 at TP8/TP16
|
|
# shapes over M=1..16 with the same >=5%-over-cuBLAS threshold as the SM103
|
|
# table. Per-(shape, M) entries take the best of the SM103-tuned config and a
|
|
# fresh SM100 measurement: several SM103 configs (7168x8448, 20480x7168,
|
|
# 40960x7168, 10240x7168, 7168x3072 M2, 7168x1536) beat the heuristic config on
|
|
# B200 outright, while a few (6288x7168 M2..4, 3072x7168 M3..5) lose to cuBLAS
|
|
# on B200 and are intentionally not inherited. The 7168x3584 residual configs
|
|
# are intentionally absent: the B200 residual path was not measured.
|
|
KIMI_K3_PROJECTIONS_SM100: dict[tuple[int, int], ProjectionSpec] = {
|
|
(1536, 128): ProjectionSpec(1536, 128, frozenset({1, 16}), name="f_b_proj"),
|
|
(3072, 128): ProjectionSpec(3072, 128, frozenset({8}), name="f_b_proj"),
|
|
# Same mla_g_proj aux-stream/PDL capture caveat as the SM103 entry applies;
|
|
# only M4/M8 measured a dsv3 win, so this is the narrow set either way.
|
|
(1536, 7168): ProjectionSpec(
|
|
1536,
|
|
7168,
|
|
frozenset({4, 8}),
|
|
(
|
|
(1, _cute(1, 224, 3, 2)),
|
|
(2, _cute(2, 128, 2, 2)),
|
|
),
|
|
name="shared_gate_up_proj/mla_g_proj",
|
|
),
|
|
(3072, 7168): ProjectionSpec(
|
|
3072,
|
|
7168,
|
|
cute_configs=(
|
|
(1, _cute(1, 224, 3, 4)),
|
|
(2, _cute(2, 128, 2, 1)),
|
|
),
|
|
name="shared_gate_up_proj",
|
|
),
|
|
(2112, 7168): ProjectionSpec(
|
|
2112,
|
|
7168,
|
|
frozenset({4, 16}),
|
|
(
|
|
(1, _cute(1, 224, 3, 2)),
|
|
(2, _cute(2, 128, 2, 2)),
|
|
),
|
|
name="fused_qkv_a_proj",
|
|
),
|
|
(2304, 1536): ProjectionSpec(2304, 1536, _M1_TO_16, name="q_b_proj"),
|
|
(4608, 1536): ProjectionSpec(4608, 1536, frozenset({1, 2, 4}), name="q_b_proj"),
|
|
# M2..4 of the SM103 configs lose to cuBLAS on B200; only M1 carries over.
|
|
(6288, 7168): ProjectionSpec(
|
|
6288,
|
|
7168,
|
|
cute_configs=((1, _cute(1, 224, 3, 4)),),
|
|
name="in_proj_qkvgfab",
|
|
),
|
|
(12448, 7168): ProjectionSpec(
|
|
12448,
|
|
7168,
|
|
cute_configs=(
|
|
(1, _cute(1, 224, 4, 2)),
|
|
(2, _cute(2, 64, 4, 2)),
|
|
(3, _cute(3, 64, 2, 2)),
|
|
(4, _cute(4, 128, 2, 1)),
|
|
),
|
|
name="in_proj_qkvgfab",
|
|
),
|
|
(7168, 768): ProjectionSpec(7168, 768, frozenset({1}), name="shared_down_proj"),
|
|
# SM103-tuned config wins on B200 (+25%) where the heuristic config tied.
|
|
(7168, 1536): ProjectionSpec(
|
|
7168,
|
|
1536,
|
|
cute_configs=((1, _cute(1, 96, 4, 2)),),
|
|
name="o_proj",
|
|
),
|
|
(7168, 3072): ProjectionSpec(
|
|
7168,
|
|
3072,
|
|
cute_configs=(
|
|
(1, _cute(1, 96, 2, 4)),
|
|
(2, _cute(2, 32, 4, 4)),
|
|
),
|
|
name="o_proj",
|
|
),
|
|
(7168, 3584): ProjectionSpec(
|
|
7168,
|
|
3584,
|
|
cute_configs=(
|
|
(1, _cute(1, 64, 4, 2)),
|
|
(2, _cute(2, 64, 4, 2)),
|
|
),
|
|
name="routed_expert_up_proj",
|
|
),
|
|
(7168, 8448): ProjectionSpec(
|
|
7168,
|
|
8448,
|
|
cute_configs=(
|
|
(1, _cute(1, 32, 4, 4)),
|
|
(2, _cute(2, 96, 4, 1)),
|
|
(3, _cute(3, 96, 4, 1)),
|
|
),
|
|
name="dense_down_proj",
|
|
),
|
|
(8448, 7168): ProjectionSpec(
|
|
8448,
|
|
7168,
|
|
cute_configs=(
|
|
(1, _cute(1, 224, 3, 4)),
|
|
(2, _cute(2, 32, 4, 4)),
|
|
),
|
|
name="dense_gate_up_proj",
|
|
),
|
|
(16896, 7168): ProjectionSpec(
|
|
16896,
|
|
7168,
|
|
cute_configs=((1, _cute(1, 224, 3, 4)),),
|
|
name="dense_gate_up_proj",
|
|
),
|
|
(20480, 7168): ProjectionSpec(
|
|
20480,
|
|
7168,
|
|
cute_configs=(
|
|
(1, _cute(1, 224, 4, 2)),
|
|
(2, _cute(2, 64, 4, 2)),
|
|
(3, _cute(3, 64, 2, 2)),
|
|
(4, _cute(4, 64, 4, 1)),
|
|
),
|
|
name="lm_head",
|
|
),
|
|
(40960, 7168): ProjectionSpec(
|
|
40960,
|
|
7168,
|
|
cute_configs=(
|
|
(1, _cute(1, 128, 4, 2)),
|
|
(2, _cute(2, 64, 4, 2)),
|
|
(3, _cute(3, 64, 4, 1)),
|
|
(4, _cute(4, 64, 4, 1)),
|
|
),
|
|
name="lm_head",
|
|
),
|
|
(10240, 7168): ProjectionSpec(
|
|
10240,
|
|
7168,
|
|
cute_configs=(
|
|
(1, _cute(1, 224, 4, 2)),
|
|
(2, _cute(2, 32, 2, 4)),
|
|
(3, _cute(3, 64, 4, 1)),
|
|
(4, _cute(4, 64, 4, 1)),
|
|
),
|
|
name="lm_head",
|
|
),
|
|
# Latent-MoE replicated projections. dsv3 measured M2..8 wins on B300 for
|
|
# 3584x7168 but loses to cuBLAS there on B200; only CuTe wins land here.
|
|
(3584, 7168): ProjectionSpec(
|
|
3584,
|
|
7168,
|
|
cute_configs=(
|
|
(1, _cute(1, 224, 2, 4)),
|
|
(2, _cute(2, 128, 2, 1)),
|
|
(3, _cute(3, 128, 2, 1)),
|
|
),
|
|
name="routed_expert_down_proj",
|
|
),
|
|
# TP16 shapes, newly measured on B200 (they only appear in TP16 deployments;
|
|
# on TP8 these table entries simply never match).
|
|
(768, 7168): ProjectionSpec(
|
|
768,
|
|
7168,
|
|
cute_configs=(
|
|
(1, _cute(1, 224, 2, 4)),
|
|
(2, _cute(2, 224, 2, 2)),
|
|
(3, _cute(3, 224, 2, 2)),
|
|
(4, _cute(4, 224, 2, 2)),
|
|
),
|
|
name="mla_g_proj/shared_gate_up_proj",
|
|
),
|
|
(1152, 1536): ProjectionSpec(
|
|
1152,
|
|
1536,
|
|
cute_configs=((1, _cute(1, 192, 3, 4)),),
|
|
name="q_b_proj",
|
|
),
|
|
(3216, 7168): ProjectionSpec(
|
|
3216,
|
|
7168,
|
|
cute_configs=(
|
|
(1, _cute(1, 224, 3, 4)),
|
|
(2, _cute(2, 128, 4, 2)),
|
|
),
|
|
name="in_proj_qkvgfab",
|
|
),
|
|
(4224, 7168): ProjectionSpec(
|
|
4224,
|
|
7168,
|
|
cute_configs=(
|
|
(1, _cute(1, 224, 3, 4)),
|
|
(2, _cute(2, 128, 2, 1)),
|
|
(3, _cute(3, 64, 2, 2)),
|
|
),
|
|
name="dense_gate_up_proj",
|
|
),
|
|
}
|
|
|
|
_SM90CuteConfig = tuple[int, int, int, int]
|
|
|
|
|
|
def _sm90_spec(
|
|
n: int,
|
|
k: int,
|
|
dsv3_tokens: frozenset[int] = frozenset(),
|
|
configs: tuple[_SM90CuteConfig, ...] = (),
|
|
) -> ProjectionSpec:
|
|
cute_configs = tuple(
|
|
(m, _cute(m, *config)) for m, config in enumerate(configs, start=1)
|
|
)
|
|
return ProjectionSpec(n, k, dsv3_tokens, cute_configs)
|
|
|
|
|
|
KIMI_K3_PROJECTIONS_SM90: dict[tuple[int, int], ProjectionSpec] = {
|
|
(1536, 128): _sm90_spec(1536, 128, frozenset(range(1, 9))),
|
|
(3072, 128): _sm90_spec(3072, 128, frozenset({1, 2, 5, 6, 7, 8, 9})),
|
|
(1536, 7168): _sm90_spec(
|
|
1536,
|
|
7168,
|
|
frozenset(range(7, 17)),
|
|
(
|
|
(224, 2, 4, 8),
|
|
(128, 3, 2, 8),
|
|
(128, 2, 1, 8),
|
|
(128, 2, 1, 8),
|
|
(128, 3, 1, 8),
|
|
(128, 3, 1, 8),
|
|
),
|
|
),
|
|
(3072, 7168): _sm90_spec(
|
|
3072,
|
|
7168,
|
|
configs=((224, 2, 4, 8), (128, 2, 2, 8), (64, 4, 1, 8), (128, 6, 1, 8)),
|
|
),
|
|
(2112, 7168): _sm90_spec(
|
|
2112,
|
|
7168,
|
|
frozenset(range(5, 17)),
|
|
((224, 2, 4, 8), (128, 4, 2, 8), (128, 2, 1, 8), (128, 2, 1, 8)),
|
|
),
|
|
(2304, 1536): _sm90_spec(
|
|
2304,
|
|
1536,
|
|
frozenset(range(3, 9)),
|
|
((96, 4, 2, 8), (96, 4, 1, 8)),
|
|
),
|
|
(4608, 1536): _sm90_spec(
|
|
4608,
|
|
1536,
|
|
configs=((96, 4, 2, 4), (96, 4, 1, 8), (64, 4, 1, 8)),
|
|
),
|
|
(3584, 7168): _sm90_spec(
|
|
3584,
|
|
7168,
|
|
configs=((224, 2, 4, 8), (128, 4, 2, 8), (128, 2, 1, 8)),
|
|
),
|
|
(6288, 7168): _sm90_spec(
|
|
6288,
|
|
7168,
|
|
configs=((224, 2, 4, 8), (128, 4, 2, 8), (64, 4, 1, 8)),
|
|
),
|
|
(12448, 7168): _sm90_spec(
|
|
12448,
|
|
7168,
|
|
configs=((224, 2, 4, 8), (224, 4, 2, 8), (128, 2, 1, 8)),
|
|
),
|
|
(7168, 768): _sm90_spec(7168, 768, configs=((96, 4, 2, 4), (96, 4, 1, 8))),
|
|
(7168, 1536): _sm90_spec(7168, 1536, configs=((96, 4, 2, 8), (96, 4, 1, 8))),
|
|
(7168, 3072): _sm90_spec(
|
|
7168,
|
|
3072,
|
|
configs=((96, 2, 4, 8), (64, 4, 2, 8), (64, 2, 2, 8)),
|
|
),
|
|
(7168, 3584): _sm90_spec(
|
|
7168,
|
|
3584,
|
|
configs=((224, 4, 2, 8), (64, 4, 2, 8), (64, 4, 2, 8)),
|
|
),
|
|
(7168, 4224): _sm90_spec(7168, 4224, configs=((96, 4, 2, 4), (32, 4, 2, 4))),
|
|
(7168, 8448): _sm90_spec(7168, 8448, configs=((96, 2, 4, 8), (32, 4, 2, 8))),
|
|
(7168, 14336): _sm90_spec(7168, 14336, configs=((224, 2, 4, 8), (128, 2, 2, 8))),
|
|
(20480, 7168): _sm90_spec(
|
|
20480,
|
|
7168,
|
|
configs=((224, 4, 2, 8), (224, 4, 2, 8), (128, 2, 1, 8)),
|
|
),
|
|
}
|
|
|
|
|
|
def _backend_for(
|
|
spec: ProjectionSpec, num_tokens: int, has_residual: bool
|
|
) -> Backend | None:
|
|
if has_residual:
|
|
return "cute" if spec.residual_config(num_tokens) is not None else None
|
|
if spec.cute_config(num_tokens) is not None:
|
|
return "cute"
|
|
if num_tokens in spec.dsv3_tokens:
|
|
return "dsv3_fused_a"
|
|
return None
|
|
|
|
|
|
def select_kimi_k3_backend(
|
|
num_tokens: int,
|
|
n: int,
|
|
k: int,
|
|
*,
|
|
has_residual: bool = False,
|
|
) -> Backend | None:
|
|
"""Backend for a local ``(N, K)`` at ``num_tokens``, or None to fall back."""
|
|
spec = KIMI_K3_PROJECTIONS.get((n, k))
|
|
return _backend_for(spec, num_tokens, has_residual) if spec is not None else None
|
|
|
|
|
|
def _build_plan(spec: ProjectionSpec) -> dict[int, ResolvedCall]:
|
|
plan: dict[int, ResolvedCall] = {}
|
|
for num_tokens in range(1, 17):
|
|
backend = _backend_for(spec, num_tokens, has_residual=False)
|
|
if backend == "cute":
|
|
plan[num_tokens] = ("cute", spec.cute_config(num_tokens))
|
|
elif backend == "dsv3_fused_a":
|
|
plan[num_tokens] = ("dsv3_fused_a", None)
|
|
return plan
|
|
|
|
|
|
def _build_residual_plan(spec: ProjectionSpec) -> dict[int, SkinnyGemmConfig]:
|
|
return {num_tokens: config for num_tokens, config in spec.residual_configs}
|
|
|
|
|
|
def _is_sm103() -> bool:
|
|
return current_platform.is_device_capability((10, 3))
|
|
|
|
|
|
def _is_sm107() -> bool:
|
|
return current_platform.is_device_capability((10, 7))
|
|
|
|
|
|
def _low_latency_table() -> dict[tuple[int, int], ProjectionSpec] | None:
|
|
"""Measured dispatch table for the current device, or None if unsupported."""
|
|
if _is_sm103() or _is_sm107():
|
|
# SM107 reuses the SM103 table; see the module docstring.
|
|
return KIMI_K3_PROJECTIONS
|
|
if current_platform.is_device_capability((10, 0)):
|
|
return KIMI_K3_PROJECTIONS_SM100
|
|
if current_platform.is_device_capability((9, 0)):
|
|
return KIMI_K3_PROJECTIONS_SM90
|
|
return None
|
|
|
|
|
|
def _is_packed_row_major(tensor: torch.Tensor) -> bool:
|
|
return tensor.dim() == 2 and tensor.stride() == (tensor.shape[1], 1)
|
|
|
|
|
|
def _runtime_ok(x: torch.Tensor, weight: torch.Tensor) -> bool:
|
|
return (
|
|
x.dim() == 2
|
|
and x.stride(1) == 1
|
|
and (x.shape[0] == 1 or x.stride(0) % 8 == 0)
|
|
and _is_packed_row_major(weight)
|
|
and x.dtype == torch.bfloat16
|
|
and weight.dtype == torch.bfloat16
|
|
and x.is_cuda
|
|
and weight.is_cuda
|
|
and x.device == weight.device
|
|
and x.shape[1] == weight.shape[1]
|
|
)
|
|
|
|
|
|
def _residual_ok(x: torch.Tensor, weight: torch.Tensor, residual: torch.Tensor) -> bool:
|
|
return (
|
|
_is_packed_row_major(x)
|
|
and residual.dim() == 2
|
|
and residual.dtype == torch.bfloat16
|
|
and residual.is_cuda
|
|
and residual.device == x.device
|
|
and residual.is_contiguous()
|
|
and residual.shape == (x.shape[0], weight.shape[0])
|
|
)
|
|
|
|
|
|
def _run_plan(
|
|
plan: dict[int, ResolvedCall], x: torch.Tensor, weight: torch.Tensor
|
|
) -> torch.Tensor | None:
|
|
entry = plan.get(x.shape[0])
|
|
if entry is None:
|
|
return None
|
|
backend, config = entry
|
|
if backend == "cute":
|
|
if x.shape[0] != 1 and not _is_packed_row_major(x):
|
|
return None
|
|
if not shape_dynamic_skinny_gemm.is_available():
|
|
return None
|
|
return shape_dynamic_skinny_gemm(x, weight, config, None)
|
|
if not hasattr(torch.ops._C, "dsv3_fused_a_gemm"):
|
|
return None
|
|
if x.shape[0] != 1 and not _is_packed_row_major(x):
|
|
return None
|
|
output = torch.empty((x.shape[0], weight.shape[0]), dtype=x.dtype, device=x.device)
|
|
ops.dsv3_fused_a_gemm(output, x, weight.t(), enable_pdl=True)
|
|
return output
|
|
|
|
|
|
def run_kda_projection_overlap(
|
|
hidden_states: torch.Tensor,
|
|
packed_weight: torch.Tensor,
|
|
f_b_weight: torch.Tensor,
|
|
aux_stream: torch.cuda.Stream,
|
|
events: tuple[torch.cuda.Event, torch.cuda.Event],
|
|
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
|
"""Run the TP8 KDA decode projection branches concurrently.
|
|
|
|
Args:
|
|
hidden_states: Packed BF16 input with shape ``[M, 7168]``.
|
|
packed_weight: Existing Q/K/V/G/F_A/beta/pad weight with shape
|
|
``[6288, 7168]``.
|
|
f_b_weight: F_B weight with shape ``[1536, 128]``.
|
|
aux_stream: Stream for the F_A/beta then F_B branch.
|
|
events: Start and completion events for the stream fork and join.
|
|
|
|
Returns:
|
|
The Q/K/V/G projection, F_B output, and beta projection.
|
|
|
|
"""
|
|
num_tokens = hidden_states.shape[0]
|
|
qkvg_weight = packed_weight[:_KDA_QKVG_SIZE]
|
|
|
|
def run_qkvg() -> torch.Tensor:
|
|
if config := KDA_QKVG_CONFIGS.get(num_tokens):
|
|
return shape_dynamic_skinny_gemm(
|
|
hidden_states,
|
|
qkvg_weight,
|
|
config,
|
|
None,
|
|
)
|
|
if num_tokens <= KDA_PROJECTION_OVERLAP_MAX_TOKENS and (
|
|
backend := _kda_qkvg_flashinfer_backend()
|
|
):
|
|
from flashinfer.gemm import mm_bf16
|
|
|
|
return mm_bf16(
|
|
hidden_states,
|
|
qkvg_weight.t(),
|
|
pdl=True,
|
|
backend=backend,
|
|
)
|
|
return torch.mm(hidden_states, qkvg_weight.t())
|
|
|
|
def run_fab_fb() -> tuple[torch.Tensor, torch.Tensor]:
|
|
if num_tokens == 1:
|
|
projected_fab = shape_dynamic_skinny_gemm(
|
|
hidden_states,
|
|
packed_weight[_KDA_QKVG_SIZE : _KDA_QKVG_SIZE + _KDA_FAB_SIZE],
|
|
KDA_M1_FAB_CONFIG,
|
|
None,
|
|
)
|
|
f_a, beta = projected_fab.split([128, 12], dim=-1)
|
|
f_a = f_a.as_strided((1, 128), (128, 1))
|
|
g1 = torch.empty(
|
|
(1, 1536), dtype=hidden_states.dtype, device=hidden_states.device
|
|
)
|
|
ops.dsv3_fused_a_gemm(g1, f_a, f_b_weight.t(), enable_pdl=True)
|
|
else:
|
|
from vllm.models.kimi_k3.nvidia.ops.cute_dsl.kda_skinny_gemm import (
|
|
kda_skinny_gemm,
|
|
)
|
|
|
|
if num_tokens <= KDA_SKINNY_N_MAX_TOKENS:
|
|
projected_fab = kda_skinny_gemm.run_n(
|
|
hidden_states,
|
|
packed_weight[_KDA_QKVG_SIZE:],
|
|
)
|
|
else:
|
|
projected_fab = torch.mm(
|
|
hidden_states,
|
|
packed_weight[_KDA_QKVG_SIZE:].t(),
|
|
)
|
|
beta = projected_fab[:, 128:140]
|
|
if num_tokens <= KDA_SKINNY_K_MAX_TOKENS:
|
|
g1 = kda_skinny_gemm.run_k(projected_fab, f_b_weight)
|
|
else:
|
|
g1 = torch.mm(projected_fab[:, :128], f_b_weight.t())
|
|
return g1, beta
|
|
|
|
projected_qkvg, (g1, beta) = maybe_execute_in_parallel(
|
|
run_qkvg,
|
|
run_fab_fb,
|
|
events[0],
|
|
events[1],
|
|
aux_stream,
|
|
)
|
|
return projected_qkvg, g1, beta
|
|
|
|
|
|
def autotune_kda_qkvg(model: nn.Module) -> None:
|
|
"""Autotune the supported QKVG GEMM before CUDA graph capture."""
|
|
from vllm.models.kimi_k3.nvidia.kda import KimiK3DeltaAttention
|
|
|
|
backend = _kda_qkvg_flashinfer_backend()
|
|
children: list[KimiK3DeltaAttention] = []
|
|
weights_by_shape: dict[
|
|
tuple[int, int, int, torch.dtype, torch.device], torch.Tensor
|
|
] = {}
|
|
for child in model.modules():
|
|
if not isinstance(child, KimiK3DeltaAttention):
|
|
continue
|
|
if child._projection_overlap_max_tokens <= 0:
|
|
continue
|
|
children.append(child)
|
|
if backend is not None:
|
|
qkvg_weight = child.in_proj_qkvgfab.weight[:_KDA_QKVG_SIZE]
|
|
shape = (
|
|
KDA_PROJECTION_OVERLAP_MAX_TOKENS,
|
|
qkvg_weight.shape[0],
|
|
qkvg_weight.shape[1],
|
|
qkvg_weight.dtype,
|
|
qkvg_weight.device,
|
|
)
|
|
weights_by_shape.setdefault(shape, qkvg_weight)
|
|
|
|
if backend is not None:
|
|
from flashinfer.gemm import mm_bf16
|
|
|
|
for shape, qkvg_weight in weights_by_shape.items():
|
|
num_tokens = shape[0]
|
|
hidden_states = torch.empty(
|
|
(num_tokens, qkvg_weight.shape[1]),
|
|
dtype=qkvg_weight.dtype,
|
|
device=qkvg_weight.device,
|
|
)
|
|
mm_bf16(
|
|
hidden_states,
|
|
qkvg_weight.t(),
|
|
pdl=True,
|
|
backend=backend,
|
|
)
|
|
for child in children:
|
|
child._projection_overlap_max_tokens = KDA_PROJECTION_OVERLAP_MAX_TOKENS
|
|
|
|
|
|
def _run_residual_plan(
|
|
residual_plan: dict[int, SkinnyGemmConfig],
|
|
x: torch.Tensor,
|
|
weight: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
) -> torch.Tensor | None:
|
|
config = residual_plan.get(x.shape[0])
|
|
if config is None or not shape_dynamic_skinny_gemm.is_available():
|
|
return None
|
|
return shape_dynamic_skinny_gemm(x, weight, config, residual)
|
|
|
|
|
|
def try_low_latency_gemm(
|
|
x: torch.Tensor,
|
|
weight: torch.Tensor,
|
|
residual: torch.Tensor | None = None,
|
|
) -> torch.Tensor | None:
|
|
"""Run the shape-selected low-latency kernel, or None to fall back.
|
|
|
|
Resolves the plan from the shape table on each call; production installs a
|
|
precomputed plan (see :func:`enable_kimi_k3_low_latency_gemm`) and does not
|
|
use this path.
|
|
"""
|
|
if envs.VLLM_BATCH_INVARIANT or not _runtime_ok(x, weight):
|
|
return None
|
|
table = _low_latency_table()
|
|
if table is None:
|
|
return None
|
|
spec = table.get((weight.shape[0], weight.shape[1]))
|
|
if spec is None:
|
|
return None
|
|
if residual is None:
|
|
return _run_plan(_build_plan(spec), x, weight)
|
|
if not _residual_ok(x, weight, residual):
|
|
return None
|
|
return _run_residual_plan(_build_residual_plan(spec), x, weight, residual)
|
|
|
|
|
|
class _KimiK3LowLatencyApply:
|
|
"""Mixin: try the precomputed plan, else defer to the base method."""
|
|
|
|
def __init__(self, plan: dict[int, ResolvedCall]) -> None:
|
|
super().__init__()
|
|
self._plan = plan
|
|
|
|
def apply(
|
|
self,
|
|
layer: nn.Module,
|
|
x: torch.Tensor,
|
|
bias: torch.Tensor | None = None,
|
|
) -> torch.Tensor:
|
|
if (
|
|
bias is None
|
|
and not envs.VLLM_BATCH_INVARIANT
|
|
and _runtime_ok(x, layer.weight)
|
|
):
|
|
output = _run_plan(self._plan, x, layer.weight)
|
|
if output is not None:
|
|
return output
|
|
return super().apply(layer, x, bias) # type: ignore[misc]
|
|
|
|
|
|
class KimiK3LowLatencyLinearMethod(_KimiK3LowLatencyApply, UnquantizedLinearMethod):
|
|
def __init__(
|
|
self,
|
|
plan: dict[int, ResolvedCall],
|
|
residual_plan: dict[int, SkinnyGemmConfig],
|
|
) -> None:
|
|
super().__init__(plan)
|
|
self._residual_plan = residual_plan
|
|
|
|
def apply_with_residual(
|
|
self,
|
|
layer: nn.Module,
|
|
x: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
) -> torch.Tensor:
|
|
if (
|
|
not envs.VLLM_BATCH_INVARIANT
|
|
and _runtime_ok(x, layer.weight)
|
|
and _residual_ok(x, layer.weight, residual)
|
|
):
|
|
output = _run_residual_plan(self._residual_plan, x, layer.weight, residual)
|
|
if output is not None:
|
|
return output
|
|
return torch.addmm(residual, x, layer.weight.t())
|
|
|
|
|
|
class KimiK3LowLatencyEmbeddingMethod(
|
|
_KimiK3LowLatencyApply, UnquantizedEmbeddingMethod
|
|
):
|
|
pass
|
|
|
|
|
|
def _enable_kda_projection_overlap(module: nn.Module) -> bool:
|
|
from vllm.models.kimi_k3.nvidia.kda import KimiK3DeltaAttention
|
|
|
|
if envs.VLLM_BATCH_INVARIANT:
|
|
return False
|
|
|
|
enabled = False
|
|
for child in module.modules():
|
|
if not isinstance(child, KimiK3DeltaAttention):
|
|
continue
|
|
if child.tp_size != _KDA_TP_SIZE:
|
|
continue
|
|
packed_weight = child.in_proj_qkvgfab.weight
|
|
f_b_weight = child.f_b_proj.weight
|
|
if (
|
|
type(child.in_proj_qkvgfab.quant_method) is not KimiK3LowLatencyLinearMethod
|
|
or type(child.f_b_proj.quant_method) is not KimiK3LowLatencyLinearMethod
|
|
or child._projection_aux_stream is None
|
|
or child._projection_events is None
|
|
or packed_weight.shape != (_KDA_PACKED_SIZE, 7168)
|
|
or f_b_weight.shape != (1536, 128)
|
|
or packed_weight.dtype != torch.bfloat16
|
|
or f_b_weight.dtype != torch.bfloat16
|
|
or not packed_weight.is_cuda
|
|
or not f_b_weight.is_cuda
|
|
or not packed_weight.is_contiguous()
|
|
or not f_b_weight.is_contiguous()
|
|
):
|
|
continue
|
|
child._projection_overlap_max_tokens = max(KDA_QKVG_CONFIGS)
|
|
enabled = True
|
|
return enabled
|
|
|
|
|
|
def enable_kimi_k3_low_latency_gemm(
|
|
module: nn.Module,
|
|
dtype: torch.dtype,
|
|
) -> None:
|
|
"""Install shape-selected low-latency GEMMs and register CuTe warmups.
|
|
|
|
Modules are matched purely by type, an exactly-unquantized method, and a
|
|
local ``(N, K)`` present in the current device's measured table
|
|
(:data:`KIMI_K3_PROJECTIONS` on SM103, :data:`KIMI_K3_PROJECTIONS_SM100`
|
|
on SM100, :data:`KIMI_K3_PROJECTIONS_SM90` on SM90).
|
|
"""
|
|
if dtype != torch.bfloat16:
|
|
return
|
|
table = _low_latency_table()
|
|
if table is None:
|
|
return
|
|
|
|
warmup_configs: set[SkinnyGemmConfig] = set()
|
|
residual_warmup_configs: set[SkinnyGemmConfig] = set()
|
|
for child in module.modules():
|
|
is_linear = (
|
|
isinstance(child, LinearBase)
|
|
and type(child.quant_method) is UnquantizedLinearMethod
|
|
)
|
|
# ParallelLMHead is a VocabParallelEmbedding subclass; embed_tokens is
|
|
# the parent type, so isinstance already excludes it.
|
|
is_head = (
|
|
isinstance(child, ParallelLMHead)
|
|
and type(child.quant_method) is UnquantizedEmbeddingMethod
|
|
)
|
|
if not (is_linear or is_head):
|
|
continue
|
|
weight = getattr(child, "weight", None)
|
|
if weight is None or weight.dim() != 2:
|
|
continue
|
|
spec = table.get((weight.shape[0], weight.shape[1]))
|
|
if spec is None:
|
|
continue
|
|
if is_linear:
|
|
child.quant_method = KimiK3LowLatencyLinearMethod(
|
|
_build_plan(spec), _build_residual_plan(spec)
|
|
)
|
|
else:
|
|
child.quant_method = KimiK3LowLatencyEmbeddingMethod(_build_plan(spec))
|
|
# Warm up only the configs measured for this module's local (N, K) so a
|
|
# TP8 deployment does not compile TP4 configs and vice versa.
|
|
warmup_configs.update(config for _, config in spec.cute_configs)
|
|
residual_warmup_configs.update(config for _, config in spec.residual_configs)
|
|
if _enable_kda_projection_overlap(module):
|
|
from vllm.models.kimi_k3.nvidia.ops.cute_dsl.kda_skinny_gemm import (
|
|
kda_skinny_gemm,
|
|
)
|
|
|
|
warmup_configs.update((*KDA_QKVG_CONFIGS.values(), KDA_M1_FAB_CONFIG))
|
|
kda_skinny_gemm.request_warmup(
|
|
set(range(2, KDA_SKINNY_N_MAX_TOKENS + 1)),
|
|
set(range(2, KDA_SKINNY_K_MAX_TOKENS + 1)),
|
|
)
|
|
|
|
if shape_dynamic_skinny_gemm.is_available():
|
|
if warmup_configs:
|
|
shape_dynamic_skinny_gemm.request_warmup_configs(dtype, warmup_configs)
|
|
if residual_warmup_configs:
|
|
shape_dynamic_skinny_gemm.request_warmup_configs(
|
|
dtype, residual_warmup_configs, has_residual=True
|
|
)
|