[ROCm][Bugfix] Kimi-K3 Fix KDA NaN on mixed batches and racy autotune config (#50649)
Signed-off-by: kliuae <kuanfu.liu@embeddedllm.com>
This commit is contained in:
@@ -0,0 +1,530 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
import torch
|
||||||
|
from einops import rearrange
|
||||||
|
from torch import nn
|
||||||
|
|
||||||
|
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||||
|
from vllm.config import VllmConfig
|
||||||
|
from vllm.distributed import divide
|
||||||
|
from vllm.forward_context import get_forward_context
|
||||||
|
from vllm.model_executor.layers.linear import (
|
||||||
|
ColumnParallelLinear,
|
||||||
|
RowParallelLinear,
|
||||||
|
)
|
||||||
|
from vllm.model_executor.layers.mamba.gdn.base import GatedDeltaNetAttention
|
||||||
|
|
||||||
|
# Generic KDA helpers, shared with Kimi-Linear. They are neither ROCm- nor
|
||||||
|
# K3-specific, so they are imported rather than duplicated. (nvidia/kda.py keeps
|
||||||
|
# its own copies; duplicating here instead would be a one-line change.)
|
||||||
|
from vllm.model_executor.layers.mamba.gdn.kimi_gdn_linear_attn import (
|
||||||
|
_KDA_GATE_LOGBOUND_MIN,
|
||||||
|
_KimiGDNMergedColumnParallelLinear,
|
||||||
|
_make_fused_conv1d_weight_loader,
|
||||||
|
a_log_weight_loader,
|
||||||
|
)
|
||||||
|
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||||
|
MambaStateDtypeCalculator,
|
||||||
|
MambaStateShapeCalculator,
|
||||||
|
is_conv_state_dim_first,
|
||||||
|
)
|
||||||
|
from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
|
||||||
|
causal_conv1d_fn,
|
||||||
|
causal_conv1d_update,
|
||||||
|
)
|
||||||
|
from vllm.model_executor.layers.mamba.ops.gather_initial_states import (
|
||||||
|
gather_initial_states,
|
||||||
|
)
|
||||||
|
from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
|
||||||
|
from vllm.model_executor.utils import set_weight_attrs
|
||||||
|
from vllm.models.kimi_k3.amd.ops.third_party.kda import (
|
||||||
|
chunk_kda_with_fused_gate,
|
||||||
|
fused_recurrent_kda,
|
||||||
|
fused_recurrent_kda_packed_decode,
|
||||||
|
)
|
||||||
|
from vllm.third_party.flash_linear_attention.ops.kda import FusedRMSNormGated
|
||||||
|
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
|
||||||
|
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
|
||||||
|
|
||||||
|
|
||||||
|
class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||||
|
def get_state_dtype(self) -> tuple[torch.dtype, torch.dtype]:
|
||||||
|
if self.model_config is None or self.cache_config is None:
|
||||||
|
raise ValueError("model_config and cache_config must be set")
|
||||||
|
return MambaStateDtypeCalculator.kda_state_dtype(
|
||||||
|
self.model_config.dtype, self.cache_config.mamba_cache_dtype
|
||||||
|
)
|
||||||
|
|
||||||
|
def get_state_shape(self) -> tuple[tuple[int, ...], tuple[int, ...]]:
|
||||||
|
return MambaStateShapeCalculator.kda_state_shape(
|
||||||
|
self.tp_size,
|
||||||
|
self.num_heads,
|
||||||
|
self.head_dim,
|
||||||
|
conv_kernel_size=self.conv_size,
|
||||||
|
num_spec=self.num_spec,
|
||||||
|
)
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
config: KimiLinearConfig,
|
||||||
|
vllm_config: VllmConfig,
|
||||||
|
prefix: str = "",
|
||||||
|
) -> None:
|
||||||
|
super().__init__(config, vllm_config, prefix)
|
||||||
|
|
||||||
|
kda_config = config.linear_attn_config # type: ignore[attr-defined]
|
||||||
|
assert kda_config is not None, "linear_attn_config must be set"
|
||||||
|
assert kda_config.get("use_full_rank_gate", False), (
|
||||||
|
"KimiK3DeltaAttention requires use_full_rank_gate; the low-rank "
|
||||||
|
"gate path belongs to the shared Kimi-Linear layer."
|
||||||
|
)
|
||||||
|
|
||||||
|
self.head_dim = kda_config["head_dim"]
|
||||||
|
self.num_heads = kda_config["num_heads"]
|
||||||
|
assert self.num_heads % self.tp_size == 0
|
||||||
|
self.local_num_heads = divide(self.num_heads, self.tp_size)
|
||||||
|
|
||||||
|
self.projection_size = self.head_dim * self.num_heads
|
||||||
|
self.local_projection_size = divide(self.projection_size, self.tp_size)
|
||||||
|
self.conv_size = kda_config["short_conv_kernel_size"]
|
||||||
|
self.use_full_rank_gate = True
|
||||||
|
|
||||||
|
# Keep f_a before the narrow beta shard, then pad each TP-local row to
|
||||||
|
# select the aligned BF16 GEMM path. The padding also avoids an Inductor
|
||||||
|
# correctness issue seen with the row-strided G view.
|
||||||
|
qkvg_output_sizes = [self.projection_size] * 4
|
||||||
|
in_proj_output_sizes = qkvg_output_sizes + [self.head_dim, self.num_heads]
|
||||||
|
local_output_size = (
|
||||||
|
4 * self.local_projection_size + self.head_dim + self.local_num_heads
|
||||||
|
)
|
||||||
|
self.in_proj_padding = -local_output_size % 16
|
||||||
|
if self.in_proj_padding:
|
||||||
|
in_proj_output_sizes.append(self.in_proj_padding * self.tp_size)
|
||||||
|
|
||||||
|
self.in_proj_qkvgfab = _KimiGDNMergedColumnParallelLinear(
|
||||||
|
self.hidden_size,
|
||||||
|
in_proj_output_sizes,
|
||||||
|
replicated_shard_id=4,
|
||||||
|
tp_size=self.tp_size,
|
||||||
|
bias=False,
|
||||||
|
quant_config=self.quant_config,
|
||||||
|
prefix=f"{prefix}.in_proj_qkvgfab",
|
||||||
|
)
|
||||||
|
if self.in_proj_padding:
|
||||||
|
self.in_proj_qkvgfab.weight.data[-self.in_proj_padding :].zero_()
|
||||||
|
|
||||||
|
self.f_b_proj = ColumnParallelLinear(
|
||||||
|
self.head_dim,
|
||||||
|
self.projection_size,
|
||||||
|
bias=False,
|
||||||
|
quant_config=self.quant_config,
|
||||||
|
prefix=f"{prefix}.f_b_proj",
|
||||||
|
)
|
||||||
|
self.dt_bias = nn.Parameter(
|
||||||
|
torch.empty(self.local_projection_size, dtype=torch.float32)
|
||||||
|
)
|
||||||
|
set_weight_attrs(self.dt_bias, {"weight_loader": sharded_weight_loader(0)})
|
||||||
|
|
||||||
|
# One packed parameter and cache let decode run a single conv update.
|
||||||
|
# Prefill slices them back into Q/K/V to obtain dense outputs cheaply.
|
||||||
|
self.conv1d = ColumnParallelLinear(
|
||||||
|
input_size=self.conv_size,
|
||||||
|
output_size=3 * self.projection_size,
|
||||||
|
bias=False,
|
||||||
|
params_dtype=torch.float32,
|
||||||
|
prefix=f"{prefix}.conv1d",
|
||||||
|
)
|
||||||
|
self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
|
||||||
|
delattr(self.conv1d.weight, "weight_loader")
|
||||||
|
set_weight_attrs(
|
||||||
|
self.conv1d.weight,
|
||||||
|
{
|
||||||
|
"weight_loader": _make_fused_conv1d_weight_loader(
|
||||||
|
[self.projection_size] * 3,
|
||||||
|
self.tp_size,
|
||||||
|
self.tp_rank,
|
||||||
|
)
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
self.A_log = nn.Parameter(
|
||||||
|
torch.empty(self.local_num_heads, dtype=torch.float32)
|
||||||
|
)
|
||||||
|
set_weight_attrs(self.A_log, {"weight_loader": a_log_weight_loader(0)})
|
||||||
|
|
||||||
|
self.gate_lower_bound: float | None = kda_config.get("gate_lower_bound", None)
|
||||||
|
if self.gate_lower_bound is not None:
|
||||||
|
assert _KDA_GATE_LOGBOUND_MIN <= self.gate_lower_bound < 0, (
|
||||||
|
"KDA gate lower bound must be in "
|
||||||
|
f"[{_KDA_GATE_LOGBOUND_MIN}, 0). "
|
||||||
|
f"Got {self.gate_lower_bound}."
|
||||||
|
)
|
||||||
|
self.use_safe_gate = self.gate_lower_bound is not None
|
||||||
|
|
||||||
|
additional_config = vllm_config.additional_config
|
||||||
|
backend = (
|
||||||
|
additional_config.get("kda_prefill_backend", "auto")
|
||||||
|
if isinstance(additional_config, dict)
|
||||||
|
else "auto"
|
||||||
|
)
|
||||||
|
backend = "triton" if backend == "auto" else backend
|
||||||
|
assert backend == "triton", (
|
||||||
|
"The ROCm Kimi-K3 KDA layer only supports the Triton KDA prefill "
|
||||||
|
f"backend, got {backend!r}."
|
||||||
|
)
|
||||||
|
|
||||||
|
self.o_norm = FusedRMSNormGated(self.head_dim, activation="sigmoid")
|
||||||
|
self.o_proj = RowParallelLinear(
|
||||||
|
self.projection_size,
|
||||||
|
self.hidden_size,
|
||||||
|
bias=False,
|
||||||
|
quant_config=self.quant_config,
|
||||||
|
prefix=f"{prefix}.o_proj",
|
||||||
|
)
|
||||||
|
|
||||||
|
compilation_config = vllm_config.compilation_config
|
||||||
|
if prefix in compilation_config.static_forward_context:
|
||||||
|
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||||
|
compilation_config.static_forward_context[prefix] = self
|
||||||
|
|
||||||
|
def rearrange_mixed_qkv(
|
||||||
|
self, mixed_qkv: torch.Tensor
|
||||||
|
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||||
|
seq_len = mixed_qkv.shape[0]
|
||||||
|
qkv = mixed_qkv.view(seq_len, 3, self.local_num_heads, self.head_dim)
|
||||||
|
# Materialize all three row-strided inputs with one token-major to
|
||||||
|
# QKV-major permutation. Each unbound tensor is then contiguous.
|
||||||
|
qkv = qkv.permute(1, 0, 2, 3).contiguous().unsqueeze(1)
|
||||||
|
return qkv.unbind(0)
|
||||||
|
|
||||||
|
def forward(
|
||||||
|
self,
|
||||||
|
hidden_states: torch.Tensor,
|
||||||
|
positions: torch.Tensor,
|
||||||
|
output: torch.Tensor,
|
||||||
|
) -> None:
|
||||||
|
num_tokens = hidden_states.size(0)
|
||||||
|
projected_qkvgfab = self.in_proj_qkvgfab(hidden_states)[0]
|
||||||
|
|
||||||
|
split_sizes = [
|
||||||
|
3 * self.local_projection_size,
|
||||||
|
self.local_projection_size,
|
||||||
|
self.head_dim,
|
||||||
|
self.local_num_heads,
|
||||||
|
]
|
||||||
|
if self.in_proj_padding:
|
||||||
|
split_sizes.append(self.in_proj_padding)
|
||||||
|
projected = projected_qkvgfab.split(split_sizes, dim=-1)
|
||||||
|
mixed_qkv, g_proj_states, f_a, beta = projected[:4]
|
||||||
|
|
||||||
|
g1 = self.f_b_proj(f_a)[0]
|
||||||
|
beta = beta.unsqueeze(0)
|
||||||
|
g1 = rearrange(g1, "n (h d) -> 1 n h d", d=self.head_dim)
|
||||||
|
g2 = rearrange(g_proj_states, "... (h d) -> ... h d", d=self.head_dim)
|
||||||
|
|
||||||
|
core_attn_out = torch.empty(
|
||||||
|
(1, num_tokens, self.local_num_heads, self.head_dim),
|
||||||
|
dtype=hidden_states.dtype,
|
||||||
|
device=hidden_states.device,
|
||||||
|
)
|
||||||
|
|
||||||
|
self._forward(
|
||||||
|
mixed_qkv=mixed_qkv,
|
||||||
|
g1=g1,
|
||||||
|
g2=g2,
|
||||||
|
beta=beta,
|
||||||
|
core_attn_out=core_attn_out,
|
||||||
|
)
|
||||||
|
core_attn_out = rearrange(core_attn_out, "1 n h d -> n (h d)")
|
||||||
|
output[:] = self.o_proj(core_attn_out)[0]
|
||||||
|
|
||||||
|
@eager_break_during_capture
|
||||||
|
def _forward(
|
||||||
|
self,
|
||||||
|
mixed_qkv: torch.Tensor,
|
||||||
|
g1: torch.Tensor,
|
||||||
|
g2: torch.Tensor,
|
||||||
|
beta: torch.Tensor,
|
||||||
|
core_attn_out: torch.Tensor,
|
||||||
|
) -> None:
|
||||||
|
forward_context = get_forward_context()
|
||||||
|
attn_metadata_raw = forward_context.attn_metadata
|
||||||
|
|
||||||
|
if attn_metadata_raw is None:
|
||||||
|
return
|
||||||
|
|
||||||
|
assert isinstance(attn_metadata_raw, dict)
|
||||||
|
attn_metadata_narrowed = attn_metadata_raw[self.prefix]
|
||||||
|
assert isinstance(attn_metadata_narrowed, GDNAttentionMetadata)
|
||||||
|
m = attn_metadata_narrowed
|
||||||
|
has_initial_state = m.has_initial_state
|
||||||
|
non_spec_query_start_loc = m.non_spec_query_start_loc
|
||||||
|
non_spec_state_indices_tensor = m.non_spec_state_indices_tensor
|
||||||
|
spec_sequence_masks = m.spec_sequence_masks
|
||||||
|
spec_token_indx = m.spec_token_indx
|
||||||
|
non_spec_token_indx = m.non_spec_token_indx
|
||||||
|
spec_state_indices_tensor = m.spec_state_indices_tensor
|
||||||
|
spec_query_start_loc = m.spec_query_start_loc
|
||||||
|
num_accepted_tokens = m.num_accepted_tokens
|
||||||
|
num_actual_tokens = m.num_actual_tokens
|
||||||
|
mixed_qkv = mixed_qkv[:num_actual_tokens]
|
||||||
|
g1 = g1[:, :num_actual_tokens]
|
||||||
|
beta = beta[:, :num_actual_tokens]
|
||||||
|
|
||||||
|
constant_caches = self.kv_cache
|
||||||
|
|
||||||
|
conv_state, recurrent_state = constant_caches
|
||||||
|
# conv_state must be (..., dim, width-1) for the conv kernels.
|
||||||
|
# DS layout stores it that way directly; SD layout needs a transpose.
|
||||||
|
if not is_conv_state_dim_first():
|
||||||
|
conv_state = conv_state.transpose(-1, -2)
|
||||||
|
|
||||||
|
conv_weights = self.conv1d.weight.view(
|
||||||
|
self.conv1d.weight.size(0), self.conv1d.weight.size(2)
|
||||||
|
)
|
||||||
|
q_conv_weight, k_conv_weight, v_conv_weight = conv_weights.split(
|
||||||
|
self.local_projection_size, dim=0
|
||||||
|
)
|
||||||
|
q_conv_state, k_conv_state, v_conv_state = conv_state.split(
|
||||||
|
self.local_projection_size, dim=-2
|
||||||
|
)
|
||||||
|
|
||||||
|
# Split tokens into the multi-query spec-decode part and the remaining
|
||||||
|
# (prefill / plain decode) part.
|
||||||
|
if spec_sequence_masks is not None:
|
||||||
|
if m.num_prefills == 0 and m.num_decodes == 0:
|
||||||
|
mixed_qkv_spec = mixed_qkv
|
||||||
|
g1_spec, beta_spec = g1, beta
|
||||||
|
mixed_qkv_ns = g1_ns = beta_ns = None
|
||||||
|
else:
|
||||||
|
mixed_qkv_spec = mixed_qkv.index_select(0, spec_token_indx)
|
||||||
|
g1_spec = g1.index_select(1, spec_token_indx)
|
||||||
|
beta_spec = beta.index_select(1, spec_token_indx)
|
||||||
|
mixed_qkv_ns = mixed_qkv.index_select(0, non_spec_token_indx)
|
||||||
|
g1_ns = g1.index_select(1, non_spec_token_indx)
|
||||||
|
beta_ns = beta.index_select(1, non_spec_token_indx)
|
||||||
|
else:
|
||||||
|
mixed_qkv_spec = g1_spec = beta_spec = None
|
||||||
|
mixed_qkv_ns, g1_ns, beta_ns = mixed_qkv, g1, beta
|
||||||
|
|
||||||
|
# ---------- spec-decode multi-query path ----------
|
||||||
|
core_attn_out_spec = None
|
||||||
|
if spec_sequence_masks is not None:
|
||||||
|
assert spec_state_indices_tensor is not None
|
||||||
|
assert spec_query_start_loc is not None
|
||||||
|
spec_conv_indices = spec_state_indices_tensor[:, 0][: m.num_spec_decodes]
|
||||||
|
spec_max_query_len = spec_state_indices_tensor.size(-1)
|
||||||
|
|
||||||
|
# Sibling beta and, for full-rank gates, output-gate views remain
|
||||||
|
# live, so write the convolution output separately.
|
||||||
|
spec_conv_out = torch.empty(
|
||||||
|
mixed_qkv_spec.shape,
|
||||||
|
dtype=mixed_qkv_spec.dtype,
|
||||||
|
device=mixed_qkv_spec.device,
|
||||||
|
)
|
||||||
|
mixed_qkv_spec = causal_conv1d_update(
|
||||||
|
mixed_qkv_spec,
|
||||||
|
conv_state,
|
||||||
|
conv_weights,
|
||||||
|
self.conv1d.bias,
|
||||||
|
activation="silu",
|
||||||
|
conv_state_indices=spec_conv_indices,
|
||||||
|
num_accepted_tokens=num_accepted_tokens,
|
||||||
|
query_start_loc=spec_query_start_loc,
|
||||||
|
max_query_len=spec_max_query_len,
|
||||||
|
validate_data=False,
|
||||||
|
out=spec_conv_out,
|
||||||
|
)
|
||||||
|
q_spec, k_spec, v_spec = (
|
||||||
|
rearrange(x, "n (h d) -> 1 n h d", d=self.head_dim)
|
||||||
|
for x in mixed_qkv_spec.split(self.local_projection_size, dim=-1)
|
||||||
|
)
|
||||||
|
spec_cu_seqlens = spec_query_start_loc[: m.num_spec_decodes + 1]
|
||||||
|
# Spec-only batches write directly into core_attn_out.
|
||||||
|
spec_out = (
|
||||||
|
core_attn_out[:, : q_spec.shape[1]]
|
||||||
|
if m.num_prefills == 0 and m.num_decodes == 0
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
core_attn_out_spec, _ = fused_recurrent_kda(
|
||||||
|
q=q_spec,
|
||||||
|
k=k_spec,
|
||||||
|
v=v_spec,
|
||||||
|
raw_g=g1_spec,
|
||||||
|
raw_beta=beta_spec,
|
||||||
|
A_log=self.A_log,
|
||||||
|
dt_bias=self.dt_bias,
|
||||||
|
lower_bound=self.gate_lower_bound,
|
||||||
|
initial_state=recurrent_state,
|
||||||
|
cu_seqlens=spec_cu_seqlens,
|
||||||
|
ssm_state_indices=spec_state_indices_tensor,
|
||||||
|
num_accepted_tokens=num_accepted_tokens,
|
||||||
|
out=spec_out,
|
||||||
|
)
|
||||||
|
|
||||||
|
# ---------- non-spec path (prefill or plain decode) ----------
|
||||||
|
core_attn_out_non_spec = None
|
||||||
|
if mixed_qkv_ns is not None:
|
||||||
|
assert g1_ns is not None and beta_ns is not None
|
||||||
|
if m.num_prefills > 0:
|
||||||
|
q_ns, k_ns, v_ns = mixed_qkv_ns.split(
|
||||||
|
self.local_projection_size, dim=-1
|
||||||
|
)
|
||||||
|
|
||||||
|
# Packed prefill conv would require copying V solely to make
|
||||||
|
# it dense for KDA. Separate calls accept the strided inputs
|
||||||
|
# and produce dense Q/K/V without that extra traffic.
|
||||||
|
# TODO: Use packed conv once every KDA prefill backend accepts
|
||||||
|
# row-strided Q/K/V directly.
|
||||||
|
def _prefill_conv(
|
||||||
|
x: torch.Tensor,
|
||||||
|
state: torch.Tensor,
|
||||||
|
weight: torch.Tensor,
|
||||||
|
) -> torch.Tensor:
|
||||||
|
return causal_conv1d_fn(
|
||||||
|
x.transpose(0, 1),
|
||||||
|
weight,
|
||||||
|
None,
|
||||||
|
activation="silu",
|
||||||
|
conv_states=state,
|
||||||
|
has_initial_state=has_initial_state,
|
||||||
|
cache_indices=non_spec_state_indices_tensor,
|
||||||
|
query_start_loc=non_spec_query_start_loc,
|
||||||
|
metadata=m,
|
||||||
|
).transpose(0, 1)
|
||||||
|
|
||||||
|
q_ns = _prefill_conv(q_ns, q_conv_state, q_conv_weight)
|
||||||
|
k_ns = _prefill_conv(k_ns, k_conv_state, k_conv_weight)
|
||||||
|
v_ns = _prefill_conv(v_ns, v_conv_state, v_conv_weight)
|
||||||
|
q_ns, k_ns, v_ns = (
|
||||||
|
rearrange(x, "n (h d) -> 1 n h d", d=self.head_dim)
|
||||||
|
for x in (q_ns, k_ns, v_ns)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert non_spec_state_indices_tensor is not None
|
||||||
|
assert has_initial_state is not None
|
||||||
|
|
||||||
|
# A mixed non-spec batch is decode-first: the decodes are
|
||||||
|
# length-1 sequences, and the chunk kernel returns NaN for those.
|
||||||
|
# Send them to the recurrent kernel and give the chunk kernel the
|
||||||
|
# prefill tail only.
|
||||||
|
core_attn_out_decode = None
|
||||||
|
split_non_spec = spec_sequence_masks is None and m.num_decodes > 0
|
||||||
|
if split_non_spec:
|
||||||
|
assert non_spec_query_start_loc is not None
|
||||||
|
nd_tok = m.num_decode_tokens
|
||||||
|
core_attn_out_decode, _ = fused_recurrent_kda(
|
||||||
|
q=q_ns[:, :nd_tok],
|
||||||
|
k=k_ns[:, :nd_tok],
|
||||||
|
v=v_ns[:, :nd_tok],
|
||||||
|
raw_g=g1_ns[:, :nd_tok],
|
||||||
|
raw_beta=beta_ns[:, :nd_tok],
|
||||||
|
A_log=self.A_log,
|
||||||
|
dt_bias=self.dt_bias,
|
||||||
|
lower_bound=self.gate_lower_bound,
|
||||||
|
initial_state=recurrent_state,
|
||||||
|
cu_seqlens=non_spec_query_start_loc[: m.num_decodes + 1],
|
||||||
|
ssm_state_indices=non_spec_state_indices_tensor[
|
||||||
|
: m.num_decodes
|
||||||
|
],
|
||||||
|
)
|
||||||
|
q_ns = q_ns[:, nd_tok:]
|
||||||
|
k_ns = k_ns[:, nd_tok:]
|
||||||
|
v_ns = v_ns[:, nd_tok:]
|
||||||
|
g1_ns = g1_ns[:, nd_tok:]
|
||||||
|
beta_ns = beta_ns[:, nd_tok:]
|
||||||
|
prefill_query_start_loc = m.prefill_query_start_loc
|
||||||
|
prefill_state_indices = m.prefill_state_indices
|
||||||
|
prefill_has_initial_state = m.prefill_has_initial_state
|
||||||
|
assert prefill_query_start_loc is not None
|
||||||
|
assert prefill_state_indices is not None
|
||||||
|
assert prefill_has_initial_state is not None
|
||||||
|
else:
|
||||||
|
prefill_query_start_loc = non_spec_query_start_loc
|
||||||
|
prefill_state_indices = non_spec_state_indices_tensor
|
||||||
|
prefill_has_initial_state = has_initial_state
|
||||||
|
|
||||||
|
initial_state = gather_initial_states(
|
||||||
|
recurrent_state,
|
||||||
|
prefill_state_indices,
|
||||||
|
prefill_has_initial_state,
|
||||||
|
)
|
||||||
|
(
|
||||||
|
core_attn_out_non_spec,
|
||||||
|
last_recurrent_state,
|
||||||
|
) = chunk_kda_with_fused_gate(
|
||||||
|
q=q_ns,
|
||||||
|
k=k_ns,
|
||||||
|
v=v_ns,
|
||||||
|
raw_g=g1_ns,
|
||||||
|
raw_beta=beta_ns,
|
||||||
|
A_log=self.A_log,
|
||||||
|
g_bias=self.dt_bias,
|
||||||
|
lower_bound=self.gate_lower_bound,
|
||||||
|
initial_state=initial_state,
|
||||||
|
output_final_state=True,
|
||||||
|
use_qk_l2norm_in_kernel=True,
|
||||||
|
cu_seqlens=prefill_query_start_loc,
|
||||||
|
)
|
||||||
|
# Init cache
|
||||||
|
recurrent_state[prefill_state_indices] = last_recurrent_state
|
||||||
|
|
||||||
|
if split_non_spec:
|
||||||
|
# Restore decode-first token order for the merge below.
|
||||||
|
core_attn_out_non_spec = torch.cat(
|
||||||
|
[core_attn_out_decode, core_attn_out_non_spec], dim=1
|
||||||
|
)
|
||||||
|
|
||||||
|
else:
|
||||||
|
# pure-decode non-spec batch
|
||||||
|
assert non_spec_state_indices_tensor is not None
|
||||||
|
decode_conv_indices = non_spec_state_indices_tensor[
|
||||||
|
: mixed_qkv_ns.size(0)
|
||||||
|
]
|
||||||
|
# Sibling beta and, for full-rank gates, output-gate views
|
||||||
|
# remain live, so write the conv output separately.
|
||||||
|
packed_conv_out = torch.empty(
|
||||||
|
mixed_qkv_ns.shape,
|
||||||
|
dtype=mixed_qkv_ns.dtype,
|
||||||
|
device=mixed_qkv_ns.device,
|
||||||
|
)
|
||||||
|
mixed_qkv_ns = causal_conv1d_update(
|
||||||
|
mixed_qkv_ns,
|
||||||
|
conv_state,
|
||||||
|
conv_weights,
|
||||||
|
self.conv1d.bias,
|
||||||
|
activation="silu",
|
||||||
|
conv_state_indices=decode_conv_indices,
|
||||||
|
validate_data=True,
|
||||||
|
out=packed_conv_out,
|
||||||
|
)
|
||||||
|
core_attn_out_non_spec, _ = fused_recurrent_kda_packed_decode(
|
||||||
|
mixed_qkv=mixed_qkv_ns,
|
||||||
|
raw_g=g1_ns,
|
||||||
|
raw_beta=beta_ns,
|
||||||
|
A_log=self.A_log,
|
||||||
|
dt_bias=self.dt_bias,
|
||||||
|
lower_bound=self.gate_lower_bound,
|
||||||
|
initial_state=recurrent_state,
|
||||||
|
state_indices=decode_conv_indices,
|
||||||
|
)
|
||||||
|
|
||||||
|
# ---------- merge spec and non-spec outputs ----------
|
||||||
|
if core_attn_out_spec is not None and core_attn_out_non_spec is not None:
|
||||||
|
# Mixed batches require indexed placement in the original order.
|
||||||
|
merged = torch.empty(
|
||||||
|
(1, num_actual_tokens, *core_attn_out_spec.shape[2:]),
|
||||||
|
dtype=core_attn_out_spec.dtype,
|
||||||
|
device=core_attn_out_spec.device,
|
||||||
|
)
|
||||||
|
merged.index_copy_(1, spec_token_indx, core_attn_out_spec)
|
||||||
|
merged.index_copy_(1, non_spec_token_indx, core_attn_out_non_spec)
|
||||||
|
core_attn_out[0, :num_actual_tokens] = merged[0, :num_actual_tokens]
|
||||||
|
elif core_attn_out_non_spec is not None:
|
||||||
|
core_attn_out[0, :num_actual_tokens] = core_attn_out_non_spec[
|
||||||
|
0, :num_actual_tokens
|
||||||
|
]
|
||||||
|
else:
|
||||||
|
assert core_attn_out_spec is not None
|
||||||
|
core_attn_out.copy_(self.o_norm(core_attn_out, g2))
|
||||||
@@ -28,7 +28,7 @@ from vllm.model_executor.layers.linear import (
|
|||||||
)
|
)
|
||||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||||
from vllm.model_executor.layers.mamba.gdn.kimi_gdn_linear_attn import (
|
from vllm.model_executor.layers.mamba.gdn.kimi_gdn_linear_attn import (
|
||||||
KimiGatedDeltaNetAttention,
|
KimiGatedDeltaNetAttention as KimiLinearGatedDeltaNetAttention,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||||
MambaStateCopyFunc,
|
MambaStateCopyFunc,
|
||||||
@@ -61,6 +61,7 @@ from vllm.model_executor.models.utils import (
|
|||||||
make_layers,
|
make_layers,
|
||||||
maybe_prefix,
|
maybe_prefix,
|
||||||
)
|
)
|
||||||
|
from vllm.models.kimi_k3.amd.kda import KimiK3DeltaAttention
|
||||||
from vllm.models.kimi_k3.amd.ops.attn_res import attn_res
|
from vllm.models.kimi_k3.amd.ops.attn_res import attn_res
|
||||||
from vllm.sequence import IntermediateTensors
|
from vllm.sequence import IntermediateTensors
|
||||||
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
|
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
|
||||||
@@ -477,11 +478,22 @@ class KimiDecoderLayer(nn.Module):
|
|||||||
quant_config = vllm_config.quant_config
|
quant_config = vllm_config.quant_config
|
||||||
|
|
||||||
if config.is_kda_layer(layer_idx):
|
if config.is_kda_layer(layer_idx):
|
||||||
self.self_attn = KimiGatedDeltaNetAttention(
|
# Kimi-K3 sets use_full_rank_gate and uses the ROCm-specific K3 KDA
|
||||||
config,
|
# layer; Kimi-Linear keeps the shared low-rank-gate implementation.
|
||||||
vllm_config,
|
kda_config = config.linear_attn_config
|
||||||
prefix=f"{prefix}.self_attn",
|
assert kda_config is not None
|
||||||
)
|
if kda_config.get("use_full_rank_gate", False):
|
||||||
|
self.self_attn = KimiK3DeltaAttention(
|
||||||
|
config,
|
||||||
|
vllm_config,
|
||||||
|
prefix=f"{prefix}.self_attn",
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
self.self_attn = KimiLinearGatedDeltaNetAttention(
|
||||||
|
config,
|
||||||
|
vllm_config,
|
||||||
|
prefix=f"{prefix}.self_attn",
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
qk_nope_head_dim = config.qk_nope_head_dim
|
qk_nope_head_dim = config.qk_nope_head_dim
|
||||||
qk_rope_head_dim = config.qk_rope_head_dim
|
qk_rope_head_dim = config.qk_rope_head_dim
|
||||||
|
|||||||
+8
-1
@@ -28,6 +28,13 @@ BT_LIST_AUTOTUNE = [32, 64, 128]
|
|||||||
NUM_WARPS_AUTOTUNE = [2, 4, 8, 16] if is_amd else [4, 8, 16, 32]
|
NUM_WARPS_AUTOTUNE = [2, 4, 8, 16] if is_amd else [4, 8, 16, 32]
|
||||||
|
|
||||||
|
|
||||||
|
# num_stages=4 is excluded. The `u` loop has a trip count of 2 (V / BV), and a
|
||||||
|
# 4-stage pipeline over it emits a third, consumer-less async copy into LDS that
|
||||||
|
# the `w` loop then reuses; at num_warps=4 that races on gfx950 and `u` comes
|
||||||
|
# back with non-deterministic O(1e38) garbage once a batch reaches 4096 tokens.
|
||||||
|
_RECOMPUTE_W_U_NUM_STAGES = [2, 3]
|
||||||
|
|
||||||
|
|
||||||
@triton.heuristics(
|
@triton.heuristics(
|
||||||
{
|
{
|
||||||
"STORE_QG": lambda args: args["qg"] is not None,
|
"STORE_QG": lambda args: args["qg"] is not None,
|
||||||
@@ -39,7 +46,7 @@ NUM_WARPS_AUTOTUNE = [2, 4, 8, 16] if is_amd else [4, 8, 16, 32]
|
|||||||
configs=[
|
configs=[
|
||||||
triton.Config({}, num_warps=num_warps, num_stages=num_stages)
|
triton.Config({}, num_warps=num_warps, num_stages=num_stages)
|
||||||
for num_warps in [2, 4, 8]
|
for num_warps in [2, 4, 8]
|
||||||
for num_stages in [2, 3, 4]
|
for num_stages in _RECOMPUTE_W_U_NUM_STAGES
|
||||||
],
|
],
|
||||||
key=["H", "K", "V", "BT", "BK", "BV", "IS_VARLEN"],
|
key=["H", "K", "V", "BT", "BK", "BV", "IS_VARLEN"],
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user