[ROCm][Bugfix] Kimi-K3 Fix KDA NaN on mixed batches and racy autotune config (#50649)
Signed-off-by: kliuae <kuanfu.liu@embeddedllm.com>
This commit is contained in:
@@ -0,0 +1,530 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
from einops import rearrange
|
||||
from torch import nn
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import divide
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.gdn.base import GatedDeltaNetAttention
|
||||
|
||||
# Generic KDA helpers, shared with Kimi-Linear. They are neither ROCm- nor
|
||||
# K3-specific, so they are imported rather than duplicated. (nvidia/kda.py keeps
|
||||
# its own copies; duplicating here instead would be a one-line change.)
|
||||
from vllm.model_executor.layers.mamba.gdn.kimi_gdn_linear_attn import (
|
||||
_KDA_GATE_LOGBOUND_MIN,
|
||||
_KimiGDNMergedColumnParallelLinear,
|
||||
_make_fused_conv1d_weight_loader,
|
||||
a_log_weight_loader,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateDtypeCalculator,
|
||||
MambaStateShapeCalculator,
|
||||
is_conv_state_dim_first,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
|
||||
causal_conv1d_fn,
|
||||
causal_conv1d_update,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.gather_initial_states import (
|
||||
gather_initial_states,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.models.kimi_k3.amd.ops.third_party.kda import (
|
||||
chunk_kda_with_fused_gate,
|
||||
fused_recurrent_kda,
|
||||
fused_recurrent_kda_packed_decode,
|
||||
)
|
||||
from vllm.third_party.flash_linear_attention.ops.kda import FusedRMSNormGated
|
||||
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
|
||||
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
|
||||
|
||||
|
||||
class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
def get_state_dtype(self) -> tuple[torch.dtype, torch.dtype]:
|
||||
if self.model_config is None or self.cache_config is None:
|
||||
raise ValueError("model_config and cache_config must be set")
|
||||
return MambaStateDtypeCalculator.kda_state_dtype(
|
||||
self.model_config.dtype, self.cache_config.mamba_cache_dtype
|
||||
)
|
||||
|
||||
def get_state_shape(self) -> tuple[tuple[int, ...], tuple[int, ...]]:
|
||||
return MambaStateShapeCalculator.kda_state_shape(
|
||||
self.tp_size,
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
conv_kernel_size=self.conv_size,
|
||||
num_spec=self.num_spec,
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: KimiLinearConfig,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__(config, vllm_config, prefix)
|
||||
|
||||
kda_config = config.linear_attn_config # type: ignore[attr-defined]
|
||||
assert kda_config is not None, "linear_attn_config must be set"
|
||||
assert kda_config.get("use_full_rank_gate", False), (
|
||||
"KimiK3DeltaAttention requires use_full_rank_gate; the low-rank "
|
||||
"gate path belongs to the shared Kimi-Linear layer."
|
||||
)
|
||||
|
||||
self.head_dim = kda_config["head_dim"]
|
||||
self.num_heads = kda_config["num_heads"]
|
||||
assert self.num_heads % self.tp_size == 0
|
||||
self.local_num_heads = divide(self.num_heads, self.tp_size)
|
||||
|
||||
self.projection_size = self.head_dim * self.num_heads
|
||||
self.local_projection_size = divide(self.projection_size, self.tp_size)
|
||||
self.conv_size = kda_config["short_conv_kernel_size"]
|
||||
self.use_full_rank_gate = True
|
||||
|
||||
# Keep f_a before the narrow beta shard, then pad each TP-local row to
|
||||
# select the aligned BF16 GEMM path. The padding also avoids an Inductor
|
||||
# correctness issue seen with the row-strided G view.
|
||||
qkvg_output_sizes = [self.projection_size] * 4
|
||||
in_proj_output_sizes = qkvg_output_sizes + [self.head_dim, self.num_heads]
|
||||
local_output_size = (
|
||||
4 * self.local_projection_size + self.head_dim + self.local_num_heads
|
||||
)
|
||||
self.in_proj_padding = -local_output_size % 16
|
||||
if self.in_proj_padding:
|
||||
in_proj_output_sizes.append(self.in_proj_padding * self.tp_size)
|
||||
|
||||
self.in_proj_qkvgfab = _KimiGDNMergedColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
in_proj_output_sizes,
|
||||
replicated_shard_id=4,
|
||||
tp_size=self.tp_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.in_proj_qkvgfab",
|
||||
)
|
||||
if self.in_proj_padding:
|
||||
self.in_proj_qkvgfab.weight.data[-self.in_proj_padding :].zero_()
|
||||
|
||||
self.f_b_proj = ColumnParallelLinear(
|
||||
self.head_dim,
|
||||
self.projection_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.f_b_proj",
|
||||
)
|
||||
self.dt_bias = nn.Parameter(
|
||||
torch.empty(self.local_projection_size, dtype=torch.float32)
|
||||
)
|
||||
set_weight_attrs(self.dt_bias, {"weight_loader": sharded_weight_loader(0)})
|
||||
|
||||
# One packed parameter and cache let decode run a single conv update.
|
||||
# Prefill slices them back into Q/K/V to obtain dense outputs cheaply.
|
||||
self.conv1d = ColumnParallelLinear(
|
||||
input_size=self.conv_size,
|
||||
output_size=3 * self.projection_size,
|
||||
bias=False,
|
||||
params_dtype=torch.float32,
|
||||
prefix=f"{prefix}.conv1d",
|
||||
)
|
||||
self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
|
||||
delattr(self.conv1d.weight, "weight_loader")
|
||||
set_weight_attrs(
|
||||
self.conv1d.weight,
|
||||
{
|
||||
"weight_loader": _make_fused_conv1d_weight_loader(
|
||||
[self.projection_size] * 3,
|
||||
self.tp_size,
|
||||
self.tp_rank,
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
self.A_log = nn.Parameter(
|
||||
torch.empty(self.local_num_heads, dtype=torch.float32)
|
||||
)
|
||||
set_weight_attrs(self.A_log, {"weight_loader": a_log_weight_loader(0)})
|
||||
|
||||
self.gate_lower_bound: float | None = kda_config.get("gate_lower_bound", None)
|
||||
if self.gate_lower_bound is not None:
|
||||
assert _KDA_GATE_LOGBOUND_MIN <= self.gate_lower_bound < 0, (
|
||||
"KDA gate lower bound must be in "
|
||||
f"[{_KDA_GATE_LOGBOUND_MIN}, 0). "
|
||||
f"Got {self.gate_lower_bound}."
|
||||
)
|
||||
self.use_safe_gate = self.gate_lower_bound is not None
|
||||
|
||||
additional_config = vllm_config.additional_config
|
||||
backend = (
|
||||
additional_config.get("kda_prefill_backend", "auto")
|
||||
if isinstance(additional_config, dict)
|
||||
else "auto"
|
||||
)
|
||||
backend = "triton" if backend == "auto" else backend
|
||||
assert backend == "triton", (
|
||||
"The ROCm Kimi-K3 KDA layer only supports the Triton KDA prefill "
|
||||
f"backend, got {backend!r}."
|
||||
)
|
||||
|
||||
self.o_norm = FusedRMSNormGated(self.head_dim, activation="sigmoid")
|
||||
self.o_proj = RowParallelLinear(
|
||||
self.projection_size,
|
||||
self.hidden_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
|
||||
compilation_config = vllm_config.compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
def rearrange_mixed_qkv(
|
||||
self, mixed_qkv: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
seq_len = mixed_qkv.shape[0]
|
||||
qkv = mixed_qkv.view(seq_len, 3, self.local_num_heads, self.head_dim)
|
||||
# Materialize all three row-strided inputs with one token-major to
|
||||
# QKV-major permutation. Each unbound tensor is then contiguous.
|
||||
qkv = qkv.permute(1, 0, 2, 3).contiguous().unsqueeze(1)
|
||||
return qkv.unbind(0)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
num_tokens = hidden_states.size(0)
|
||||
projected_qkvgfab = self.in_proj_qkvgfab(hidden_states)[0]
|
||||
|
||||
split_sizes = [
|
||||
3 * self.local_projection_size,
|
||||
self.local_projection_size,
|
||||
self.head_dim,
|
||||
self.local_num_heads,
|
||||
]
|
||||
if self.in_proj_padding:
|
||||
split_sizes.append(self.in_proj_padding)
|
||||
projected = projected_qkvgfab.split(split_sizes, dim=-1)
|
||||
mixed_qkv, g_proj_states, f_a, beta = projected[:4]
|
||||
|
||||
g1 = self.f_b_proj(f_a)[0]
|
||||
beta = beta.unsqueeze(0)
|
||||
g1 = rearrange(g1, "n (h d) -> 1 n h d", d=self.head_dim)
|
||||
g2 = rearrange(g_proj_states, "... (h d) -> ... h d", d=self.head_dim)
|
||||
|
||||
core_attn_out = torch.empty(
|
||||
(1, num_tokens, self.local_num_heads, self.head_dim),
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
self._forward(
|
||||
mixed_qkv=mixed_qkv,
|
||||
g1=g1,
|
||||
g2=g2,
|
||||
beta=beta,
|
||||
core_attn_out=core_attn_out,
|
||||
)
|
||||
core_attn_out = rearrange(core_attn_out, "1 n h d -> n (h d)")
|
||||
output[:] = self.o_proj(core_attn_out)[0]
|
||||
|
||||
@eager_break_during_capture
|
||||
def _forward(
|
||||
self,
|
||||
mixed_qkv: torch.Tensor,
|
||||
g1: torch.Tensor,
|
||||
g2: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
core_attn_out: torch.Tensor,
|
||||
) -> None:
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
|
||||
if attn_metadata_raw is None:
|
||||
return
|
||||
|
||||
assert isinstance(attn_metadata_raw, dict)
|
||||
attn_metadata_narrowed = attn_metadata_raw[self.prefix]
|
||||
assert isinstance(attn_metadata_narrowed, GDNAttentionMetadata)
|
||||
m = attn_metadata_narrowed
|
||||
has_initial_state = m.has_initial_state
|
||||
non_spec_query_start_loc = m.non_spec_query_start_loc
|
||||
non_spec_state_indices_tensor = m.non_spec_state_indices_tensor
|
||||
spec_sequence_masks = m.spec_sequence_masks
|
||||
spec_token_indx = m.spec_token_indx
|
||||
non_spec_token_indx = m.non_spec_token_indx
|
||||
spec_state_indices_tensor = m.spec_state_indices_tensor
|
||||
spec_query_start_loc = m.spec_query_start_loc
|
||||
num_accepted_tokens = m.num_accepted_tokens
|
||||
num_actual_tokens = m.num_actual_tokens
|
||||
mixed_qkv = mixed_qkv[:num_actual_tokens]
|
||||
g1 = g1[:, :num_actual_tokens]
|
||||
beta = beta[:, :num_actual_tokens]
|
||||
|
||||
constant_caches = self.kv_cache
|
||||
|
||||
conv_state, recurrent_state = constant_caches
|
||||
# conv_state must be (..., dim, width-1) for the conv kernels.
|
||||
# DS layout stores it that way directly; SD layout needs a transpose.
|
||||
if not is_conv_state_dim_first():
|
||||
conv_state = conv_state.transpose(-1, -2)
|
||||
|
||||
conv_weights = self.conv1d.weight.view(
|
||||
self.conv1d.weight.size(0), self.conv1d.weight.size(2)
|
||||
)
|
||||
q_conv_weight, k_conv_weight, v_conv_weight = conv_weights.split(
|
||||
self.local_projection_size, dim=0
|
||||
)
|
||||
q_conv_state, k_conv_state, v_conv_state = conv_state.split(
|
||||
self.local_projection_size, dim=-2
|
||||
)
|
||||
|
||||
# Split tokens into the multi-query spec-decode part and the remaining
|
||||
# (prefill / plain decode) part.
|
||||
if spec_sequence_masks is not None:
|
||||
if m.num_prefills == 0 and m.num_decodes == 0:
|
||||
mixed_qkv_spec = mixed_qkv
|
||||
g1_spec, beta_spec = g1, beta
|
||||
mixed_qkv_ns = g1_ns = beta_ns = None
|
||||
else:
|
||||
mixed_qkv_spec = mixed_qkv.index_select(0, spec_token_indx)
|
||||
g1_spec = g1.index_select(1, spec_token_indx)
|
||||
beta_spec = beta.index_select(1, spec_token_indx)
|
||||
mixed_qkv_ns = mixed_qkv.index_select(0, non_spec_token_indx)
|
||||
g1_ns = g1.index_select(1, non_spec_token_indx)
|
||||
beta_ns = beta.index_select(1, non_spec_token_indx)
|
||||
else:
|
||||
mixed_qkv_spec = g1_spec = beta_spec = None
|
||||
mixed_qkv_ns, g1_ns, beta_ns = mixed_qkv, g1, beta
|
||||
|
||||
# ---------- spec-decode multi-query path ----------
|
||||
core_attn_out_spec = None
|
||||
if spec_sequence_masks is not None:
|
||||
assert spec_state_indices_tensor is not None
|
||||
assert spec_query_start_loc is not None
|
||||
spec_conv_indices = spec_state_indices_tensor[:, 0][: m.num_spec_decodes]
|
||||
spec_max_query_len = spec_state_indices_tensor.size(-1)
|
||||
|
||||
# Sibling beta and, for full-rank gates, output-gate views remain
|
||||
# live, so write the convolution output separately.
|
||||
spec_conv_out = torch.empty(
|
||||
mixed_qkv_spec.shape,
|
||||
dtype=mixed_qkv_spec.dtype,
|
||||
device=mixed_qkv_spec.device,
|
||||
)
|
||||
mixed_qkv_spec = causal_conv1d_update(
|
||||
mixed_qkv_spec,
|
||||
conv_state,
|
||||
conv_weights,
|
||||
self.conv1d.bias,
|
||||
activation="silu",
|
||||
conv_state_indices=spec_conv_indices,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
query_start_loc=spec_query_start_loc,
|
||||
max_query_len=spec_max_query_len,
|
||||
validate_data=False,
|
||||
out=spec_conv_out,
|
||||
)
|
||||
q_spec, k_spec, v_spec = (
|
||||
rearrange(x, "n (h d) -> 1 n h d", d=self.head_dim)
|
||||
for x in mixed_qkv_spec.split(self.local_projection_size, dim=-1)
|
||||
)
|
||||
spec_cu_seqlens = spec_query_start_loc[: m.num_spec_decodes + 1]
|
||||
# Spec-only batches write directly into core_attn_out.
|
||||
spec_out = (
|
||||
core_attn_out[:, : q_spec.shape[1]]
|
||||
if m.num_prefills == 0 and m.num_decodes == 0
|
||||
else None
|
||||
)
|
||||
core_attn_out_spec, _ = fused_recurrent_kda(
|
||||
q=q_spec,
|
||||
k=k_spec,
|
||||
v=v_spec,
|
||||
raw_g=g1_spec,
|
||||
raw_beta=beta_spec,
|
||||
A_log=self.A_log,
|
||||
dt_bias=self.dt_bias,
|
||||
lower_bound=self.gate_lower_bound,
|
||||
initial_state=recurrent_state,
|
||||
cu_seqlens=spec_cu_seqlens,
|
||||
ssm_state_indices=spec_state_indices_tensor,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
out=spec_out,
|
||||
)
|
||||
|
||||
# ---------- non-spec path (prefill or plain decode) ----------
|
||||
core_attn_out_non_spec = None
|
||||
if mixed_qkv_ns is not None:
|
||||
assert g1_ns is not None and beta_ns is not None
|
||||
if m.num_prefills > 0:
|
||||
q_ns, k_ns, v_ns = mixed_qkv_ns.split(
|
||||
self.local_projection_size, dim=-1
|
||||
)
|
||||
|
||||
# Packed prefill conv would require copying V solely to make
|
||||
# it dense for KDA. Separate calls accept the strided inputs
|
||||
# and produce dense Q/K/V without that extra traffic.
|
||||
# TODO: Use packed conv once every KDA prefill backend accepts
|
||||
# row-strided Q/K/V directly.
|
||||
def _prefill_conv(
|
||||
x: torch.Tensor,
|
||||
state: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return causal_conv1d_fn(
|
||||
x.transpose(0, 1),
|
||||
weight,
|
||||
None,
|
||||
activation="silu",
|
||||
conv_states=state,
|
||||
has_initial_state=has_initial_state,
|
||||
cache_indices=non_spec_state_indices_tensor,
|
||||
query_start_loc=non_spec_query_start_loc,
|
||||
metadata=m,
|
||||
).transpose(0, 1)
|
||||
|
||||
q_ns = _prefill_conv(q_ns, q_conv_state, q_conv_weight)
|
||||
k_ns = _prefill_conv(k_ns, k_conv_state, k_conv_weight)
|
||||
v_ns = _prefill_conv(v_ns, v_conv_state, v_conv_weight)
|
||||
q_ns, k_ns, v_ns = (
|
||||
rearrange(x, "n (h d) -> 1 n h d", d=self.head_dim)
|
||||
for x in (q_ns, k_ns, v_ns)
|
||||
)
|
||||
|
||||
assert non_spec_state_indices_tensor is not None
|
||||
assert has_initial_state is not None
|
||||
|
||||
# A mixed non-spec batch is decode-first: the decodes are
|
||||
# length-1 sequences, and the chunk kernel returns NaN for those.
|
||||
# Send them to the recurrent kernel and give the chunk kernel the
|
||||
# prefill tail only.
|
||||
core_attn_out_decode = None
|
||||
split_non_spec = spec_sequence_masks is None and m.num_decodes > 0
|
||||
if split_non_spec:
|
||||
assert non_spec_query_start_loc is not None
|
||||
nd_tok = m.num_decode_tokens
|
||||
core_attn_out_decode, _ = fused_recurrent_kda(
|
||||
q=q_ns[:, :nd_tok],
|
||||
k=k_ns[:, :nd_tok],
|
||||
v=v_ns[:, :nd_tok],
|
||||
raw_g=g1_ns[:, :nd_tok],
|
||||
raw_beta=beta_ns[:, :nd_tok],
|
||||
A_log=self.A_log,
|
||||
dt_bias=self.dt_bias,
|
||||
lower_bound=self.gate_lower_bound,
|
||||
initial_state=recurrent_state,
|
||||
cu_seqlens=non_spec_query_start_loc[: m.num_decodes + 1],
|
||||
ssm_state_indices=non_spec_state_indices_tensor[
|
||||
: m.num_decodes
|
||||
],
|
||||
)
|
||||
q_ns = q_ns[:, nd_tok:]
|
||||
k_ns = k_ns[:, nd_tok:]
|
||||
v_ns = v_ns[:, nd_tok:]
|
||||
g1_ns = g1_ns[:, nd_tok:]
|
||||
beta_ns = beta_ns[:, nd_tok:]
|
||||
prefill_query_start_loc = m.prefill_query_start_loc
|
||||
prefill_state_indices = m.prefill_state_indices
|
||||
prefill_has_initial_state = m.prefill_has_initial_state
|
||||
assert prefill_query_start_loc is not None
|
||||
assert prefill_state_indices is not None
|
||||
assert prefill_has_initial_state is not None
|
||||
else:
|
||||
prefill_query_start_loc = non_spec_query_start_loc
|
||||
prefill_state_indices = non_spec_state_indices_tensor
|
||||
prefill_has_initial_state = has_initial_state
|
||||
|
||||
initial_state = gather_initial_states(
|
||||
recurrent_state,
|
||||
prefill_state_indices,
|
||||
prefill_has_initial_state,
|
||||
)
|
||||
(
|
||||
core_attn_out_non_spec,
|
||||
last_recurrent_state,
|
||||
) = chunk_kda_with_fused_gate(
|
||||
q=q_ns,
|
||||
k=k_ns,
|
||||
v=v_ns,
|
||||
raw_g=g1_ns,
|
||||
raw_beta=beta_ns,
|
||||
A_log=self.A_log,
|
||||
g_bias=self.dt_bias,
|
||||
lower_bound=self.gate_lower_bound,
|
||||
initial_state=initial_state,
|
||||
output_final_state=True,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
cu_seqlens=prefill_query_start_loc,
|
||||
)
|
||||
# Init cache
|
||||
recurrent_state[prefill_state_indices] = last_recurrent_state
|
||||
|
||||
if split_non_spec:
|
||||
# Restore decode-first token order for the merge below.
|
||||
core_attn_out_non_spec = torch.cat(
|
||||
[core_attn_out_decode, core_attn_out_non_spec], dim=1
|
||||
)
|
||||
|
||||
else:
|
||||
# pure-decode non-spec batch
|
||||
assert non_spec_state_indices_tensor is not None
|
||||
decode_conv_indices = non_spec_state_indices_tensor[
|
||||
: mixed_qkv_ns.size(0)
|
||||
]
|
||||
# Sibling beta and, for full-rank gates, output-gate views
|
||||
# remain live, so write the conv output separately.
|
||||
packed_conv_out = torch.empty(
|
||||
mixed_qkv_ns.shape,
|
||||
dtype=mixed_qkv_ns.dtype,
|
||||
device=mixed_qkv_ns.device,
|
||||
)
|
||||
mixed_qkv_ns = causal_conv1d_update(
|
||||
mixed_qkv_ns,
|
||||
conv_state,
|
||||
conv_weights,
|
||||
self.conv1d.bias,
|
||||
activation="silu",
|
||||
conv_state_indices=decode_conv_indices,
|
||||
validate_data=True,
|
||||
out=packed_conv_out,
|
||||
)
|
||||
core_attn_out_non_spec, _ = fused_recurrent_kda_packed_decode(
|
||||
mixed_qkv=mixed_qkv_ns,
|
||||
raw_g=g1_ns,
|
||||
raw_beta=beta_ns,
|
||||
A_log=self.A_log,
|
||||
dt_bias=self.dt_bias,
|
||||
lower_bound=self.gate_lower_bound,
|
||||
initial_state=recurrent_state,
|
||||
state_indices=decode_conv_indices,
|
||||
)
|
||||
|
||||
# ---------- merge spec and non-spec outputs ----------
|
||||
if core_attn_out_spec is not None and core_attn_out_non_spec is not None:
|
||||
# Mixed batches require indexed placement in the original order.
|
||||
merged = torch.empty(
|
||||
(1, num_actual_tokens, *core_attn_out_spec.shape[2:]),
|
||||
dtype=core_attn_out_spec.dtype,
|
||||
device=core_attn_out_spec.device,
|
||||
)
|
||||
merged.index_copy_(1, spec_token_indx, core_attn_out_spec)
|
||||
merged.index_copy_(1, non_spec_token_indx, core_attn_out_non_spec)
|
||||
core_attn_out[0, :num_actual_tokens] = merged[0, :num_actual_tokens]
|
||||
elif core_attn_out_non_spec is not None:
|
||||
core_attn_out[0, :num_actual_tokens] = core_attn_out_non_spec[
|
||||
0, :num_actual_tokens
|
||||
]
|
||||
else:
|
||||
assert core_attn_out_spec is not None
|
||||
core_attn_out.copy_(self.o_norm(core_attn_out, g2))
|
||||
@@ -28,7 +28,7 @@ from vllm.model_executor.layers.linear import (
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.mamba.gdn.kimi_gdn_linear_attn import (
|
||||
KimiGatedDeltaNetAttention,
|
||||
KimiGatedDeltaNetAttention as KimiLinearGatedDeltaNetAttention,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateCopyFunc,
|
||||
@@ -61,6 +61,7 @@ from vllm.model_executor.models.utils import (
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.models.kimi_k3.amd.kda import KimiK3DeltaAttention
|
||||
from vllm.models.kimi_k3.amd.ops.attn_res import attn_res
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
|
||||
@@ -477,11 +478,22 @@ class KimiDecoderLayer(nn.Module):
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
if config.is_kda_layer(layer_idx):
|
||||
self.self_attn = KimiGatedDeltaNetAttention(
|
||||
config,
|
||||
vllm_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
)
|
||||
# Kimi-K3 sets use_full_rank_gate and uses the ROCm-specific K3 KDA
|
||||
# layer; Kimi-Linear keeps the shared low-rank-gate implementation.
|
||||
kda_config = config.linear_attn_config
|
||||
assert kda_config is not None
|
||||
if kda_config.get("use_full_rank_gate", False):
|
||||
self.self_attn = KimiK3DeltaAttention(
|
||||
config,
|
||||
vllm_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
)
|
||||
else:
|
||||
self.self_attn = KimiLinearGatedDeltaNetAttention(
|
||||
config,
|
||||
vllm_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
)
|
||||
else:
|
||||
qk_nope_head_dim = config.qk_nope_head_dim
|
||||
qk_rope_head_dim = config.qk_rope_head_dim
|
||||
|
||||
+8
-1
@@ -28,6 +28,13 @@ BT_LIST_AUTOTUNE = [32, 64, 128]
|
||||
NUM_WARPS_AUTOTUNE = [2, 4, 8, 16] if is_amd else [4, 8, 16, 32]
|
||||
|
||||
|
||||
# num_stages=4 is excluded. The `u` loop has a trip count of 2 (V / BV), and a
|
||||
# 4-stage pipeline over it emits a third, consumer-less async copy into LDS that
|
||||
# the `w` loop then reuses; at num_warps=4 that races on gfx950 and `u` comes
|
||||
# back with non-deterministic O(1e38) garbage once a batch reaches 4096 tokens.
|
||||
_RECOMPUTE_W_U_NUM_STAGES = [2, 3]
|
||||
|
||||
|
||||
@triton.heuristics(
|
||||
{
|
||||
"STORE_QG": lambda args: args["qg"] is not None,
|
||||
@@ -39,7 +46,7 @@ NUM_WARPS_AUTOTUNE = [2, 4, 8, 16] if is_amd else [4, 8, 16, 32]
|
||||
configs=[
|
||||
triton.Config({}, num_warps=num_warps, num_stages=num_stages)
|
||||
for num_warps in [2, 4, 8]
|
||||
for num_stages in [2, 3, 4]
|
||||
for num_stages in _RECOMPUTE_W_U_NUM_STAGES
|
||||
],
|
||||
key=["H", "K", "V", "BT", "BK", "BV", "IS_VARLEN"],
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user