[K3] support recoverssm for K3 (#51855)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Benjamin Chislett <bchislett@nvidia.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Benjamin Chislett <bchislett@nvidia.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Benjamin Chislett
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
f27ae25473
commit
70afdedc10
@@ -282,23 +282,37 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
|
||||
def get_state_dtype(
|
||||
self,
|
||||
) -> tuple[torch.dtype, torch.dtype]:
|
||||
) -> tuple[torch.dtype, ...]:
|
||||
if self.model_config is None or self.cache_config is None:
|
||||
raise ValueError("model_config and cache_config must be set")
|
||||
return MambaStateDtypeCalculator.kda_state_dtype(
|
||||
base_dtypes = MambaStateDtypeCalculator.kda_state_dtype(
|
||||
self.model_config.dtype, self.cache_config.mamba_cache_dtype
|
||||
)
|
||||
if self.cache_config.use_kda_recoverssm:
|
||||
return MambaStateDtypeCalculator.append_kda_recoverssm_record(
|
||||
base_dtypes, self.model_config.dtype
|
||||
)
|
||||
return base_dtypes
|
||||
|
||||
def get_state_shape(
|
||||
self,
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...]]:
|
||||
return MambaStateShapeCalculator.kda_state_shape(
|
||||
) -> tuple[tuple[int, ...], ...]:
|
||||
base_shapes = MambaStateShapeCalculator.kda_state_shape(
|
||||
self.tp_size,
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
conv_kernel_size=self.conv_size,
|
||||
num_spec=self.num_spec,
|
||||
)
|
||||
if self.cache_config.use_kda_recoverssm:
|
||||
return MambaStateShapeCalculator.append_kda_recoverssm_record(
|
||||
base_shapes,
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
tp_world_size=self.tp_size,
|
||||
spec_query_len=1 + self.num_spec,
|
||||
)
|
||||
return base_shapes
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -308,6 +322,12 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
run_gemm_rs: bool = False,
|
||||
) -> None:
|
||||
super().__init__(config, vllm_config, prefix)
|
||||
self.use_recoverssm = self.cache_config.use_kda_recoverssm
|
||||
if self.cache_config.use_replayssm and not self.use_recoverssm:
|
||||
raise ValueError(
|
||||
"Kimi-K3 supports --use-replayssm only with speculative decoding"
|
||||
)
|
||||
self.spec_query_len = 1 + self.num_spec
|
||||
|
||||
kda_config = config.linear_attn_config # type: ignore[attr-defined]
|
||||
assert kda_config is not None, "linear_attn_config must be set"
|
||||
@@ -371,7 +391,7 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
|
||||
# Keep a width-major copy for fused decode without changing the layout
|
||||
# consumed by the prefill and fallback decode kernels.
|
||||
conv_state_dtype, _ = self.get_state_dtype()
|
||||
conv_state_dtype = self.get_state_dtype()[0]
|
||||
decode_conv1d_weight = None
|
||||
if is_fused_kda_decode_supported(
|
||||
self.local_num_heads,
|
||||
@@ -567,7 +587,7 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
g1 = g1[:, :num_actual_tokens]
|
||||
beta = beta[:, :num_actual_tokens]
|
||||
|
||||
conv_state, recurrent_state = self.kv_cache
|
||||
conv_state, recurrent_state, *recoverssm_records = self.kv_cache
|
||||
# The convolution kernels consume (..., dim, width - 1).
|
||||
if not is_conv_state_dim_first():
|
||||
conv_state = conv_state.transpose(-1, -2)
|
||||
@@ -634,7 +654,11 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
assert spec_state_indices_tensor is not None
|
||||
assert spec_query_start_loc is not None
|
||||
spec_conv_indices = spec_state_indices_tensor[:, 0][: m.num_spec_decodes]
|
||||
spec_max_query_len = spec_state_indices_tensor.size(-1)
|
||||
spec_max_query_len = (
|
||||
self.spec_query_len
|
||||
if self.use_recoverssm
|
||||
else spec_state_indices_tensor.size(-1)
|
||||
)
|
||||
spec_conv_out = torch.empty_like(mixed_qkv_spec)
|
||||
mixed_qkv_spec = causal_conv1d_update(
|
||||
mixed_qkv_spec,
|
||||
@@ -659,21 +683,48 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
if m.num_prefills == 0 and m.num_decodes == 0
|
||||
else None
|
||||
)
|
||||
core_attn_out_spec, _ = fused_recurrent_kda(
|
||||
q=q_spec,
|
||||
k=k_spec,
|
||||
v=v_spec,
|
||||
raw_g=g1_spec,
|
||||
raw_beta=beta_spec,
|
||||
A_log=self.A_log,
|
||||
dt_bias=self.dt_bias,
|
||||
lower_bound=self.gate_lower_bound,
|
||||
initial_state=recurrent_state,
|
||||
cu_seqlens=spec_cu_seqlens,
|
||||
ssm_state_indices=spec_state_indices_tensor,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
out=spec_out,
|
||||
)
|
||||
if self.use_recoverssm:
|
||||
from vllm.models.kimi_k3.nvidia.ops.recoverssm import (
|
||||
kda_recoverssm_verify,
|
||||
)
|
||||
|
||||
if len(recoverssm_records) != 2:
|
||||
raise ValueError(
|
||||
"KDA RecoverSSM requires correction and key/gate buffers"
|
||||
)
|
||||
core_attn_out_spec = kda_recoverssm_verify(
|
||||
q=q_spec,
|
||||
k=k_spec,
|
||||
v=v_spec,
|
||||
raw_g=g1_spec,
|
||||
raw_beta=beta_spec,
|
||||
A_log=self.A_log,
|
||||
dt_bias=self.dt_bias,
|
||||
lower_bound=self.gate_lower_bound,
|
||||
checkpoint_state=recurrent_state,
|
||||
correction_cache=recoverssm_records[0],
|
||||
kg_cache=recoverssm_records[1],
|
||||
query_start_loc=spec_cu_seqlens,
|
||||
state_indices=spec_state_indices_tensor[: m.num_spec_decodes, 0],
|
||||
spec_query_len=self.spec_query_len,
|
||||
out=spec_out,
|
||||
)
|
||||
else:
|
||||
core_attn_out_spec, _ = fused_recurrent_kda(
|
||||
q=q_spec,
|
||||
k=k_spec,
|
||||
v=v_spec,
|
||||
raw_g=g1_spec,
|
||||
raw_beta=beta_spec,
|
||||
A_log=self.A_log,
|
||||
dt_bias=self.dt_bias,
|
||||
lower_bound=self.gate_lower_bound,
|
||||
initial_state=recurrent_state,
|
||||
cu_seqlens=spec_cu_seqlens,
|
||||
ssm_state_indices=spec_state_indices_tensor,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
out=spec_out,
|
||||
)
|
||||
|
||||
# Prefill or plain-decode path.
|
||||
core_attn_out_non_spec = None
|
||||
|
||||
@@ -6,13 +6,18 @@ The request classification and cudagraph staging intentionally mirror
|
||||
``GDNAttentionMetadataBuilder``. Kimi-K3 builds the metadata required by its
|
||||
prefill KDA kernel internally, so this builder omits the shared FLA chunk
|
||||
metadata construction.
|
||||
|
||||
For Kimi-K3 speculative decoding, ``--use-replayssm`` selects the simplified
|
||||
RecoverSSM path implemented here instead of the Mamba2 ReplaySSM kernel.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from functools import cache
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
@@ -22,6 +27,10 @@ from vllm.v1.attention.backends.gdn_attn import (
|
||||
GDNAttentionMetadata,
|
||||
GDNAttentionMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.backends.recoverssm_metadata import (
|
||||
RecoverSSMMetadata,
|
||||
RecoverSSMPostprocessMetadata,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
NULL_BLOCK_ID,
|
||||
compute_causal_conv1d_metadata,
|
||||
@@ -29,6 +38,11 @@ from vllm.v1.attention.backends.utils import (
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.models.kimi_k3.nvidia.ops.recoverssm import (
|
||||
KDARecoverSSMCommitContext,
|
||||
)
|
||||
|
||||
|
||||
@cache
|
||||
def _metadata_launch_pdl() -> bool:
|
||||
@@ -229,11 +243,103 @@ def stage_spec_decode_metadata(
|
||||
|
||||
|
||||
@dataclass
|
||||
class KimiK3KDAMetadata(GDNAttentionMetadata):
|
||||
pass
|
||||
class KDARecoverSSMAlignMetadata:
|
||||
block_table: torch.Tensor
|
||||
num_computed_tokens: torch.Tensor
|
||||
block_size: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class KDARecoverSSMCommitMetadata:
|
||||
state_indices: torch.Tensor
|
||||
query_start_loc: torch.Tensor
|
||||
request_indices: torch.Tensor | None
|
||||
align: KDARecoverSSMAlignMetadata | None
|
||||
|
||||
|
||||
@dataclass
|
||||
class KimiK3KDAMetadata(GDNAttentionMetadata, RecoverSSMMetadata):
|
||||
recoverssm_commit: KDARecoverSSMCommitMetadata | None = None
|
||||
recoverssm_context: "KDARecoverSSMCommitContext | None" = field(
|
||||
default=None, repr=False, compare=False
|
||||
)
|
||||
|
||||
def commit_recoverssm_state(
|
||||
self, num_accepted_tokens: torch.Tensor
|
||||
) -> RecoverSSMPostprocessMetadata | None:
|
||||
commit = self.recoverssm_commit
|
||||
if commit is None:
|
||||
return None
|
||||
context = self.recoverssm_context
|
||||
assert context is not None
|
||||
align = commit.align
|
||||
context.commit(
|
||||
num_accepted_tokens,
|
||||
commit.state_indices[: self.num_spec_decodes, 0],
|
||||
commit.query_start_loc[: self.num_spec_decodes + 1],
|
||||
request_indices=commit.request_indices,
|
||||
block_table=align.block_table if align is not None else None,
|
||||
num_computed_tokens=(
|
||||
align.num_computed_tokens if align is not None else None
|
||||
),
|
||||
mamba_block_size=align.block_size if align is not None else None,
|
||||
)
|
||||
if align is None:
|
||||
return None
|
||||
return RecoverSSMPostprocessMetadata(
|
||||
num_spec_decodes=self.num_spec_decodes,
|
||||
request_indices=commit.request_indices,
|
||||
block_table=align.block_table,
|
||||
num_computed_tokens=align.num_computed_tokens,
|
||||
block_size=align.block_size,
|
||||
)
|
||||
|
||||
|
||||
class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_spec: MambaSpec,
|
||||
layer_names: list[str],
|
||||
vllm_config: VllmConfig,
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
||||
self.use_recoverssm = vllm_config.cache_config.use_kda_recoverssm
|
||||
self.spec_state_slots = 1 if self.use_recoverssm else self.num_spec + 1
|
||||
self.recoverssm_num_accepted_tokens: torch.Tensor | None = None
|
||||
self.recoverssm_context: KDARecoverSSMCommitContext | None = None
|
||||
if self.use_recoverssm:
|
||||
max_num_reqs = vllm_config.scheduler_config.max_num_seqs
|
||||
self.spec_state_indices_tensor = torch.empty(
|
||||
(max_num_reqs, 1),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
self.recoverssm_num_accepted_tokens = torch.ones(
|
||||
max_num_reqs,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
|
||||
def _get_recoverssm_context(self) -> "KDARecoverSSMCommitContext":
|
||||
context = self.recoverssm_context
|
||||
if context is not None:
|
||||
return context
|
||||
|
||||
from vllm.models.kimi_k3.nvidia.ops.recoverssm import (
|
||||
KDARecoverSSMCommitContext,
|
||||
)
|
||||
|
||||
forward_context = self.vllm_config.compilation_config.static_forward_context
|
||||
layers = [forward_context[layer_name] for layer_name in self.layer_names]
|
||||
context = KDARecoverSSMCommitContext.create(
|
||||
layers,
|
||||
spec_query_len=1 + self.vllm_config.num_speculative_tokens,
|
||||
max_num_reqs=self.vllm_config.scheduler_config.max_num_seqs,
|
||||
)
|
||||
self.recoverssm_context = context
|
||||
return context
|
||||
|
||||
def build( # type: ignore[override]
|
||||
self,
|
||||
common_prefix_len: int,
|
||||
@@ -263,14 +369,26 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
|
||||
num_spec_decodes = 0
|
||||
else:
|
||||
spec_sequence_masks_cpu = num_decode_draft_tokens_cpu >= 0
|
||||
# A nonnegative entry identifies a spec request. If no draft token
|
||||
# was scheduled, process the whole batch as non-spec instead.
|
||||
if num_decode_draft_tokens_cpu[spec_sequence_masks_cpu].sum().item() == 0:
|
||||
if self.use_recoverssm:
|
||||
assert m.is_prefilling is not None
|
||||
assert m.is_prefilling.device.type == "cpu"
|
||||
active_decode_mask_cpu = (~m.is_prefilling) & (
|
||||
query_start_loc_cpu.diff() > 0
|
||||
)
|
||||
spec_sequence_masks_cpu |= active_decode_mask_cpu
|
||||
# Native KDA can use its regular decode path when no draft token
|
||||
# was scheduled. RecoverSSM must preserve its extended conv window.
|
||||
if (
|
||||
not self.use_recoverssm
|
||||
and num_decode_draft_tokens_cpu[spec_sequence_masks_cpu].sum().item()
|
||||
== 0
|
||||
):
|
||||
spec_sequence_masks_cpu = None
|
||||
num_spec_decodes = 0
|
||||
else:
|
||||
num_spec_decodes = spec_sequence_masks_cpu.sum().item()
|
||||
|
||||
spec_request_indices = None
|
||||
if num_spec_decodes == 0:
|
||||
# The runner orders ordinary decodes before prefills.
|
||||
num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = (
|
||||
@@ -289,6 +407,16 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
|
||||
assert spec_sequence_masks_cpu is not None
|
||||
assert num_accepted_tokens is not None
|
||||
query_lens_cpu = query_start_loc_cpu.diff()
|
||||
if (
|
||||
self.use_recoverssm
|
||||
and torch.any(
|
||||
query_lens_cpu[spec_sequence_masks_cpu] > self.num_spec + 1
|
||||
).item()
|
||||
):
|
||||
raise ValueError(
|
||||
"KDA RecoverSSM speculative decode query length exceeds "
|
||||
f"its activation capacity ({self.num_spec + 1})"
|
||||
)
|
||||
num_query_tokens = query_start_loc_cpu[-1].item()
|
||||
|
||||
# Exclude zero-length cudagraph padding from request-indexed
|
||||
@@ -325,7 +453,7 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
|
||||
non_spec_token_indx = None
|
||||
# Real requests precede trailing cudagraph padding.
|
||||
spec_state_indices_tensor = block_table_tensor[
|
||||
:num_spec_decodes, : self.num_spec + 1
|
||||
:num_spec_decodes, : self.spec_state_slots
|
||||
]
|
||||
non_spec_state_indices_tensor = None
|
||||
# Padding trails real requests, so this prefix already contains
|
||||
@@ -339,6 +467,12 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
|
||||
spec_sequence_masks_gpu = async_tensor_h2d(
|
||||
spec_sequence_masks_cpu, device=query_start_loc.device
|
||||
)
|
||||
if self.use_recoverssm:
|
||||
spec_request_indices = async_tensor_h2d(
|
||||
spec_sequence_masks_cpu.nonzero(as_tuple=True)[0],
|
||||
dtype=torch.int32,
|
||||
device=query_start_loc.device,
|
||||
)
|
||||
spec_token_masks = torch.repeat_interleave(
|
||||
spec_sequence_masks_gpu,
|
||||
query_lens,
|
||||
@@ -351,10 +485,10 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
|
||||
non_spec_token_indx = index[:num_non_spec_tokens]
|
||||
spec_token_indx = index[num_non_spec_tokens:]
|
||||
|
||||
# Spec requests carry one state slot per speculative step;
|
||||
# non-spec requests use only their current state slot.
|
||||
# Native spec uses one state slot per step. RecoverSSM keeps
|
||||
# only the current checkpoint slot.
|
||||
spec_state_indices_tensor = block_table_tensor[
|
||||
spec_sequence_masks_cpu, : self.num_spec + 1
|
||||
spec_sequence_masks_cpu, : self.spec_state_slots
|
||||
]
|
||||
non_spec_state_indices_tensor = block_table_tensor[
|
||||
active_non_spec_mask_cpu, 0
|
||||
@@ -400,12 +534,18 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
|
||||
|
||||
num_accepted_tokens = num_accepted_tokens[spec_sequence_masks_cpu]
|
||||
|
||||
if self.use_recoverssm:
|
||||
assert self.recoverssm_num_accepted_tokens is not None
|
||||
num_accepted_tokens = self.recoverssm_num_accepted_tokens[
|
||||
:num_spec_decodes
|
||||
]
|
||||
|
||||
# Unlike the shared GDN layer, Kimi-K3's prefill KDA wrapper prepares
|
||||
# its own chunk indices. Only causal-convolution metadata is needed here.
|
||||
nums_dict, batch_ptr, token_chunk_offset_ptr = None, None, None
|
||||
if num_prefills > 0:
|
||||
has_initial_state = m.compute_num_computed_tokens() > 0
|
||||
if spec_sequence_masks_cpu is not None:
|
||||
if num_spec_decodes > 0:
|
||||
has_initial_state = has_initial_state[active_non_spec_mask_cpu]
|
||||
assert non_spec_query_start_loc_cpu is not None
|
||||
nums_dict, batch_ptr, token_chunk_offset_ptr = (
|
||||
@@ -426,7 +566,7 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
|
||||
and num_spec_decodes > 0
|
||||
and num_prefills == 0
|
||||
and num_decodes == 0
|
||||
and num_spec_decodes <= self.decode_cudagraph_max_bs
|
||||
and batch_size <= self.spec_state_indices_tensor.shape[0]
|
||||
and num_spec_decode_tokens <= self.decode_cudagraph_max_bs
|
||||
):
|
||||
# Equivalent PyTorch staging:
|
||||
@@ -463,6 +603,24 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
|
||||
:batch_size
|
||||
]
|
||||
|
||||
recoverssm_commit = None
|
||||
if self.use_recoverssm and num_spec_decodes > 0:
|
||||
assert spec_state_indices_tensor is not None
|
||||
assert spec_query_start_loc is not None
|
||||
align = None
|
||||
if self.kv_cache_spec.mamba_cache_mode == "align":
|
||||
align = KDARecoverSSMAlignMetadata(
|
||||
block_table=m.block_table_tensor,
|
||||
num_computed_tokens=m.compute_num_computed_tokens(),
|
||||
block_size=self.kv_cache_spec.block_size,
|
||||
)
|
||||
recoverssm_commit = KDARecoverSSMCommitMetadata(
|
||||
state_indices=spec_state_indices_tensor,
|
||||
query_start_loc=spec_query_start_loc,
|
||||
request_indices=spec_request_indices,
|
||||
align=align,
|
||||
)
|
||||
|
||||
return KimiK3KDAMetadata(
|
||||
num_prefills=num_prefills,
|
||||
num_prefill_tokens=num_prefill_tokens,
|
||||
@@ -480,6 +638,12 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
|
||||
spec_token_indx=spec_token_indx,
|
||||
non_spec_token_indx=non_spec_token_indx,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
recoverssm_commit=recoverssm_commit,
|
||||
recoverssm_context=(
|
||||
self._get_recoverssm_context()
|
||||
if recoverssm_commit is not None
|
||||
else None
|
||||
),
|
||||
nums_dict=nums_dict,
|
||||
batch_ptr=batch_ptr,
|
||||
token_chunk_offset_ptr=token_chunk_offset_ptr,
|
||||
|
||||
@@ -62,12 +62,14 @@ from vllm.model_executor.models.interfaces import (
|
||||
EagleModelMixin,
|
||||
HasInnerState,
|
||||
IsHybrid,
|
||||
MambaStateShapes,
|
||||
MixtureOfExperts,
|
||||
SupportsEagle3,
|
||||
SupportsEncoderCudaGraph,
|
||||
SupportsMultiModal,
|
||||
SupportsPP,
|
||||
SupportsQuant,
|
||||
SupportsReplaySSM,
|
||||
)
|
||||
from vllm.model_executor.models.kimi_k25 import KimiK25MediaPixelInputs
|
||||
from vllm.model_executor.models.kimi_k25_vit import (
|
||||
@@ -1559,7 +1561,13 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
|
||||
|
||||
|
||||
class KimiLinearForCausalLM(
|
||||
nn.Module, HasInnerState, SupportsPP, MixtureOfExperts, IsHybrid, SupportsEagle3
|
||||
nn.Module,
|
||||
HasInnerState,
|
||||
SupportsPP,
|
||||
MixtureOfExperts,
|
||||
IsHybrid,
|
||||
SupportsEagle3,
|
||||
SupportsReplaySSM,
|
||||
):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
@@ -1613,15 +1621,20 @@ class KimiLinearForCausalLM(
|
||||
def get_mamba_state_dtype_from_config(
|
||||
cls,
|
||||
vllm_config: "VllmConfig",
|
||||
) -> tuple[torch.dtype, torch.dtype]:
|
||||
return MambaStateDtypeCalculator.kda_state_dtype(
|
||||
) -> tuple[torch.dtype, ...]:
|
||||
dtypes = MambaStateDtypeCalculator.kda_state_dtype(
|
||||
vllm_config.model_config.dtype, vllm_config.cache_config.mamba_cache_dtype
|
||||
)
|
||||
if vllm_config.cache_config.use_kda_recoverssm:
|
||||
dtypes = MambaStateDtypeCalculator.append_kda_recoverssm_record(
|
||||
dtypes, vllm_config.model_config.dtype
|
||||
)
|
||||
return dtypes
|
||||
|
||||
@classmethod
|
||||
def get_mamba_state_shape_from_config(
|
||||
cls, vllm_config: "VllmConfig"
|
||||
) -> tuple[tuple[int, int], tuple[int, int, int]]:
|
||||
) -> MambaStateShapes:
|
||||
parallel_config = vllm_config.parallel_config
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
tp_size = parallel_config.tensor_parallel_size
|
||||
@@ -1630,13 +1643,22 @@ class KimiLinearForCausalLM(
|
||||
if vllm_config.speculative_config
|
||||
else 0
|
||||
)
|
||||
return MambaStateShapeCalculator.kda_state_shape(
|
||||
shapes = MambaStateShapeCalculator.kda_state_shape(
|
||||
tp_size,
|
||||
hf_config.linear_attn_config["num_heads"],
|
||||
hf_config.linear_attn_config["head_dim"],
|
||||
conv_kernel_size=hf_config.linear_attn_config["short_conv_kernel_size"],
|
||||
num_spec=num_spec,
|
||||
)
|
||||
if vllm_config.cache_config.use_kda_recoverssm:
|
||||
return MambaStateShapeCalculator.append_kda_recoverssm_record(
|
||||
shapes,
|
||||
hf_config.linear_attn_config["num_heads"],
|
||||
hf_config.linear_attn_config["head_dim"],
|
||||
tp_world_size=tp_size,
|
||||
spec_query_len=1 + num_spec,
|
||||
)
|
||||
return shapes
|
||||
|
||||
@classmethod
|
||||
def get_mamba_state_copy_func(
|
||||
@@ -1697,6 +1719,7 @@ class KimiK3ForConditionalGeneration(
|
||||
SupportsEagle3,
|
||||
HasInnerState,
|
||||
IsHybrid,
|
||||
SupportsReplaySSM,
|
||||
):
|
||||
"""Kimi-K3 model with Kimi-K2.5 vision and KimiLinear text."""
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user