[Perf] Kimi K3 Specialized Decode: CuTeDSL Optimized GB300 (1.5-2x) (#61867)

Signed-off-by: Benjamin Chislett <bchislett@nvidia.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: vllm-agent <claw@inferact.ai>
This commit is contained in:
Benjamin Chislett
2026-10-15 22:50:41 -04:00
committed by GitHub
co-authored by Claude Opus 5.5 vllm-agent
parent 3ddc6702a8
commit 642c18d64c
50 changed files with 12805 additions and 3 deletions
+48 -3
View File
@@ -4,7 +4,7 @@
import math
from collections.abc import Hashable, Iterable
from typing import Any, cast
from typing import TYPE_CHECKING, Any, cast
import torch
from torch import nn
@@ -121,6 +121,12 @@ from ..common.mm_preprocess import (
KimiK3MultiModalProcessor,
KimiK3ProcessingInfo,
)
from .specialized_decode.runtime import specialized_decode_config
if TYPE_CHECKING:
from vllm.v1.worker.gpu.model_states.interface import ModelState
from .specialized_decode.runtime.decode import SpecializedDecode
logger = init_logger(__name__)
@@ -1421,8 +1427,34 @@ class KimiLinearForCausalLM(
self.logits_processor = LogitsProcessor(
self.config.vocab_size, scale=logit_scale
)
# decode_backend=latency_specialized: a specialized decode executor over
# self.model, built by the model state once the weights are loaded.
self.specialized_decode_config = specialized_decode_config(vllm_config)
self.specialized_decode: SpecializedDecode | None = None
# The final norm is applied in compute_logits unless the specialized decode
# applies it in forward.
self.needs_norm_in_logits = True
def build_specialized_decode(self) -> None:
from .specialized_decode.runtime.decode import SpecializedDecode
self.specialized_decode = SpecializedDecode(
self.model, self.specialized_decode_config
)
# From here on forward() returns normalized hidden states on every step: the
# specialized decode's last AttnRes applies the norm, and forward() applies it
# to each step that falls back to the stock forward (prefills, larger batches,
# PIECEWISE steps).
self.needs_norm_in_logits = False
def get_model_state_cls(self) -> "type[ModelState]":
from .model_state import KimiK3ModelState
return KimiK3ModelState
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
if self.specialized_decode is not None:
return self.specialized_decode.embed_input_ids(input_ids)
return self.model.embed_input_ids(input_ids)
def make_empty_intermediate_tensors(
@@ -1441,9 +1473,18 @@ class KimiLinearForCausalLM(
inputs_embeds: torch.Tensor | None = None,
**kwargs,
) -> torch.Tensor | IntermediateTensors | tuple[torch.Tensor, list[torch.Tensor]]:
return self.model(
decode = self.specialized_decode
if decode is not None:
step = decode.plan(positions, intermediate_tensors)
if step is not None:
return decode.run(step, input_ids, inputs_embeds)
hidden_states = self.model(
input_ids, positions, intermediate_tensors, inputs_embeds, **kwargs
)
if not self.needs_norm_in_logits:
assert decode is not None
hidden_states = decode.normalize(hidden_states)
return hidden_states
@classmethod
def get_mamba_state_dtype_from_config(
@@ -1502,7 +1543,8 @@ class KimiLinearForCausalLM(
) -> torch.Tensor | None:
# The model's final norm is applied here (not at the end of forward) so
# that the pre-norm hidden states can be fed to the MTP draft model.
hidden_states = self.model.norm(hidden_states, None)
if self.needs_norm_in_logits:
hidden_states = self.model.norm(hidden_states, None)
return self.logits_processor(self.lm_head, hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
@@ -1926,6 +1968,9 @@ class KimiK3ForConditionalGeneration(
def compute_logits(self, hidden_states: torch.Tensor, **kwargs) -> torch.Tensor:
return self.language_model.compute_logits(hidden_states)
def get_model_state_cls(self) -> "type[ModelState]":
return self.language_model.get_model_state_cls()
def copy_inputs_before_cuda_graphs(self, input_buffers, **kwargs):
return self.language_model.mamba_cache.copy_inputs_before_cuda_graphs(
input_buffers, **kwargs