[Perf] Kimi K3 Specialized Decode: CuTeDSL Optimized GB300 (1.5-2x) (#61867)
Signed-off-by: Benjamin Chislett <bchislett@nvidia.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: vllm-agent <claw@inferact.ai>
This commit is contained in:
co-authored by
Claude Opus 5.5
vllm-agent
parent
3ddc6702a8
commit
642c18d64c
@@ -0,0 +1,26 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Kimi K3's model runner state: the hybrid (KDA + MLA) state, plus the lifecycle of
|
||||
the specialized decode when ``decode_backend=latency_specialized`` selects it."""
|
||||
|
||||
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
|
||||
|
||||
|
||||
class KimiK3ModelState(MambaHybridModelState):
|
||||
"""Builds the specialized decode once the weights are loaded and the aux layers
|
||||
set, and prepares it for every bound KV cache (the profiling cache, then the
|
||||
real one)."""
|
||||
|
||||
def __init__(self, vllm_config, model, encoder_cache, device):
|
||||
super().__init__(vllm_config, model, encoder_cache, device)
|
||||
if hasattr(model, "get_language_model"):
|
||||
model = model.get_language_model()
|
||||
self.language_model = model
|
||||
if model.specialized_decode_config is not None:
|
||||
model.build_specialized_decode()
|
||||
|
||||
def initialize_kv_cache(self, kv_cache_config, block_tables):
|
||||
super().initialize_kv_cache(kv_cache_config, block_tables)
|
||||
decode = self.language_model.specialized_decode
|
||||
if decode is not None:
|
||||
decode.prepare_kv()
|
||||
Reference in New Issue
Block a user