[New model] Kimi K3 (#50000)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Summer Yang <girasoleyang@gmail.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
Co-authored-by: Bowen Wang <abmfy@icloud.com>
Co-authored-by: gnovack <novackgm@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com>
Co-authored-by: Roy Wang <yasong.wang@inferact.ai>
Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com>
Co-authored-by: Codex <codex@openai.com>
Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
This commit is contained in:
Jiangyun Zhu
2026-07-30 03:49:58 -07:00
committed by GitHub
co-authored by Isotr0py Thien Tran Bugen Zhao Yongye Zhu Ziming Huang Roger Wang Isotr0py aoshen02 Woosuk Kwon Jee Jee Li aoshen02 Summer Yang Kevin H. Luu Bowen Wang gnovack Nick Hill Yifan Qiao xiaozhoupy Roy Wang Jeff Ma Codex Wei Zhao
parent 0c64be8873
commit aeeb36b1f1
82 changed files with 2927 additions and 1345 deletions
@@ -40,6 +40,8 @@ def test_mamba_align_split_partial_tail_schedule():
hash_block_size = 32
mock = SimpleNamespace(
cache_config=SimpleNamespace(block_size=block_size),
max_num_scheduled_tokens=8192,
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
use_eagle=False,
hash_block_size=hash_block_size,
mamba_partial_cache_hit=True,
@@ -75,6 +77,78 @@ def test_mamba_align_split_partial_tail_schedule():
assert split(self=mock, request=req2, num_new_tokens=1000) == 512
def test_mamba_align_split_when_block_exceeds_scheduling_budget():
"""Sub-block chunks make progress only when no step can fit a full block."""
block_size = 11392
token_budget = 8192
prompt_length = 30000
mock = SimpleNamespace(
cache_config=SimpleNamespace(block_size=block_size),
max_num_scheduled_tokens=token_budget,
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
use_eagle=False,
hash_block_size=32,
mamba_partial_cache_hit=False,
)
req = make_request("0", [0] * prompt_length, 32, sha256)
split = Scheduler._mamba_block_aligned_split
mock.max_num_scheduled_tokens = block_size
assert split(self=mock, request=req, num_new_tokens=token_budget) == 0
mock.max_num_scheduled_tokens = token_budget
scheduled_chunks = []
while req.num_computed_tokens < prompt_length:
num_new_tokens = min(token_budget, prompt_length - req.num_computed_tokens)
num_scheduled_tokens = split(
self=mock,
request=req,
num_new_tokens=num_new_tokens,
)
assert 0 < num_scheduled_tokens <= token_budget
scheduled_chunks.append(num_scheduled_tokens)
req.num_computed_tokens += num_scheduled_tokens
assert scheduled_chunks == [8192, 3200, 8192, 3200, 7216]
def test_mamba_align_split_when_block_exceeds_long_prefill_threshold():
"""A long-prefill cap below the block size permits sub-block progress."""
block_size = 512
token_budget = 8192
long_prefill_threshold = 384
prompt_length = 1300
mock = SimpleNamespace(
cache_config=SimpleNamespace(block_size=block_size),
max_num_scheduled_tokens=token_budget,
scheduler_config=SimpleNamespace(
long_prefill_token_threshold=long_prefill_threshold
),
use_eagle=False,
hash_block_size=32,
mamba_partial_cache_hit=False,
)
req = make_request("0", [0] * prompt_length, 32, sha256)
split = Scheduler._mamba_block_aligned_split
scheduled_chunks = []
while req.num_computed_tokens < prompt_length:
num_new_tokens = min(
long_prefill_threshold,
prompt_length - req.num_computed_tokens,
)
num_scheduled_tokens = split(
self=mock,
request=req,
num_new_tokens=num_new_tokens,
)
assert 0 < num_scheduled_tokens <= long_prefill_threshold
scheduled_chunks.append(num_scheduled_tokens)
req.num_computed_tokens += num_scheduled_tokens
assert scheduled_chunks == [384, 128, 384, 128, 276]
def test_hybrid_mamba_align_partial_hash_hit():
hash_block_size = 2
mamba_block_size = 2 * hash_block_size