[New model] Kimi K3 (#50000)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com> Co-authored-by: Bowen Wang <abmfy@icloud.com> Co-authored-by: gnovack <novackgm@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com> Co-authored-by: Roy Wang <yasong.wang@inferact.ai> Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com> Co-authored-by: Codex <codex@openai.com> Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
This commit is contained in:
co-authored by
Isotr0py
Thien Tran
Bugen Zhao
Yongye Zhu
Ziming Huang
Roger Wang
Isotr0py
aoshen02
Woosuk Kwon
Jee Jee Li
aoshen02
Summer Yang
Kevin H. Luu
Bowen Wang
gnovack
Nick Hill
Yifan Qiao
xiaozhoupy
Roy Wang
Jeff Ma
Codex
Wei Zhao
parent
0c64be8873
commit
aeeb36b1f1
@@ -40,6 +40,8 @@ def test_mamba_align_split_partial_tail_schedule():
|
||||
hash_block_size = 32
|
||||
mock = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=block_size),
|
||||
max_num_scheduled_tokens=8192,
|
||||
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
|
||||
use_eagle=False,
|
||||
hash_block_size=hash_block_size,
|
||||
mamba_partial_cache_hit=True,
|
||||
@@ -75,6 +77,78 @@ def test_mamba_align_split_partial_tail_schedule():
|
||||
assert split(self=mock, request=req2, num_new_tokens=1000) == 512
|
||||
|
||||
|
||||
def test_mamba_align_split_when_block_exceeds_scheduling_budget():
|
||||
"""Sub-block chunks make progress only when no step can fit a full block."""
|
||||
block_size = 11392
|
||||
token_budget = 8192
|
||||
prompt_length = 30000
|
||||
mock = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=block_size),
|
||||
max_num_scheduled_tokens=token_budget,
|
||||
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
|
||||
use_eagle=False,
|
||||
hash_block_size=32,
|
||||
mamba_partial_cache_hit=False,
|
||||
)
|
||||
req = make_request("0", [0] * prompt_length, 32, sha256)
|
||||
split = Scheduler._mamba_block_aligned_split
|
||||
|
||||
mock.max_num_scheduled_tokens = block_size
|
||||
assert split(self=mock, request=req, num_new_tokens=token_budget) == 0
|
||||
mock.max_num_scheduled_tokens = token_budget
|
||||
|
||||
scheduled_chunks = []
|
||||
while req.num_computed_tokens < prompt_length:
|
||||
num_new_tokens = min(token_budget, prompt_length - req.num_computed_tokens)
|
||||
num_scheduled_tokens = split(
|
||||
self=mock,
|
||||
request=req,
|
||||
num_new_tokens=num_new_tokens,
|
||||
)
|
||||
assert 0 < num_scheduled_tokens <= token_budget
|
||||
scheduled_chunks.append(num_scheduled_tokens)
|
||||
req.num_computed_tokens += num_scheduled_tokens
|
||||
|
||||
assert scheduled_chunks == [8192, 3200, 8192, 3200, 7216]
|
||||
|
||||
|
||||
def test_mamba_align_split_when_block_exceeds_long_prefill_threshold():
|
||||
"""A long-prefill cap below the block size permits sub-block progress."""
|
||||
block_size = 512
|
||||
token_budget = 8192
|
||||
long_prefill_threshold = 384
|
||||
prompt_length = 1300
|
||||
mock = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=block_size),
|
||||
max_num_scheduled_tokens=token_budget,
|
||||
scheduler_config=SimpleNamespace(
|
||||
long_prefill_token_threshold=long_prefill_threshold
|
||||
),
|
||||
use_eagle=False,
|
||||
hash_block_size=32,
|
||||
mamba_partial_cache_hit=False,
|
||||
)
|
||||
req = make_request("0", [0] * prompt_length, 32, sha256)
|
||||
split = Scheduler._mamba_block_aligned_split
|
||||
|
||||
scheduled_chunks = []
|
||||
while req.num_computed_tokens < prompt_length:
|
||||
num_new_tokens = min(
|
||||
long_prefill_threshold,
|
||||
prompt_length - req.num_computed_tokens,
|
||||
)
|
||||
num_scheduled_tokens = split(
|
||||
self=mock,
|
||||
request=req,
|
||||
num_new_tokens=num_new_tokens,
|
||||
)
|
||||
assert 0 < num_scheduled_tokens <= long_prefill_threshold
|
||||
scheduled_chunks.append(num_scheduled_tokens)
|
||||
req.num_computed_tokens += num_scheduled_tokens
|
||||
|
||||
assert scheduled_chunks == [384, 128, 384, 128, 276]
|
||||
|
||||
|
||||
def test_hybrid_mamba_align_partial_hash_hit():
|
||||
hash_block_size = 2
|
||||
mamba_block_size = 2 * hash_block_size
|
||||
|
||||
Reference in New Issue
Block a user