[New model] Kimi K3 (#50000)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com> Co-authored-by: Bowen Wang <abmfy@icloud.com> Co-authored-by: gnovack <novackgm@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com> Co-authored-by: Roy Wang <yasong.wang@inferact.ai> Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com> Co-authored-by: Codex <codex@openai.com> Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
This commit is contained in:
co-authored by
Isotr0py
Thien Tran
Bugen Zhao
Yongye Zhu
Ziming Huang
Roger Wang
Isotr0py
aoshen02
Woosuk Kwon
Jee Jee Li
aoshen02
Summer Yang
Kevin H. Luu
Bowen Wang
gnovack
Nick Hill
Yifan Qiao
xiaozhoupy
Roy Wang
Jeff Ma
Codex
Wei Zhao
parent
0c64be8873
commit
aeeb36b1f1
@@ -385,6 +385,8 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
alignment: int | None = None # Default to None for no padding.
|
||||
compress_ratio: int = 1 # Default to 1 for no compression.
|
||||
model_version: str | None = None
|
||||
# Marks draft groups that flatten a non-causal query block into decode rows.
|
||||
non_causal_multi_token_decode: bool = False
|
||||
|
||||
def __post_init__(self):
|
||||
super().__post_init__()
|
||||
@@ -434,7 +436,7 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
"quantization method, compress ratio, model version, and KV block "
|
||||
"stride indexing."
|
||||
)
|
||||
return cls(
|
||||
merged_spec = cls(
|
||||
block_size=specs[0].block_size,
|
||||
num_kv_heads=specs[0].num_kv_heads,
|
||||
head_size=specs[0].head_size,
|
||||
@@ -445,7 +447,17 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
cache_dtype_str=cache_dtype_str_set.pop(),
|
||||
compress_ratio=compress_ratio_set.pop(),
|
||||
model_version=model_version_set.pop(),
|
||||
non_causal_multi_token_decode=any(
|
||||
spec.non_causal_multi_token_decode for spec in specs
|
||||
),
|
||||
)
|
||||
for spec in specs:
|
||||
for f in fields(AttentionSpec):
|
||||
assert getattr(spec, f.name) == getattr(merged_spec, f.name), (
|
||||
"All attention layers in the same KV cache group must have "
|
||||
"the same attention spec."
|
||||
)
|
||||
return merged_spec
|
||||
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
|
||||
Reference in New Issue
Block a user