[Model] Support Qwen3.8-Flash-Next (#53896)
Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com> Signed-off-by: andyluo7 <andy.luo@amd.com> Signed-off-by: lishunyang12 <lishunyang03@gmail.com> Signed-off-by: Harry Huang <huanghaoyan.hhy@alibaba-inc.com> Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: KungYork <firelikest@163.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: andyluo7 <43718156+andyluo7@users.noreply.github.com> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: andyluo7 <andy.luo@amd.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: lishunyang12 <lishunyang03@gmail.com> Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
zjy0516
KungYork
Thien Tran
Yifan Qiao
andyluo7
Isotr0py
andyluo7
OpenAI Codex
lishunyang12
Vadim Gimpelson
Isotr0py
Lucas Wilkinson
Claude Opus 5
OpenAI Codex
parent
687db59744
commit
e126687a9a
@@ -150,6 +150,10 @@ class KVCacheSpec:
|
||||
# number of tokens in a block
|
||||
block_size: int
|
||||
|
||||
@property
|
||||
def prefix_cacheable(self) -> bool:
|
||||
return True
|
||||
|
||||
@property
|
||||
def num_heads(self) -> int:
|
||||
raise NotImplementedError
|
||||
@@ -745,6 +749,38 @@ class SlidingWindowSpec(AttentionSpec):
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class CircularBufferSpec(AttentionSpec):
|
||||
"""One block per request holding the raw keys of the token group that
|
||||
is still being compressed.
|
||||
|
||||
``block_size`` is the ring capacity. It must exceed the compression ratio
|
||||
by the speculative lookahead: a speculative step stores all of its rows,
|
||||
drafts included, before acceptance is known, while the next step still
|
||||
reads the open group's committed keys from the ring.
|
||||
"""
|
||||
|
||||
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
|
||||
# The ring occupies one block per request for its whole lifetime.
|
||||
del vllm_config
|
||||
return self.page_size_bytes
|
||||
|
||||
def max_num_blocks_per_req(self, vllm_config: VllmConfig, max_len: int) -> int:
|
||||
del vllm_config, max_len
|
||||
return 1
|
||||
|
||||
def is_uniform_with_collection(
|
||||
self, kv_cache_specs: dict[str, KVCacheSpec]
|
||||
) -> bool:
|
||||
return all(
|
||||
isinstance(spec, CircularBufferSpec) for spec in kv_cache_specs.values()
|
||||
)
|
||||
|
||||
@property
|
||||
def prefix_cacheable(self) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
"""Sliding window attention with MLA cache format."""
|
||||
@@ -814,7 +850,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
@dataclass(frozen=True)
|
||||
class MambaSpec(KVCacheSpec):
|
||||
shapes: tuple[tuple[int, ...], ...]
|
||||
dtypes: tuple[torch.dtype]
|
||||
dtypes: tuple[torch.dtype, ...]
|
||||
page_size_padded: int | None = None
|
||||
mamba_type: MambaAttentionBackendEnum = MambaAttentionBackendEnum.MAMBA2
|
||||
mamba_cache_mode: str = "none"
|
||||
@@ -822,6 +858,9 @@ class MambaSpec(KVCacheSpec):
|
||||
num_prefill_checkpoint_blocks: int = 0
|
||||
num_heads: int = 1
|
||||
tokens_per_state: int = -1
|
||||
# False: the state is sharded across TP ranks (e.g. GDN). True: every TP
|
||||
# rank holds the full state (e.g. the replicated PLE conv state).
|
||||
tp_replicated: bool = False
|
||||
|
||||
@property
|
||||
def state_content_size_bytes(self) -> int:
|
||||
@@ -873,6 +912,8 @@ class MambaSpec(KVCacheSpec):
|
||||
isinstance(spec, MambaSpec)
|
||||
and spec.num_speculative_blocks == self.num_speculative_blocks
|
||||
and spec.num_prefill_checkpoint_blocks == self.num_prefill_checkpoint_blocks
|
||||
and spec.page_size_bytes == self.page_size_bytes
|
||||
and spec.tp_replicated == self.tp_replicated
|
||||
for spec in kv_cache_specs.values()
|
||||
)
|
||||
|
||||
@@ -963,6 +1004,15 @@ class UniformTypeKVCacheSpecs(KVCacheSpec):
|
||||
|
||||
kv_cache_specs: dict[str, KVCacheSpec]
|
||||
|
||||
@property
|
||||
def prefix_cacheable(self) -> bool:
|
||||
return all(spec.prefix_cacheable for spec in self.kv_cache_specs.values())
|
||||
|
||||
@property
|
||||
def first_spec(self) -> KVCacheSpec:
|
||||
"""Return the first spec in the group."""
|
||||
return next(iter(self.kv_cache_specs.values()))
|
||||
|
||||
@property
|
||||
def page_size_bytes(self) -> int:
|
||||
return sum(spec.page_size_bytes for spec in self.kv_cache_specs.values())
|
||||
@@ -1015,8 +1065,9 @@ class UniformTypeKVCacheSpecs(KVCacheSpec):
|
||||
else:
|
||||
return None
|
||||
|
||||
# NOTE: below util functions are only used by DeepseekV4 for now.
|
||||
def get_num_layer_tuples(self) -> int:
|
||||
def get_max_layers_per_page_size(self) -> int:
|
||||
"""Max number of layers sharing a page size. For a balanced bucket
|
||||
this equals the number of repetitions of the layer pattern."""
|
||||
return Counter(
|
||||
spec.page_size_bytes for spec in self.kv_cache_specs.values()
|
||||
).most_common(1)[0][1]
|
||||
@@ -1212,7 +1263,11 @@ class KVCacheConfig:
|
||||
|
||||
@property
|
||||
def has_mamba_layers(self) -> bool:
|
||||
return any(isinstance(g.kv_cache_spec, MambaSpec) for g in self.kv_cache_groups)
|
||||
return any(
|
||||
isinstance(spec, MambaSpec)
|
||||
for group in self.kv_cache_groups
|
||||
for spec in iter_layer_specs(group.kv_cache_spec)
|
||||
)
|
||||
|
||||
@property
|
||||
def has_mixed_precision_kv_cache(self) -> bool:
|
||||
|
||||
Reference in New Issue
Block a user