[Model] Support Qwen3.8-Flash-Next (#53896)

Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com>
Signed-off-by: andyluo7 <andy.luo@amd.com>
Signed-off-by: lishunyang12 <lishunyang03@gmail.com>
Signed-off-by: Harry Huang <huanghaoyan.hhy@alibaba-inc.com>
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: KungYork <firelikest@163.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: andyluo7 <43718156+andyluo7@users.noreply.github.com>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: andyluo7 <andy.luo@amd.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: lishunyang12 <lishunyang03@gmail.com>
Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Harry Huang
2026-08-31 05:57:56 +00:00
committed by GitHub
co-authored by zjy0516 KungYork Thien Tran Yifan Qiao andyluo7 Isotr0py andyluo7 OpenAI Codex lishunyang12 Vadim Gimpelson Isotr0py Lucas Wilkinson Claude Opus 5 OpenAI Codex
parent 687db59744
commit e126687a9a
124 changed files with 19424 additions and 450 deletions
+59 -4
View File
@@ -150,6 +150,10 @@ class KVCacheSpec:
# number of tokens in a block
block_size: int
@property
def prefix_cacheable(self) -> bool:
return True
@property
def num_heads(self) -> int:
raise NotImplementedError
@@ -745,6 +749,38 @@ class SlidingWindowSpec(AttentionSpec):
)
@dataclass(frozen=True, kw_only=True)
class CircularBufferSpec(AttentionSpec):
"""One block per request holding the raw keys of the token group that
is still being compressed.
``block_size`` is the ring capacity. It must exceed the compression ratio
by the speculative lookahead: a speculative step stores all of its rows,
drafts included, before acceptance is known, while the next step still
reads the open group's committed keys from the ring.
"""
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
# The ring occupies one block per request for its whole lifetime.
del vllm_config
return self.page_size_bytes
def max_num_blocks_per_req(self, vllm_config: VllmConfig, max_len: int) -> int:
del vllm_config, max_len
return 1
def is_uniform_with_collection(
self, kv_cache_specs: dict[str, KVCacheSpec]
) -> bool:
return all(
isinstance(spec, CircularBufferSpec) for spec in kv_cache_specs.values()
)
@property
def prefix_cacheable(self) -> bool:
return False
@dataclass(frozen=True, kw_only=True)
class SlidingWindowMLASpec(SlidingWindowSpec):
"""Sliding window attention with MLA cache format."""
@@ -814,7 +850,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
@dataclass(frozen=True)
class MambaSpec(KVCacheSpec):
shapes: tuple[tuple[int, ...], ...]
dtypes: tuple[torch.dtype]
dtypes: tuple[torch.dtype, ...]
page_size_padded: int | None = None
mamba_type: MambaAttentionBackendEnum = MambaAttentionBackendEnum.MAMBA2
mamba_cache_mode: str = "none"
@@ -822,6 +858,9 @@ class MambaSpec(KVCacheSpec):
num_prefill_checkpoint_blocks: int = 0
num_heads: int = 1
tokens_per_state: int = -1
# False: the state is sharded across TP ranks (e.g. GDN). True: every TP
# rank holds the full state (e.g. the replicated PLE conv state).
tp_replicated: bool = False
@property
def state_content_size_bytes(self) -> int:
@@ -873,6 +912,8 @@ class MambaSpec(KVCacheSpec):
isinstance(spec, MambaSpec)
and spec.num_speculative_blocks == self.num_speculative_blocks
and spec.num_prefill_checkpoint_blocks == self.num_prefill_checkpoint_blocks
and spec.page_size_bytes == self.page_size_bytes
and spec.tp_replicated == self.tp_replicated
for spec in kv_cache_specs.values()
)
@@ -963,6 +1004,15 @@ class UniformTypeKVCacheSpecs(KVCacheSpec):
kv_cache_specs: dict[str, KVCacheSpec]
@property
def prefix_cacheable(self) -> bool:
return all(spec.prefix_cacheable for spec in self.kv_cache_specs.values())
@property
def first_spec(self) -> KVCacheSpec:
"""Return the first spec in the group."""
return next(iter(self.kv_cache_specs.values()))
@property
def page_size_bytes(self) -> int:
return sum(spec.page_size_bytes for spec in self.kv_cache_specs.values())
@@ -1015,8 +1065,9 @@ class UniformTypeKVCacheSpecs(KVCacheSpec):
else:
return None
# NOTE: below util functions are only used by DeepseekV4 for now.
def get_num_layer_tuples(self) -> int:
def get_max_layers_per_page_size(self) -> int:
"""Max number of layers sharing a page size. For a balanced bucket
this equals the number of repetitions of the layer pattern."""
return Counter(
spec.page_size_bytes for spec in self.kv_cache_specs.values()
).most_common(1)[0][1]
@@ -1212,7 +1263,11 @@ class KVCacheConfig:
@property
def has_mamba_layers(self) -> bool:
return any(isinstance(g.kv_cache_spec, MambaSpec) for g in self.kv_cache_groups)
return any(
isinstance(spec, MambaSpec)
for group in self.kv_cache_groups
for spec in iter_layer_specs(group.kv_cache_spec)
)
@property
def has_mixed_precision_kv_cache(self) -> bool: