[Model] Support Qwen3.8-Flash-Next (#53896)
Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com> Signed-off-by: andyluo7 <andy.luo@amd.com> Signed-off-by: lishunyang12 <lishunyang03@gmail.com> Signed-off-by: Harry Huang <huanghaoyan.hhy@alibaba-inc.com> Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: KungYork <firelikest@163.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: andyluo7 <43718156+andyluo7@users.noreply.github.com> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: andyluo7 <andy.luo@amd.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: lishunyang12 <lishunyang03@gmail.com> Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
zjy0516
KungYork
Thien Tran
Yifan Qiao
andyluo7
Isotr0py
andyluo7
OpenAI Codex
lishunyang12
Vadim Gimpelson
Isotr0py
Lucas Wilkinson
Claude Opus 5
OpenAI Codex
parent
687db59744
commit
e126687a9a
@@ -56,6 +56,10 @@ class MambaBase(AttentionLayerBase):
|
||||
def mamba_type(self) -> MambaAttentionBackendEnum:
|
||||
pass
|
||||
|
||||
@property
|
||||
def is_kv_cache_tp_replicated(self) -> bool:
|
||||
return False
|
||||
|
||||
@abstractmethod
|
||||
def get_state_dtype(self) -> tuple[torch.dtype, ...]:
|
||||
pass
|
||||
@@ -70,6 +74,7 @@ class MambaBase(AttentionLayerBase):
|
||||
block_size=mamba_block_size,
|
||||
page_size_padded=page_size_padded,
|
||||
mamba_type=self.mamba_type,
|
||||
tp_replicated=self.is_kv_cache_tp_replicated,
|
||||
mamba_cache_mode=vllm_config.cache_config.mamba_cache_mode,
|
||||
# RecoverSSM verifies the whole window off one checkpoint, so it
|
||||
# never writes the baseline's per-draft-token state slots.
|
||||
|
||||
@@ -519,7 +519,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
self.gqa_interleaved_layout
|
||||
or self.head_k_dim != 128
|
||||
or self.head_v_dim != 128
|
||||
or self.norm.activation != "silu"
|
||||
or self.norm.activation not in ("silu", "sigmoid")
|
||||
or vllm_config.model_config.dtype != torch.bfloat16
|
||||
or conv_state_dtype != torch.bfloat16
|
||||
or recurrent_state_dtype not in FUSED_GDN_STATE_DTYPES
|
||||
@@ -527,9 +527,9 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
):
|
||||
return (
|
||||
"the fused CUDA kernel requires a BF16 GDN model with "
|
||||
"K=V=128, SiLU gating, non-interleaved GQA layout, BF16 "
|
||||
"convolution cache, BF16 or FP32 recurrent state, and a "
|
||||
"GPU with compute capability 8.0+"
|
||||
"K=V=128, SiLU or sigmoid gating, non-interleaved GQA "
|
||||
"layout, BF16 convolution cache, BF16 or FP32 recurrent "
|
||||
"state, and a GPU with compute capability 8.0+"
|
||||
)
|
||||
if not hasattr(torch.ops._C, "fused_gdn_decode_post_conv_mtp"):
|
||||
return "torch.ops._C.fused_gdn_decode_post_conv_mtp is not built"
|
||||
@@ -1766,6 +1766,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
out=core_attn_out,
|
||||
scale=self.head_k_dim**-0.5,
|
||||
norm_eps=self.layer_norm_epsilon,
|
||||
output_gate_activation=self.norm.activation,
|
||||
)
|
||||
|
||||
def _forward_core_fused_norm_packed(
|
||||
|
||||
@@ -17,6 +17,7 @@ from vllm.utils.torch_utils import (
|
||||
STR_DTYPE_TO_TORCH_DTYPE,
|
||||
get_kv_cache_torch_dtype,
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -352,6 +353,10 @@ Parameters:
|
||||
num_accepted_tokens: int - number of accepted tokens used to compute the copy offset.
|
||||
Range: 1 .. 1 + num_speculative_tokens (inclusive).
|
||||
"""
|
||||
MambaStateCopyFuncs: TypeAlias = tuple[MambaStateCopyFunc, ...]
|
||||
MambaStateCopyFuncsByType: TypeAlias = dict[
|
||||
MambaAttentionBackendEnum, MambaStateCopyFuncs
|
||||
]
|
||||
|
||||
|
||||
def get_conv_copy_spec(
|
||||
|
||||
Reference in New Issue
Block a user