[Model] Support Qwen3.8-Flash-Next (#53896)

Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com>
Signed-off-by: andyluo7 <andy.luo@amd.com>
Signed-off-by: lishunyang12 <lishunyang03@gmail.com>
Signed-off-by: Harry Huang <huanghaoyan.hhy@alibaba-inc.com>
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: KungYork <firelikest@163.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: andyluo7 <43718156+andyluo7@users.noreply.github.com>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: andyluo7 <andy.luo@amd.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: lishunyang12 <lishunyang03@gmail.com>
Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Harry Huang
2026-08-31 05:57:56 +00:00
committed by GitHub
co-authored by zjy0516 KungYork Thien Tran Yifan Qiao andyluo7 Isotr0py andyluo7 OpenAI Codex lishunyang12 Vadim Gimpelson Isotr0py Lucas Wilkinson Claude Opus 5 OpenAI Codex
parent 687db59744
commit e126687a9a
124 changed files with 19424 additions and 450 deletions
@@ -56,6 +56,10 @@ class MambaBase(AttentionLayerBase):
def mamba_type(self) -> MambaAttentionBackendEnum:
pass
@property
def is_kv_cache_tp_replicated(self) -> bool:
return False
@abstractmethod
def get_state_dtype(self) -> tuple[torch.dtype, ...]:
pass
@@ -70,6 +74,7 @@ class MambaBase(AttentionLayerBase):
block_size=mamba_block_size,
page_size_padded=page_size_padded,
mamba_type=self.mamba_type,
tp_replicated=self.is_kv_cache_tp_replicated,
mamba_cache_mode=vllm_config.cache_config.mamba_cache_mode,
# RecoverSSM verifies the whole window off one checkpoint, so it
# never writes the baseline's per-draft-token state slots.
@@ -519,7 +519,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
self.gqa_interleaved_layout
or self.head_k_dim != 128
or self.head_v_dim != 128
or self.norm.activation != "silu"
or self.norm.activation not in ("silu", "sigmoid")
or vllm_config.model_config.dtype != torch.bfloat16
or conv_state_dtype != torch.bfloat16
or recurrent_state_dtype not in FUSED_GDN_STATE_DTYPES
@@ -527,9 +527,9 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
):
return (
"the fused CUDA kernel requires a BF16 GDN model with "
"K=V=128, SiLU gating, non-interleaved GQA layout, BF16 "
"convolution cache, BF16 or FP32 recurrent state, and a "
"GPU with compute capability 8.0+"
"K=V=128, SiLU or sigmoid gating, non-interleaved GQA "
"layout, BF16 convolution cache, BF16 or FP32 recurrent "
"state, and a GPU with compute capability 8.0+"
)
if not hasattr(torch.ops._C, "fused_gdn_decode_post_conv_mtp"):
return "torch.ops._C.fused_gdn_decode_post_conv_mtp is not built"
@@ -1766,6 +1766,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
out=core_attn_out,
scale=self.head_k_dim**-0.5,
norm_eps=self.layer_norm_epsilon,
output_gate_activation=self.norm.activation,
)
def _forward_core_fused_norm_packed(
@@ -17,6 +17,7 @@ from vllm.utils.torch_utils import (
STR_DTYPE_TO_TORCH_DTYPE,
get_kv_cache_torch_dtype,
)
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
logger = init_logger(__name__)
@@ -352,6 +353,10 @@ Parameters:
num_accepted_tokens: int - number of accepted tokens used to compute the copy offset.
Range: 1 .. 1 + num_speculative_tokens (inclusive).
"""
MambaStateCopyFuncs: TypeAlias = tuple[MambaStateCopyFunc, ...]
MambaStateCopyFuncsByType: TypeAlias = dict[
MambaAttentionBackendEnum, MambaStateCopyFuncs
]
def get_conv_copy_spec(