[Model] Support Qwen3.8-Flash-Next (#53896)
Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com> Signed-off-by: andyluo7 <andy.luo@amd.com> Signed-off-by: lishunyang12 <lishunyang03@gmail.com> Signed-off-by: Harry Huang <huanghaoyan.hhy@alibaba-inc.com> Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: KungYork <firelikest@163.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: andyluo7 <43718156+andyluo7@users.noreply.github.com> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: andyluo7 <andy.luo@amd.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: lishunyang12 <lishunyang03@gmail.com> Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
zjy0516
KungYork
Thien Tran
Yifan Qiao
andyluo7
Isotr0py
andyluo7
OpenAI Codex
lishunyang12
Vadim Gimpelson
Isotr0py
Lucas Wilkinson
Claude Opus 5
OpenAI Codex
parent
687db59744
commit
e126687a9a
@@ -839,6 +839,83 @@ class Qwen3_5ForCausalLMConfig(Qwen3_5ForConditionalGenerationConfig):
|
||||
rope_parameters.pop("mrope_interleaved", None)
|
||||
|
||||
|
||||
def _strip_qwen4_exp_mrope(model_config: "ModelConfig") -> None:
|
||||
configs = {
|
||||
id(config): config
|
||||
for config in (
|
||||
getattr(model_config, "hf_config", None),
|
||||
model_config.hf_text_config,
|
||||
)
|
||||
if config is not None
|
||||
}
|
||||
for config in configs.values():
|
||||
rope_parameters = getattr(config, "rope_parameters", None)
|
||||
if rope_parameters is not None:
|
||||
rope_parameters.pop("mrope_section", None)
|
||||
rope_parameters.pop("mrope_interleaved", None)
|
||||
|
||||
|
||||
class Qwen4ExpForConditionalGenerationConfig(Qwen3_5ForConditionalGenerationConfig):
|
||||
"""Apply the Qwen3.5 hybrid-cache contract to Qwen4Exp."""
|
||||
|
||||
@staticmethod
|
||||
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
|
||||
Qwen3_5ForConditionalGenerationConfig.verify_and_update_config(vllm_config)
|
||||
text_config = vllm_config.model_config.hf_text_config
|
||||
if text_config.hc_count <= 1:
|
||||
raise ValueError("Qwen4Exp requires hc_count > 1")
|
||||
parallel_config = vllm_config.parallel_config
|
||||
uses_ple_or_qsa = bool(text_config.ple_layer_ids) or (
|
||||
getattr(text_config, "indexer_n_heads", None) is not None
|
||||
)
|
||||
if uses_ple_or_qsa and (
|
||||
parallel_config.enable_dbo or parallel_config.ubatch_size > 1
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp PLE/QSA does not support dual-batch overlap or microbatching"
|
||||
)
|
||||
# Checked again in Qwen4ExpModelState; rejecting it here keeps the
|
||||
# engine from loading weights first.
|
||||
if text_config.ple_layer_ids and parallel_config.pipeline_parallel_size > 1:
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp N-gram PLE embedding requires pipeline_parallel_size=1 "
|
||||
"because non-first pipeline ranks do not receive the raw input_ids "
|
||||
"it needs. Please run with PP=1."
|
||||
)
|
||||
multimodal_config = vllm_config.model_config.multimodal_config
|
||||
if multimodal_config is not None and multimodal_config.language_model_only:
|
||||
_strip_qwen4_exp_mrope(vllm_config.model_config)
|
||||
spec_config = vllm_config.speculative_config
|
||||
if spec_config is not None and spec_config.method not in {
|
||||
"mtp",
|
||||
"ngram",
|
||||
"ngram_gpu",
|
||||
}:
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp speculative decoding supports only its native MTP "
|
||||
"checkpoint and linear n-gram proposers"
|
||||
)
|
||||
|
||||
|
||||
class Qwen4ExpForCausalLMConfig(Qwen4ExpForConditionalGenerationConfig):
|
||||
@staticmethod
|
||||
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
|
||||
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(vllm_config)
|
||||
|
||||
_strip_qwen4_exp_mrope(vllm_config.model_config)
|
||||
|
||||
|
||||
class Qwen4ExpMTPConfig(Qwen4ExpForConditionalGenerationConfig):
|
||||
"""Preserve MRoPE for a VL target and use 1D RoPE for a text target."""
|
||||
|
||||
@staticmethod
|
||||
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
|
||||
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(vllm_config)
|
||||
if hasattr(vllm_config.model_config.hf_config, "vision_config"):
|
||||
return
|
||||
_strip_qwen4_exp_mrope(vllm_config.model_config)
|
||||
|
||||
|
||||
class ColQwen3_5Config(Qwen3_5ForConditionalGenerationConfig):
|
||||
"""Apply the attention contract declared by a ColQwen3.5 checkpoint."""
|
||||
|
||||
@@ -963,6 +1040,9 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
|
||||
"Qwen3_5ForConditionalGeneration": Qwen3_5ForConditionalGenerationConfig,
|
||||
"Qwen3_5MoeForCausalLM": Qwen3_5ForCausalLMConfig,
|
||||
"Qwen3_5MoeForConditionalGeneration": Qwen3_5ForConditionalGenerationConfig,
|
||||
"Qwen4ExpForCausalLM": Qwen4ExpForCausalLMConfig,
|
||||
"Qwen4ExpForConditionalGeneration": (Qwen4ExpForConditionalGenerationConfig),
|
||||
"Qwen4ExpMTP": Qwen4ExpMTPConfig,
|
||||
"UnlimitedOCRForCausalLM": UnlimitedOCRForCausalLMConfig,
|
||||
"VoyageQwen3BidirectionalEmbedModel": VoyageQwen3BidirectionalEmbedModelConfig,
|
||||
"XLMRobertaModel": JinaRobertaModelConfig,
|
||||
|
||||
@@ -49,13 +49,17 @@ if TYPE_CHECKING:
|
||||
from vllm.lora.model_manager import LoRAModelManager
|
||||
from vllm.model_executor.layers.fused_moe import MoERunner
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateCopyFunc,
|
||||
MambaStateCopyFuncsByType,
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalKwargsItem
|
||||
from vllm.multimodal.registry import _ProcessorFactories
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tasks import ScoreType
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import (
|
||||
EncoderCudaGraphCaptureInputs,
|
||||
EncoderCudaGraphConfig,
|
||||
@@ -970,6 +974,14 @@ class IsHybrid(Protocol):
|
||||
"""
|
||||
...
|
||||
|
||||
@classmethod
|
||||
def get_mamba_state_copy_funcs(
|
||||
cls,
|
||||
mamba_types: set["MambaAttentionBackendEnum"],
|
||||
) -> "MambaStateCopyFuncsByType":
|
||||
copy_funcs = cls.get_mamba_state_copy_func()
|
||||
return {mamba_type: copy_funcs for mamba_type in mamba_types}
|
||||
|
||||
|
||||
@overload
|
||||
def is_hybrid(model: type[object]) -> TypeIs[type[IsHybrid]]: ...
|
||||
@@ -1131,6 +1143,20 @@ class SupportsMambaPrefixCaching(Protocol):
|
||||
|
||||
supports_mamba_prefix_caching: ClassVar[Literal[True]] = True
|
||||
|
||||
@classmethod
|
||||
def get_mamba_state_copy_func(cls) -> tuple["MambaStateCopyFunc", ...]:
|
||||
"""Return copy functions for the model's Mamba states."""
|
||||
...
|
||||
|
||||
@classmethod
|
||||
def get_mamba_state_copy_funcs(
|
||||
cls,
|
||||
mamba_types: set["MambaAttentionBackendEnum"],
|
||||
) -> "MambaStateCopyFuncsByType":
|
||||
"""Map legacy copy functions to each requested Mamba backend."""
|
||||
copy_funcs = cls.get_mamba_state_copy_func()
|
||||
return {mamba_type: copy_funcs for mamba_type in mamba_types}
|
||||
|
||||
|
||||
@overload
|
||||
def supports_mamba_prefix_caching(
|
||||
|
||||
@@ -80,15 +80,17 @@ class Qwen2MoeMLP(nn.Module):
|
||||
reduce_results: bool = True,
|
||||
expert_gate: torch.nn.Linear | None = None,
|
||||
is_sequence_parallel: bool = False,
|
||||
disable_tp: bool = False,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
disable_tp = disable_tp or is_sequence_parallel
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
hidden_size,
|
||||
[intermediate_size] * 2,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
disable_tp=is_sequence_parallel,
|
||||
disable_tp=disable_tp,
|
||||
prefix=f"{prefix}.gate_up_proj",
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
@@ -97,7 +99,7 @@ class Qwen2MoeMLP(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
reduce_results=reduce_results,
|
||||
disable_tp=is_sequence_parallel,
|
||||
disable_tp=disable_tp,
|
||||
prefix=f"{prefix}.down_proj",
|
||||
)
|
||||
if hidden_act != "silu":
|
||||
|
||||
@@ -43,6 +43,9 @@ from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateShapeCalculator,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.utils.config_utils import (
|
||||
get_quark_ocp_mx_group_size,
|
||||
)
|
||||
from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding, get_rope
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
@@ -90,6 +93,38 @@ def _should_use_sequence_parallel(vllm_config: VllmConfig) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def _should_replicate_misaligned_shared_expert(
|
||||
intermediate_size: int,
|
||||
tp_size: int,
|
||||
group_size: int | None,
|
||||
enable_expert_parallel: bool,
|
||||
is_sequence_parallel: bool,
|
||||
) -> bool:
|
||||
if intermediate_size <= 0 or group_size is None:
|
||||
return False
|
||||
|
||||
partition_size, remainder = divmod(intermediate_size, tp_size)
|
||||
if remainder == 0 and partition_size % group_size == 0:
|
||||
return False
|
||||
|
||||
if enable_expert_parallel or is_sequence_parallel:
|
||||
return True
|
||||
|
||||
if remainder != 0:
|
||||
raise ValueError(
|
||||
f"Shared-expert intermediate size {intermediate_size} must be "
|
||||
f"divisible by tensor-parallel size {tp_size}."
|
||||
)
|
||||
|
||||
raise ValueError(
|
||||
"The Quark OCP MX shared expert cannot be tensor-parallelized: "
|
||||
f"intermediate size {intermediate_size} with TP size {tp_size} "
|
||||
f"produces a partition of {partition_size}, which is not divisible by "
|
||||
f"the OCP MX group size {group_size}. Choose a compatible "
|
||||
"tensor-parallel size or enable expert parallelism."
|
||||
)
|
||||
|
||||
|
||||
class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
def __init__(self, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
@@ -106,6 +141,17 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
|
||||
self.is_sequence_parallel = parallel_config.use_sequence_parallel_moe
|
||||
|
||||
shared_expert_group_size = get_quark_ocp_mx_group_size(
|
||||
quant_config,
|
||||
f"{prefix}.shared_expert.down_proj",
|
||||
)
|
||||
self.replicate_shared_expert = _should_replicate_misaligned_shared_expert(
|
||||
config.shared_expert_intermediate_size,
|
||||
self.tp_size,
|
||||
shared_expert_group_size,
|
||||
parallel_config.enable_expert_parallel,
|
||||
self.is_sequence_parallel,
|
||||
)
|
||||
if self.tp_size > config.num_experts:
|
||||
raise ValueError(
|
||||
f"Tensor parallel size {self.tp_size} is greater than "
|
||||
@@ -138,7 +184,10 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
)
|
||||
|
||||
self.is_fused_shared_expert_enabled = False
|
||||
if config.shared_expert_intermediate_size > 0:
|
||||
if (
|
||||
config.shared_expert_intermediate_size > 0
|
||||
and not self.replicate_shared_expert
|
||||
):
|
||||
self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert(
|
||||
quant_config,
|
||||
prefix,
|
||||
@@ -159,11 +208,14 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
reduce_results=False,
|
||||
expert_gate=self.shared_expert_gate,
|
||||
is_sequence_parallel=self.is_sequence_parallel,
|
||||
disable_tp=self.replicate_shared_expert,
|
||||
prefix=f"{prefix}.shared_expert",
|
||||
)
|
||||
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_expert,
|
||||
shared_experts=(
|
||||
None if self.replicate_shared_expert else self.shared_expert
|
||||
),
|
||||
gate=self.gate,
|
||||
num_experts=self.n_routed_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
@@ -195,9 +247,16 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
if self.is_sequence_parallel and not already_sequence_parallel:
|
||||
hidden_states = sequence_parallel_chunk(hidden_states)
|
||||
|
||||
replicated_shared_output = (
|
||||
self.shared_expert(hidden_states)
|
||||
if self.replicate_shared_expert and self.shared_expert is not None
|
||||
else None
|
||||
)
|
||||
final_hidden_states = self.experts(
|
||||
hidden_states=hidden_states, router_logits=hidden_states
|
||||
)
|
||||
if replicated_shared_output is not None:
|
||||
final_hidden_states += replicated_shared_output
|
||||
|
||||
if self.is_sequence_parallel and not already_sequence_parallel:
|
||||
final_hidden_states = tensor_model_parallel_all_gather(
|
||||
@@ -488,7 +547,7 @@ class Qwen3NextDecoderLayer(nn.Module):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None,
|
||||
positions: torch.Tensor = None,
|
||||
positions: torch.Tensor,
|
||||
**kwargs: object,
|
||||
):
|
||||
full_num_tokens = positions.shape[-1]
|
||||
|
||||
@@ -111,6 +111,10 @@ _TEXT_GENERATION_MODELS = {
|
||||
"Gemma3nForCausalLM": ("gemma3n", "Gemma3nForCausalLM"),
|
||||
"Gemma4ForCausalLM": ("gemma4", "Gemma4ForCausalLM"),
|
||||
"Qwen3NextForCausalLM": ("qwen3_next", "Qwen3NextForCausalLM"),
|
||||
"Qwen4ExpForCausalLM": (
|
||||
"vllm.models.qwen4_exp",
|
||||
"Qwen4ExpForCausalLM",
|
||||
),
|
||||
"GlmForCausalLM": ("glm", "GlmForCausalLM"),
|
||||
"Glm4ForCausalLM": ("glm4", "Glm4ForCausalLM"),
|
||||
"Glm4MoeForCausalLM": ("glm4_moe", "Glm4MoeForCausalLM"),
|
||||
@@ -573,6 +577,10 @@ _MULTIMODAL_MODELS = {
|
||||
"qwen3_5",
|
||||
"Qwen3_5MoeForConditionalGeneration",
|
||||
),
|
||||
"Qwen4ExpForConditionalGeneration": (
|
||||
"vllm.models.qwen4_exp",
|
||||
"Qwen4ExpForConditionalGeneration",
|
||||
),
|
||||
"RForConditionalGeneration": ("rvl", "RForConditionalGeneration"),
|
||||
"SkyworkR1VChatModel": ("skyworkr1v", "SkyworkR1VChatModel"),
|
||||
"SmolVLMForConditionalGeneration": ("smolvlm", "SmolVLMForConditionalGeneration"),
|
||||
@@ -659,6 +667,7 @@ _SPECULATIVE_DECODING_MODELS = {
|
||||
"MedusaModel": ("medusa", "Medusa"),
|
||||
"OpenPanguMTPModel": ("openpangu_mtp", "OpenPanguMTP"),
|
||||
"Qwen3NextMTP": ("qwen3_next_mtp", "Qwen3NextMTP"),
|
||||
"Qwen4ExpMTP": ("vllm.models.qwen4_exp", "Qwen4ExpMTP"),
|
||||
"Step3p5MTP": ("step3p5_mtp", "Step3p5MTP"),
|
||||
"Qwen3_5MTP": ("qwen3_5_mtp", "Qwen3_5MTP"),
|
||||
"Qwen3_5MoeMTP": ("qwen3_5_mtp", "Qwen3_5MoeMTP"),
|
||||
|
||||
Reference in New Issue
Block a user