[Model] Support Qwen3.8-Flash-Next (#53896)

Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com>
Signed-off-by: andyluo7 <andy.luo@amd.com>
Signed-off-by: lishunyang12 <lishunyang03@gmail.com>
Signed-off-by: Harry Huang <huanghaoyan.hhy@alibaba-inc.com>
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: KungYork <firelikest@163.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: andyluo7 <43718156+andyluo7@users.noreply.github.com>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: andyluo7 <andy.luo@amd.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: lishunyang12 <lishunyang03@gmail.com>
Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Harry Huang
2026-08-31 05:57:56 +00:00
committed by GitHub
co-authored by zjy0516 KungYork Thien Tran Yifan Qiao andyluo7 Isotr0py andyluo7 OpenAI Codex lishunyang12 Vadim Gimpelson Isotr0py Lucas Wilkinson Claude Opus 5 OpenAI Codex
parent 687db59744
commit e126687a9a
124 changed files with 19424 additions and 450 deletions
@@ -39,6 +39,7 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
is_deep_gemm_e8m0_used,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
FP8_DTYPE,
QuantKey,
kFp8Dynamic128Sym,
kFp8DynamicTensorSym,
@@ -153,6 +154,19 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
(kFp8StaticTensorSym, kFp8StaticTensorSym),
(kFp8StaticTensorSym, kFp8DynamicTensorSym),
]
# Block-quantized FP8 with arbitrary block shape: the Triton
# kernels take the block shape as a runtime argument.
if (
weight_key is not None
and activation_key == kFp8Dynamic128Sym
and weight_key.dtype == FP8_DTYPE
and weight_key.symmetric
and weight_key.scale.static
and weight_key.scale.dtype == torch.float32
and weight_key.scale.group_shape.row > 1
and weight_key.scale.group_shape.col > 1
):
return True
return (weight_key, activation_key) in supported
@staticmethod
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from enum import Enum
from typing import Any
@@ -268,6 +269,39 @@ def map_fp8_backend(runner_backend: MoEBackend) -> Fp8MoeBackend:
)
def refine_fp8_moe_block_shape(
config: FusedMoEConfig,
weight_block_size: list[int],
) -> list[int] | None:
"""
Compute a refined block shape for block-quantized FP8 MoE weights whose
checkpoint blocks cannot be sharded exactly across TP ranks.
TP shards the intermediate dim of the expert weights, so a per-shard size
that is not a multiple of the checkpoint's block size makes the
checkpoint's block scales impossible to shard exactly. When a finer block
size (>= 32) divides both the checkpoint blocks and all involved dims,
the weight scales can be refined to that granularity at load time (a
lossless upsampling, since the refined block divides the checkpoint
block). Only Triton-based kernels can consume the refined block shape:
they take it as a runtime argument, while the other backends require the
native 128x128 blocks. The refined shape is encoded in the QuantKey used
for backend selection, so backends that only support 128x128 blocks are
rejected by the oracle automatically.
Returns the refined [block_n, block_k] shape, or None if no refinement
is needed or possible.
"""
block_n, block_k = weight_block_size
ispp = config.intermediate_size_per_partition
if ispp % block_n == 0 and (config.tp_size == 1 or ispp % block_k == 0):
return None
refine = math.gcd(block_n, block_k, ispp, config.hidden_dim)
if refine < 32:
return None
return [refine, refine]
def select_fp8_moe_backend(
config: FusedMoEConfig,
weight_key: QuantKey | None,
@@ -474,6 +474,39 @@ def _get_requested_backends(
return _filter_by_activation(backends, requested_activation_key)
def _requires_qwen38_tep8_emulation(
config: FusedMoEConfig,
activation_key: QuantKey | None,
) -> bool:
"""Avoid the inaccurate native gfx950 kernel for Qwen3.8 TEP8.
Qwen3.8 Flash Next's routed experts use the distinctive
``E=512, H=2560, N=640`` W4A4 shape. With eight-way expert parallelism,
AITER operates on 64 local experts and pads ``N`` to 768. That native path
is not numerically reliable on gfx950, while OCP MX emulation preserves
model accuracy. Keep this guard exact so other MXFP4 shapes and smaller EP
configurations retain the native backend.
"""
parallel = config.moe_parallel_config
if not (
activation_key == kMxfp4Dynamic
and parallel.use_ep
and parallel.ep_size == 8
and config.num_experts == 512
and config.num_local_experts == 64
and config.hidden_dim == 2560
and config.intermediate_size == 640
):
return False
if not current_platform.is_rocm():
return False
from vllm.platforms.rocm import on_gfx950
return on_gfx950()
def select_mxfp4_moe_backend(
config: FusedMoEConfig,
activation_key: QuantKey | None = None,
@@ -532,6 +565,21 @@ def select_mxfp4_moe_backend(
assert last_error is not None
raise last_error
if _requires_qwen38_tep8_emulation(config, requested_activation_key):
backend = Mxfp4MoeBackend.EMULATION
logger.warning_once(
"Using OCP MX emulation for the Qwen3.8 Flash Next TEP8 routed "
"experts on gfx950 because the native AITER W4A4 kernel is not "
"numerically reliable for this shape. Performance will be lower."
)
return _return_or_raise(
backend,
config,
kMxfp4Static,
requested_activation_key,
activation_format,
)
# Select kernels in order of backend.
AVAILABLE_BACKENDS = _filter_by_activation(
_get_priority_backends_for_gpt_oss(), requested_activation_key
@@ -345,14 +345,13 @@ def convert_to_unquantized_kernel_format(
elif unquantized_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM:
is_act_and_mul = moe_config.is_act_and_mul
if not is_act_and_mul:
# Kernel requires intermediate_size_per_partition % 128 == 0 (BlockMajorK
# weight layout uses block_k=128). Pad along the intermediate dim when
# the model + TP split don't satisfy the constraint.
w13_weight, w2_weight, padded_intermediate = align_moe_weights_for_fi(
w13_weight, w2_weight, is_act_and_mul, min_alignment=128
)
moe_config.intermediate_size_per_partition = padded_intermediate
# Kernel requires intermediate_size_per_partition % 128 == 0 (BlockMajorK
# weight layout uses block_k=128). Pad along the intermediate dim when
# the model + TP split don't satisfy the constraint.
w13_weight, w2_weight, padded_intermediate = align_moe_weights_for_fi(
w13_weight, w2_weight, is_act_and_mul, min_alignment=128
)
moe_config.intermediate_size_per_partition = padded_intermediate
_cache_permute_indices: dict[torch.Size, torch.Tensor] = {}
w13_weight, w2_weight = convert_moe_weights_to_flashinfer_trtllm_block_layout(
@@ -832,6 +832,20 @@ class RoutedExperts(PluggableLayer):
FusedMoeWeightScaleSupported.GROUP.value,
FusedMoeWeightScaleSupported.BLOCK.value,
]:
scale_refine = getattr(self.quant_method, "weight_scale_refine", None)
if (
quant_method == FusedMoeWeightScaleSupported.BLOCK.value
and scale_refine is not None
):
# FP8 block scales are stored per (block_n, block_k) tile
# of the unsharded weight, while the TP-sharded parameters
# use a refined block grid (see Fp8MoEMethod). Upsample the
# scales to the refined grid (lossless: the refined block
# divides the checkpoint block) so per-rank slicing stays
# exact. Dim -2 is the weight's N dim, dim -1 is K.
loaded_weight = loaded_weight.repeat_interleave(
scale_refine[0], dim=-2
).repeat_interleave(scale_refine[1], dim=-1)
self._load_model_weight_or_group_weight_scale(
shard_id=shard_id,
shard_dim=shard_dim,
@@ -56,6 +56,10 @@ class MambaBase(AttentionLayerBase):
def mamba_type(self) -> MambaAttentionBackendEnum:
pass
@property
def is_kv_cache_tp_replicated(self) -> bool:
return False
@abstractmethod
def get_state_dtype(self) -> tuple[torch.dtype, ...]:
pass
@@ -70,6 +74,7 @@ class MambaBase(AttentionLayerBase):
block_size=mamba_block_size,
page_size_padded=page_size_padded,
mamba_type=self.mamba_type,
tp_replicated=self.is_kv_cache_tp_replicated,
mamba_cache_mode=vllm_config.cache_config.mamba_cache_mode,
# RecoverSSM verifies the whole window off one checkpoint, so it
# never writes the baseline's per-draft-token state slots.
@@ -519,7 +519,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
self.gqa_interleaved_layout
or self.head_k_dim != 128
or self.head_v_dim != 128
or self.norm.activation != "silu"
or self.norm.activation not in ("silu", "sigmoid")
or vllm_config.model_config.dtype != torch.bfloat16
or conv_state_dtype != torch.bfloat16
or recurrent_state_dtype not in FUSED_GDN_STATE_DTYPES
@@ -527,9 +527,9 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
):
return (
"the fused CUDA kernel requires a BF16 GDN model with "
"K=V=128, SiLU gating, non-interleaved GQA layout, BF16 "
"convolution cache, BF16 or FP32 recurrent state, and a "
"GPU with compute capability 8.0+"
"K=V=128, SiLU or sigmoid gating, non-interleaved GQA "
"layout, BF16 convolution cache, BF16 or FP32 recurrent "
"state, and a GPU with compute capability 8.0+"
)
if not hasattr(torch.ops._C, "fused_gdn_decode_post_conv_mtp"):
return "torch.ops._C.fused_gdn_decode_post_conv_mtp is not built"
@@ -1766,6 +1766,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
out=core_attn_out,
scale=self.head_k_dim**-0.5,
norm_eps=self.layer_norm_epsilon,
output_gate_activation=self.norm.activation,
)
def _forward_core_fused_norm_packed(
@@ -17,6 +17,7 @@ from vllm.utils.torch_utils import (
STR_DTYPE_TO_TORCH_DTYPE,
get_kv_cache_torch_dtype,
)
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
logger = init_logger(__name__)
@@ -352,6 +353,10 @@ Parameters:
num_accepted_tokens: int - number of accepted tokens used to compute the copy offset.
Range: 1 .. 1 + num_speculative_tokens (inclusive).
"""
MambaStateCopyFuncs: TypeAlias = tuple[MambaStateCopyFunc, ...]
MambaStateCopyFuncsByType: TypeAlias = dict[
MambaAttentionBackendEnum, MambaStateCopyFuncs
]
def get_conv_copy_spec(
+53 -13
View File
@@ -31,6 +31,7 @@ from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
convert_to_fp8_moe_kernel_format,
make_fp8_moe_kernel,
make_fp8_moe_quant_config,
refine_fp8_moe_block_shape,
select_fp8_moe_backend,
)
from vllm.model_executor.layers.linear import (
@@ -63,7 +64,6 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Dynamic128Sym,
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
kFp8Static128BlockSym,
kFp8StaticTensorSym,
)
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
@@ -481,9 +481,41 @@ class Fp8MoEMethod(FusedMoEMethodBase):
"weight_scale_inv" if self.block_quant else "weight_scale"
)
self.weight_scale_refine: tuple[int, int] | None = None
self.moe_block_shape = self.weight_block_size
# Set weight key and activation key for kernel compatibility
if self.block_quant:
weight_key = kFp8Static128BlockSym
assert self.weight_block_size is not None
# TP shards the intermediate dim of the expert weights, so a
# per-shard size that is not a multiple of the checkpoint's block
# size makes the checkpoint's block scales impossible to shard
# exactly. When a finer block size (>= 32) divides both the
# checkpoint blocks and all involved dims, the weight scales are
# refined to that granularity at load time (a lossless upsampling,
# since the refined block divides the checkpoint block). The
# refined block shape is encoded in the weight key, so the oracle
# only selects kernels that support it (e.g. Triton, which takes
# the block shape as a runtime argument).
refined_shape = refine_fp8_moe_block_shape(self.moe, self.weight_block_size)
if refined_shape is not None:
block_n, block_k = self.weight_block_size
self.weight_scale_refine = (
block_n // refined_shape[0],
block_k // refined_shape[1],
)
self.moe_block_shape = refined_shape
logger.info_once(
"FP8 MoE block scales refined from %s to %s to fit "
"the TP-sharded intermediate size %d.",
str(self.weight_block_size),
str(refined_shape),
self.moe.intermediate_size_per_partition,
)
assert self.moe_block_shape is not None
weight_key = create_fp8_quant_key(
static=True, group_shape=GroupShape(*self.moe_block_shape)
)
activation_key = kFp8Dynamic128Sym
else:
weight_key = kFp8StaticTensorSym
@@ -519,6 +551,8 @@ class Fp8MoEMethod(FusedMoEMethodBase):
if self.block_quant:
assert self.weight_block_size is not None
assert self.moe_block_shape is not None
moe_block_shape = self.moe_block_shape
layer.weight_block_size = self.weight_block_size
tp_size = get_tensor_model_parallel_world_size()
block_n, block_k = (
@@ -530,18 +564,24 @@ class Fp8MoEMethod(FusedMoEMethodBase):
# layers must be divisible by block_n.
# Required by column parallel or enabling merged weights
if intermediate_size_per_partition % block_n != 0:
raise ValueError(
f"The output_size of gate's and up's weight = "
f"{intermediate_size_per_partition} is not divisible by "
f"weight quantization block_n = {block_n}."
)
if self.weight_scale_refine is None:
raise ValueError(
f"The output_size of gate's and up's weight = "
f"{intermediate_size_per_partition} is not divisible by "
f"weight quantization block_n = {block_n}."
)
# Use the refined block grid for the scale parameters; the
# loader upsamples the checkpoint scales accordingly.
block_n, block_k = moe_block_shape
if tp_size > 1 and intermediate_size_per_partition % block_k != 0:
# Required by row parallel
raise ValueError(
f"The input_size of down's weight = "
f"{intermediate_size_per_partition} is not divisible by "
f"weight quantization block_k = {block_k}."
)
if self.weight_scale_refine is None:
raise ValueError(
f"The input_size of down's weight = "
f"{intermediate_size_per_partition} is not divisible by "
f"weight quantization block_k = {block_k}."
)
block_n, block_k = moe_block_shape
# WEIGHTS
w13_weight = torch.nn.Parameter(
@@ -747,7 +787,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
w2_scale=w2_scale,
a1_scale=a1_scale,
a2_scale=a2_scale,
block_shape=self.weight_block_size,
block_shape=self.moe_block_shape,
swiglu_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
@@ -161,6 +161,16 @@ class QuarkOCP_MX(QuarkScheme):
weight_loader: Callable,
**kwargs,
):
if input_size_per_partition % OCP_MX_BLOCK_SIZE != 0:
layer_name = getattr(layer, "prefix", "") or type(layer).__name__
raise ValueError(
f"OCP MX linear layer {layer_name!r} has an input size per "
f"partition of {input_size_per_partition}, which must be "
f"divisible by the OCP MX group size {OCP_MX_BLOCK_SIZE}. "
"Choose a compatible tensor-parallel size or avoid "
"tensor-parallel sharding for this layer."
)
if self.dynamic_mxfp4_quant:
weight = ModelWeightParameter(
data=torch.empty(
@@ -9,6 +9,41 @@ if TYPE_CHECKING:
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
def get_quark_ocp_mx_group_size(
quant_config: "QuantizationConfig | None",
layer_name: str,
) -> int | None:
"""Return the OCP MX group size for a quantized Quark linear layer."""
if quant_config is None:
return None
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
from vllm.model_executor.layers.quantization.quark.utils import should_ignore_layer
if not isinstance(quant_config, QuarkConfig):
return None
if should_ignore_layer(
layer_name,
ignore=quant_config.quant_config.get("exclude") or [],
fused_mapping=quant_config.packed_modules_mapping,
):
return None
layer_quant_config = (
quant_config.get_layer_quant_config_from_name(layer_name)
or quant_config.quant_config.get("global_quant_config")
or {}
)
weight_quant = layer_quant_config.get("weight")
input_quant = layer_quant_config.get("input_tensors")
if not quant_config._is_w_ocp_mx_a_x(weight_quant, input_quant):
return None
assert weight_quant is not None
return int(weight_quant["group_size"])
def is_shared_expert_quant_fse_compatible(
quant_config: "QuantizationConfig | None",
expert_prefix: str,
@@ -232,7 +232,16 @@ def align_fp4_moe_weights_for_fi(
# Pad w13 and w2 along its intermediate dimension.
padded_w13 = w13.new_zeros((num_experts, padded_gate_up_dim, hidden_size // 2))
padded_w13[:, : w13.shape[1], :] = w13
if is_act_and_mul:
# Keep the two logical projections independently aligned. Copying the
# fused [gate, up] tensor contiguously would move the up projection into
# the padded tail of gate when intermediate is rounded up.
padded_w13[:, :intermediate, :] = w13[:, :intermediate, :]
padded_w13[:, padded_intermediate : padded_intermediate + intermediate, :] = (
w13[:, intermediate:, :]
)
else:
padded_w13[:, : w13.shape[1], :] = w13
padded_w2 = w2.new_zeros((num_experts, hidden_size, padded_intermediate // 2))
padded_w2[:, :, : w2.shape[2]] = w2
@@ -240,7 +249,13 @@ def align_fp4_moe_weights_for_fi(
padded_w13_scale = w13_scale.new_zeros(
(num_experts, padded_gate_up_dim, hidden_size // 16)
)
padded_w13_scale[:, : w13_scale.shape[1], :] = w13_scale
if is_act_and_mul:
padded_w13_scale[:, :intermediate, :] = w13_scale[:, :intermediate, :]
padded_w13_scale[
:, padded_intermediate : padded_intermediate + intermediate, :
] = w13_scale[:, intermediate:, :]
else:
padded_w13_scale[:, : w13_scale.shape[1], :] = w13_scale
padded_w2_scale = w2_scale.new_zeros(
(num_experts, hidden_size, padded_intermediate // 16)
@@ -322,7 +337,13 @@ def align_moe_weights_for_fi(
# Pad w13 and w2 along its intermediate dimension.
padded_w13 = w13.new_zeros((num_experts, padded_gate_up_dim, hidden_size))
padded_w13[:, : w13.shape[1], :] = w13
if is_act_and_mul:
padded_w13[:, :intermediate, :] = w13[:, :intermediate, :]
padded_w13[:, padded_intermediate : padded_intermediate + intermediate, :] = (
w13[:, intermediate:, :]
)
else:
padded_w13[:, :intermediate, :] = w13
padded_w2 = w2.new_zeros((num_experts, hidden_size, padded_intermediate))
padded_w2[:, :, :intermediate] = w2
@@ -233,6 +233,7 @@ class VocabParallelEmbedding(PluggableLayer):
quant_config: quant config for the layer
prefix: full name of the layer in the state dict
disable_tp: If true, tensor parallelism will be disabled for this layer.
quant_method: Preselected quantization method for model-specific layers.
""" # noqa: E501
# --8<-- [end:vocab_parallel_embedding]
@@ -248,6 +249,7 @@ class VocabParallelEmbedding(PluggableLayer):
prefix: str = "",
*,
disable_tp: bool = False,
quant_method: QuantizeMethodBase | None = None,
):
super().__init__()
@@ -281,8 +283,9 @@ class VocabParallelEmbedding(PluggableLayer):
)
self.embedding_dim = embedding_dim
quant_method = None
if quant_config is not None:
# Avoid overriding a preselected model-specific method with generic
# config-based dispatch.
if quant_method is None and quant_config is not None:
quant_method = quant_config.get_quant_method(self, prefix=prefix)
if quant_method is None:
quant_method = UnquantizedEmbeddingMethod()
@@ -501,8 +504,16 @@ class VocabParallelEmbedding(PluggableLayer):
output_parallel = self.quant_method.embedding(self, masked_input.long())
# Mask the output embedding.
if self.tp_size > 1:
if output_parallel.dtype in (
torch.float8_e4m3fn,
torch.float8_e5m2,
):
# Each vocab token has one owner, so FP8 bytes can use int8 SUM.
comm_output = output_parallel.view(torch.int8)
comm_output.masked_fill_(input_mask.unsqueeze(-1), 0)
output = tensor_model_parallel_all_reduce(comm_output)
return output.view(output_parallel.dtype)
output_parallel.masked_fill_(input_mask.unsqueeze(-1), 0)
# Reduce across all the model parallel GPUs.
return tensor_model_parallel_all_reduce(output_parallel)
return output_parallel
@@ -289,12 +289,17 @@ def get_quant_config(
# if hf_quant_config is None, we will try to get config from
# hf_overrides
hf_overrides = model_config.hf_overrides
if callable(hf_overrides):
# A callable hf_overrides is a config-to-config transform (e.g. the
# one SpeculativeConfig installs on draft model configs); it cannot
# carry quantization config entries, so treat it as no overrides.
hf_overrides = {}
if not isinstance(hf_overrides, dict):
raise ValueError(
"hf_overrides must be a dict for get_quant_config "
"to get the quantization config from it."
)
quantization_config_file = hf_overrides.get("quantization_config_file", None)
quantization_config_file = hf_overrides.get("quantization_config_file")
if quantization_config_file is not None:
if hasattr(quant_cls, "from_config_file"):
return quant_cls.from_config_file(quantization_config_file)
@@ -304,7 +309,7 @@ def get_quant_config(
"but quant_cls.from_config_file is not implemented in "
f"{quant_cls}"
)
quantization_config_json = hf_overrides.get("quantization_config_dict_json", None)
quantization_config_json = hf_overrides.get("quantization_config_dict_json")
if quantization_config_json is not None:
if hasattr(quant_cls, "from_config_dict_json"):
return quant_cls.from_config_dict_json(quantization_config_json)
+80
View File
@@ -839,6 +839,83 @@ class Qwen3_5ForCausalLMConfig(Qwen3_5ForConditionalGenerationConfig):
rope_parameters.pop("mrope_interleaved", None)
def _strip_qwen4_exp_mrope(model_config: "ModelConfig") -> None:
configs = {
id(config): config
for config in (
getattr(model_config, "hf_config", None),
model_config.hf_text_config,
)
if config is not None
}
for config in configs.values():
rope_parameters = getattr(config, "rope_parameters", None)
if rope_parameters is not None:
rope_parameters.pop("mrope_section", None)
rope_parameters.pop("mrope_interleaved", None)
class Qwen4ExpForConditionalGenerationConfig(Qwen3_5ForConditionalGenerationConfig):
"""Apply the Qwen3.5 hybrid-cache contract to Qwen4Exp."""
@staticmethod
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
Qwen3_5ForConditionalGenerationConfig.verify_and_update_config(vllm_config)
text_config = vllm_config.model_config.hf_text_config
if text_config.hc_count <= 1:
raise ValueError("Qwen4Exp requires hc_count > 1")
parallel_config = vllm_config.parallel_config
uses_ple_or_qsa = bool(text_config.ple_layer_ids) or (
getattr(text_config, "indexer_n_heads", None) is not None
)
if uses_ple_or_qsa and (
parallel_config.enable_dbo or parallel_config.ubatch_size > 1
):
raise NotImplementedError(
"Qwen4Exp PLE/QSA does not support dual-batch overlap or microbatching"
)
# Checked again in Qwen4ExpModelState; rejecting it here keeps the
# engine from loading weights first.
if text_config.ple_layer_ids and parallel_config.pipeline_parallel_size > 1:
raise NotImplementedError(
"Qwen4Exp N-gram PLE embedding requires pipeline_parallel_size=1 "
"because non-first pipeline ranks do not receive the raw input_ids "
"it needs. Please run with PP=1."
)
multimodal_config = vllm_config.model_config.multimodal_config
if multimodal_config is not None and multimodal_config.language_model_only:
_strip_qwen4_exp_mrope(vllm_config.model_config)
spec_config = vllm_config.speculative_config
if spec_config is not None and spec_config.method not in {
"mtp",
"ngram",
"ngram_gpu",
}:
raise NotImplementedError(
"Qwen4Exp speculative decoding supports only its native MTP "
"checkpoint and linear n-gram proposers"
)
class Qwen4ExpForCausalLMConfig(Qwen4ExpForConditionalGenerationConfig):
@staticmethod
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(vllm_config)
_strip_qwen4_exp_mrope(vllm_config.model_config)
class Qwen4ExpMTPConfig(Qwen4ExpForConditionalGenerationConfig):
"""Preserve MRoPE for a VL target and use 1D RoPE for a text target."""
@staticmethod
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(vllm_config)
if hasattr(vllm_config.model_config.hf_config, "vision_config"):
return
_strip_qwen4_exp_mrope(vllm_config.model_config)
class ColQwen3_5Config(Qwen3_5ForConditionalGenerationConfig):
"""Apply the attention contract declared by a ColQwen3.5 checkpoint."""
@@ -963,6 +1040,9 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
"Qwen3_5ForConditionalGeneration": Qwen3_5ForConditionalGenerationConfig,
"Qwen3_5MoeForCausalLM": Qwen3_5ForCausalLMConfig,
"Qwen3_5MoeForConditionalGeneration": Qwen3_5ForConditionalGenerationConfig,
"Qwen4ExpForCausalLM": Qwen4ExpForCausalLMConfig,
"Qwen4ExpForConditionalGeneration": (Qwen4ExpForConditionalGenerationConfig),
"Qwen4ExpMTP": Qwen4ExpMTPConfig,
"UnlimitedOCRForCausalLM": UnlimitedOCRForCausalLMConfig,
"VoyageQwen3BidirectionalEmbedModel": VoyageQwen3BidirectionalEmbedModelConfig,
"XLMRobertaModel": JinaRobertaModelConfig,
+27 -1
View File
@@ -49,13 +49,17 @@ if TYPE_CHECKING:
from vllm.lora.model_manager import LoRAModelManager
from vllm.model_executor.layers.fused_moe import MoERunner
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc
from vllm.model_executor.layers.mamba.mamba_utils import (
MambaStateCopyFunc,
MambaStateCopyFuncsByType,
)
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.models.utils import WeightsMapper
from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalKwargsItem
from vllm.multimodal.registry import _ProcessorFactories
from vllm.sequence import IntermediateTensors
from vllm.tasks import ScoreType
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.worker.encoder_cudagraph_defs import (
EncoderCudaGraphCaptureInputs,
EncoderCudaGraphConfig,
@@ -970,6 +974,14 @@ class IsHybrid(Protocol):
"""
...
@classmethod
def get_mamba_state_copy_funcs(
cls,
mamba_types: set["MambaAttentionBackendEnum"],
) -> "MambaStateCopyFuncsByType":
copy_funcs = cls.get_mamba_state_copy_func()
return {mamba_type: copy_funcs for mamba_type in mamba_types}
@overload
def is_hybrid(model: type[object]) -> TypeIs[type[IsHybrid]]: ...
@@ -1131,6 +1143,20 @@ class SupportsMambaPrefixCaching(Protocol):
supports_mamba_prefix_caching: ClassVar[Literal[True]] = True
@classmethod
def get_mamba_state_copy_func(cls) -> tuple["MambaStateCopyFunc", ...]:
"""Return copy functions for the model's Mamba states."""
...
@classmethod
def get_mamba_state_copy_funcs(
cls,
mamba_types: set["MambaAttentionBackendEnum"],
) -> "MambaStateCopyFuncsByType":
"""Map legacy copy functions to each requested Mamba backend."""
copy_funcs = cls.get_mamba_state_copy_func()
return {mamba_type: copy_funcs for mamba_type in mamba_types}
@overload
def supports_mamba_prefix_caching(
+4 -2
View File
@@ -80,15 +80,17 @@ class Qwen2MoeMLP(nn.Module):
reduce_results: bool = True,
expert_gate: torch.nn.Linear | None = None,
is_sequence_parallel: bool = False,
disable_tp: bool = False,
prefix: str = "",
) -> None:
super().__init__()
disable_tp = disable_tp or is_sequence_parallel
self.gate_up_proj = MergedColumnParallelLinear(
hidden_size,
[intermediate_size] * 2,
bias=False,
quant_config=quant_config,
disable_tp=is_sequence_parallel,
disable_tp=disable_tp,
prefix=f"{prefix}.gate_up_proj",
)
self.down_proj = RowParallelLinear(
@@ -97,7 +99,7 @@ class Qwen2MoeMLP(nn.Module):
bias=False,
quant_config=quant_config,
reduce_results=reduce_results,
disable_tp=is_sequence_parallel,
disable_tp=disable_tp,
prefix=f"{prefix}.down_proj",
)
if hidden_act != "silu":
+62 -3
View File
@@ -43,6 +43,9 @@ from vllm.model_executor.layers.mamba.mamba_utils import (
MambaStateShapeCalculator,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.utils.config_utils import (
get_quark_ocp_mx_group_size,
)
from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding, get_rope
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
@@ -90,6 +93,38 @@ def _should_use_sequence_parallel(vllm_config: VllmConfig) -> bool:
)
def _should_replicate_misaligned_shared_expert(
intermediate_size: int,
tp_size: int,
group_size: int | None,
enable_expert_parallel: bool,
is_sequence_parallel: bool,
) -> bool:
if intermediate_size <= 0 or group_size is None:
return False
partition_size, remainder = divmod(intermediate_size, tp_size)
if remainder == 0 and partition_size % group_size == 0:
return False
if enable_expert_parallel or is_sequence_parallel:
return True
if remainder != 0:
raise ValueError(
f"Shared-expert intermediate size {intermediate_size} must be "
f"divisible by tensor-parallel size {tp_size}."
)
raise ValueError(
"The Quark OCP MX shared expert cannot be tensor-parallelized: "
f"intermediate size {intermediate_size} with TP size {tp_size} "
f"produces a partition of {partition_size}, which is not divisible by "
f"the OCP MX group size {group_size}. Choose a compatible "
"tensor-parallel size or enable expert parallelism."
)
class Qwen3NextSparseMoeBlock(nn.Module):
def __init__(self, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -106,6 +141,17 @@ class Qwen3NextSparseMoeBlock(nn.Module):
self.is_sequence_parallel = parallel_config.use_sequence_parallel_moe
shared_expert_group_size = get_quark_ocp_mx_group_size(
quant_config,
f"{prefix}.shared_expert.down_proj",
)
self.replicate_shared_expert = _should_replicate_misaligned_shared_expert(
config.shared_expert_intermediate_size,
self.tp_size,
shared_expert_group_size,
parallel_config.enable_expert_parallel,
self.is_sequence_parallel,
)
if self.tp_size > config.num_experts:
raise ValueError(
f"Tensor parallel size {self.tp_size} is greater than "
@@ -138,7 +184,10 @@ class Qwen3NextSparseMoeBlock(nn.Module):
)
self.is_fused_shared_expert_enabled = False
if config.shared_expert_intermediate_size > 0:
if (
config.shared_expert_intermediate_size > 0
and not self.replicate_shared_expert
):
self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert(
quant_config,
prefix,
@@ -159,11 +208,14 @@ class Qwen3NextSparseMoeBlock(nn.Module):
reduce_results=False,
expert_gate=self.shared_expert_gate,
is_sequence_parallel=self.is_sequence_parallel,
disable_tp=self.replicate_shared_expert,
prefix=f"{prefix}.shared_expert",
)
self.experts = FusedMoEFactory(
shared_experts=self.shared_expert,
shared_experts=(
None if self.replicate_shared_expert else self.shared_expert
),
gate=self.gate,
num_experts=self.n_routed_experts,
top_k=config.num_experts_per_tok,
@@ -195,9 +247,16 @@ class Qwen3NextSparseMoeBlock(nn.Module):
if self.is_sequence_parallel and not already_sequence_parallel:
hidden_states = sequence_parallel_chunk(hidden_states)
replicated_shared_output = (
self.shared_expert(hidden_states)
if self.replicate_shared_expert and self.shared_expert is not None
else None
)
final_hidden_states = self.experts(
hidden_states=hidden_states, router_logits=hidden_states
)
if replicated_shared_output is not None:
final_hidden_states += replicated_shared_output
if self.is_sequence_parallel and not already_sequence_parallel:
final_hidden_states = tensor_model_parallel_all_gather(
@@ -488,7 +547,7 @@ class Qwen3NextDecoderLayer(nn.Module):
self,
hidden_states: torch.Tensor,
residual: torch.Tensor | None,
positions: torch.Tensor = None,
positions: torch.Tensor,
**kwargs: object,
):
full_num_tokens = positions.shape[-1]
+9
View File
@@ -111,6 +111,10 @@ _TEXT_GENERATION_MODELS = {
"Gemma3nForCausalLM": ("gemma3n", "Gemma3nForCausalLM"),
"Gemma4ForCausalLM": ("gemma4", "Gemma4ForCausalLM"),
"Qwen3NextForCausalLM": ("qwen3_next", "Qwen3NextForCausalLM"),
"Qwen4ExpForCausalLM": (
"vllm.models.qwen4_exp",
"Qwen4ExpForCausalLM",
),
"GlmForCausalLM": ("glm", "GlmForCausalLM"),
"Glm4ForCausalLM": ("glm4", "Glm4ForCausalLM"),
"Glm4MoeForCausalLM": ("glm4_moe", "Glm4MoeForCausalLM"),
@@ -573,6 +577,10 @@ _MULTIMODAL_MODELS = {
"qwen3_5",
"Qwen3_5MoeForConditionalGeneration",
),
"Qwen4ExpForConditionalGeneration": (
"vllm.models.qwen4_exp",
"Qwen4ExpForConditionalGeneration",
),
"RForConditionalGeneration": ("rvl", "RForConditionalGeneration"),
"SkyworkR1VChatModel": ("skyworkr1v", "SkyworkR1VChatModel"),
"SmolVLMForConditionalGeneration": ("smolvlm", "SmolVLMForConditionalGeneration"),
@@ -659,6 +667,7 @@ _SPECULATIVE_DECODING_MODELS = {
"MedusaModel": ("medusa", "Medusa"),
"OpenPanguMTPModel": ("openpangu_mtp", "OpenPanguMTP"),
"Qwen3NextMTP": ("qwen3_next_mtp", "Qwen3NextMTP"),
"Qwen4ExpMTP": ("vllm.models.qwen4_exp", "Qwen4ExpMTP"),
"Step3p5MTP": ("step3p5_mtp", "Step3p5MTP"),
"Qwen3_5MTP": ("qwen3_5_mtp", "Qwen3_5MTP"),
"Qwen3_5MoeMTP": ("qwen3_5_mtp", "Qwen3_5MoeMTP"),