[Model] Support Qwen3.8-Flash-Next (#53896)
Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com> Signed-off-by: andyluo7 <andy.luo@amd.com> Signed-off-by: lishunyang12 <lishunyang03@gmail.com> Signed-off-by: Harry Huang <huanghaoyan.hhy@alibaba-inc.com> Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: KungYork <firelikest@163.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: andyluo7 <43718156+andyluo7@users.noreply.github.com> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: andyluo7 <andy.luo@amd.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: lishunyang12 <lishunyang03@gmail.com> Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
zjy0516
KungYork
Thien Tran
Yifan Qiao
andyluo7
Isotr0py
andyluo7
OpenAI Codex
lishunyang12
Vadim Gimpelson
Isotr0py
Lucas Wilkinson
Claude Opus 5
OpenAI Codex
parent
687db59744
commit
e126687a9a
@@ -39,6 +39,7 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
is_deep_gemm_e8m0_used,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
FP8_DTYPE,
|
||||
QuantKey,
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8DynamicTensorSym,
|
||||
@@ -153,6 +154,19 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
|
||||
(kFp8StaticTensorSym, kFp8StaticTensorSym),
|
||||
(kFp8StaticTensorSym, kFp8DynamicTensorSym),
|
||||
]
|
||||
# Block-quantized FP8 with arbitrary block shape: the Triton
|
||||
# kernels take the block shape as a runtime argument.
|
||||
if (
|
||||
weight_key is not None
|
||||
and activation_key == kFp8Dynamic128Sym
|
||||
and weight_key.dtype == FP8_DTYPE
|
||||
and weight_key.symmetric
|
||||
and weight_key.scale.static
|
||||
and weight_key.scale.dtype == torch.float32
|
||||
and weight_key.scale.group_shape.row > 1
|
||||
and weight_key.scale.group_shape.col > 1
|
||||
):
|
||||
return True
|
||||
return (weight_key, activation_key) in supported
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import math
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
|
||||
@@ -268,6 +269,39 @@ def map_fp8_backend(runner_backend: MoEBackend) -> Fp8MoeBackend:
|
||||
)
|
||||
|
||||
|
||||
def refine_fp8_moe_block_shape(
|
||||
config: FusedMoEConfig,
|
||||
weight_block_size: list[int],
|
||||
) -> list[int] | None:
|
||||
"""
|
||||
Compute a refined block shape for block-quantized FP8 MoE weights whose
|
||||
checkpoint blocks cannot be sharded exactly across TP ranks.
|
||||
|
||||
TP shards the intermediate dim of the expert weights, so a per-shard size
|
||||
that is not a multiple of the checkpoint's block size makes the
|
||||
checkpoint's block scales impossible to shard exactly. When a finer block
|
||||
size (>= 32) divides both the checkpoint blocks and all involved dims,
|
||||
the weight scales can be refined to that granularity at load time (a
|
||||
lossless upsampling, since the refined block divides the checkpoint
|
||||
block). Only Triton-based kernels can consume the refined block shape:
|
||||
they take it as a runtime argument, while the other backends require the
|
||||
native 128x128 blocks. The refined shape is encoded in the QuantKey used
|
||||
for backend selection, so backends that only support 128x128 blocks are
|
||||
rejected by the oracle automatically.
|
||||
|
||||
Returns the refined [block_n, block_k] shape, or None if no refinement
|
||||
is needed or possible.
|
||||
"""
|
||||
block_n, block_k = weight_block_size
|
||||
ispp = config.intermediate_size_per_partition
|
||||
if ispp % block_n == 0 and (config.tp_size == 1 or ispp % block_k == 0):
|
||||
return None
|
||||
refine = math.gcd(block_n, block_k, ispp, config.hidden_dim)
|
||||
if refine < 32:
|
||||
return None
|
||||
return [refine, refine]
|
||||
|
||||
|
||||
def select_fp8_moe_backend(
|
||||
config: FusedMoEConfig,
|
||||
weight_key: QuantKey | None,
|
||||
|
||||
@@ -474,6 +474,39 @@ def _get_requested_backends(
|
||||
return _filter_by_activation(backends, requested_activation_key)
|
||||
|
||||
|
||||
def _requires_qwen38_tep8_emulation(
|
||||
config: FusedMoEConfig,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
"""Avoid the inaccurate native gfx950 kernel for Qwen3.8 TEP8.
|
||||
|
||||
Qwen3.8 Flash Next's routed experts use the distinctive
|
||||
``E=512, H=2560, N=640`` W4A4 shape. With eight-way expert parallelism,
|
||||
AITER operates on 64 local experts and pads ``N`` to 768. That native path
|
||||
is not numerically reliable on gfx950, while OCP MX emulation preserves
|
||||
model accuracy. Keep this guard exact so other MXFP4 shapes and smaller EP
|
||||
configurations retain the native backend.
|
||||
"""
|
||||
parallel = config.moe_parallel_config
|
||||
if not (
|
||||
activation_key == kMxfp4Dynamic
|
||||
and parallel.use_ep
|
||||
and parallel.ep_size == 8
|
||||
and config.num_experts == 512
|
||||
and config.num_local_experts == 64
|
||||
and config.hidden_dim == 2560
|
||||
and config.intermediate_size == 640
|
||||
):
|
||||
return False
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
return on_gfx950()
|
||||
|
||||
|
||||
def select_mxfp4_moe_backend(
|
||||
config: FusedMoEConfig,
|
||||
activation_key: QuantKey | None = None,
|
||||
@@ -532,6 +565,21 @@ def select_mxfp4_moe_backend(
|
||||
assert last_error is not None
|
||||
raise last_error
|
||||
|
||||
if _requires_qwen38_tep8_emulation(config, requested_activation_key):
|
||||
backend = Mxfp4MoeBackend.EMULATION
|
||||
logger.warning_once(
|
||||
"Using OCP MX emulation for the Qwen3.8 Flash Next TEP8 routed "
|
||||
"experts on gfx950 because the native AITER W4A4 kernel is not "
|
||||
"numerically reliable for this shape. Performance will be lower."
|
||||
)
|
||||
return _return_or_raise(
|
||||
backend,
|
||||
config,
|
||||
kMxfp4Static,
|
||||
requested_activation_key,
|
||||
activation_format,
|
||||
)
|
||||
|
||||
# Select kernels in order of backend.
|
||||
AVAILABLE_BACKENDS = _filter_by_activation(
|
||||
_get_priority_backends_for_gpt_oss(), requested_activation_key
|
||||
|
||||
@@ -345,14 +345,13 @@ def convert_to_unquantized_kernel_format(
|
||||
|
||||
elif unquantized_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM:
|
||||
is_act_and_mul = moe_config.is_act_and_mul
|
||||
if not is_act_and_mul:
|
||||
# Kernel requires intermediate_size_per_partition % 128 == 0 (BlockMajorK
|
||||
# weight layout uses block_k=128). Pad along the intermediate dim when
|
||||
# the model + TP split don't satisfy the constraint.
|
||||
w13_weight, w2_weight, padded_intermediate = align_moe_weights_for_fi(
|
||||
w13_weight, w2_weight, is_act_and_mul, min_alignment=128
|
||||
)
|
||||
moe_config.intermediate_size_per_partition = padded_intermediate
|
||||
# Kernel requires intermediate_size_per_partition % 128 == 0 (BlockMajorK
|
||||
# weight layout uses block_k=128). Pad along the intermediate dim when
|
||||
# the model + TP split don't satisfy the constraint.
|
||||
w13_weight, w2_weight, padded_intermediate = align_moe_weights_for_fi(
|
||||
w13_weight, w2_weight, is_act_and_mul, min_alignment=128
|
||||
)
|
||||
moe_config.intermediate_size_per_partition = padded_intermediate
|
||||
|
||||
_cache_permute_indices: dict[torch.Size, torch.Tensor] = {}
|
||||
w13_weight, w2_weight = convert_moe_weights_to_flashinfer_trtllm_block_layout(
|
||||
|
||||
@@ -832,6 +832,20 @@ class RoutedExperts(PluggableLayer):
|
||||
FusedMoeWeightScaleSupported.GROUP.value,
|
||||
FusedMoeWeightScaleSupported.BLOCK.value,
|
||||
]:
|
||||
scale_refine = getattr(self.quant_method, "weight_scale_refine", None)
|
||||
if (
|
||||
quant_method == FusedMoeWeightScaleSupported.BLOCK.value
|
||||
and scale_refine is not None
|
||||
):
|
||||
# FP8 block scales are stored per (block_n, block_k) tile
|
||||
# of the unsharded weight, while the TP-sharded parameters
|
||||
# use a refined block grid (see Fp8MoEMethod). Upsample the
|
||||
# scales to the refined grid (lossless: the refined block
|
||||
# divides the checkpoint block) so per-rank slicing stays
|
||||
# exact. Dim -2 is the weight's N dim, dim -1 is K.
|
||||
loaded_weight = loaded_weight.repeat_interleave(
|
||||
scale_refine[0], dim=-2
|
||||
).repeat_interleave(scale_refine[1], dim=-1)
|
||||
self._load_model_weight_or_group_weight_scale(
|
||||
shard_id=shard_id,
|
||||
shard_dim=shard_dim,
|
||||
|
||||
@@ -56,6 +56,10 @@ class MambaBase(AttentionLayerBase):
|
||||
def mamba_type(self) -> MambaAttentionBackendEnum:
|
||||
pass
|
||||
|
||||
@property
|
||||
def is_kv_cache_tp_replicated(self) -> bool:
|
||||
return False
|
||||
|
||||
@abstractmethod
|
||||
def get_state_dtype(self) -> tuple[torch.dtype, ...]:
|
||||
pass
|
||||
@@ -70,6 +74,7 @@ class MambaBase(AttentionLayerBase):
|
||||
block_size=mamba_block_size,
|
||||
page_size_padded=page_size_padded,
|
||||
mamba_type=self.mamba_type,
|
||||
tp_replicated=self.is_kv_cache_tp_replicated,
|
||||
mamba_cache_mode=vllm_config.cache_config.mamba_cache_mode,
|
||||
# RecoverSSM verifies the whole window off one checkpoint, so it
|
||||
# never writes the baseline's per-draft-token state slots.
|
||||
|
||||
@@ -519,7 +519,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
self.gqa_interleaved_layout
|
||||
or self.head_k_dim != 128
|
||||
or self.head_v_dim != 128
|
||||
or self.norm.activation != "silu"
|
||||
or self.norm.activation not in ("silu", "sigmoid")
|
||||
or vllm_config.model_config.dtype != torch.bfloat16
|
||||
or conv_state_dtype != torch.bfloat16
|
||||
or recurrent_state_dtype not in FUSED_GDN_STATE_DTYPES
|
||||
@@ -527,9 +527,9 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
):
|
||||
return (
|
||||
"the fused CUDA kernel requires a BF16 GDN model with "
|
||||
"K=V=128, SiLU gating, non-interleaved GQA layout, BF16 "
|
||||
"convolution cache, BF16 or FP32 recurrent state, and a "
|
||||
"GPU with compute capability 8.0+"
|
||||
"K=V=128, SiLU or sigmoid gating, non-interleaved GQA "
|
||||
"layout, BF16 convolution cache, BF16 or FP32 recurrent "
|
||||
"state, and a GPU with compute capability 8.0+"
|
||||
)
|
||||
if not hasattr(torch.ops._C, "fused_gdn_decode_post_conv_mtp"):
|
||||
return "torch.ops._C.fused_gdn_decode_post_conv_mtp is not built"
|
||||
@@ -1766,6 +1766,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
out=core_attn_out,
|
||||
scale=self.head_k_dim**-0.5,
|
||||
norm_eps=self.layer_norm_epsilon,
|
||||
output_gate_activation=self.norm.activation,
|
||||
)
|
||||
|
||||
def _forward_core_fused_norm_packed(
|
||||
|
||||
@@ -17,6 +17,7 @@ from vllm.utils.torch_utils import (
|
||||
STR_DTYPE_TO_TORCH_DTYPE,
|
||||
get_kv_cache_torch_dtype,
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -352,6 +353,10 @@ Parameters:
|
||||
num_accepted_tokens: int - number of accepted tokens used to compute the copy offset.
|
||||
Range: 1 .. 1 + num_speculative_tokens (inclusive).
|
||||
"""
|
||||
MambaStateCopyFuncs: TypeAlias = tuple[MambaStateCopyFunc, ...]
|
||||
MambaStateCopyFuncsByType: TypeAlias = dict[
|
||||
MambaAttentionBackendEnum, MambaStateCopyFuncs
|
||||
]
|
||||
|
||||
|
||||
def get_conv_copy_spec(
|
||||
|
||||
@@ -31,6 +31,7 @@ from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
convert_to_fp8_moe_kernel_format,
|
||||
make_fp8_moe_kernel,
|
||||
make_fp8_moe_quant_config,
|
||||
refine_fp8_moe_block_shape,
|
||||
select_fp8_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
@@ -63,7 +64,6 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8DynamicTensorSym,
|
||||
kFp8DynamicTokenSym,
|
||||
kFp8Static128BlockSym,
|
||||
kFp8StaticTensorSym,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
|
||||
@@ -481,9 +481,41 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
"weight_scale_inv" if self.block_quant else "weight_scale"
|
||||
)
|
||||
|
||||
self.weight_scale_refine: tuple[int, int] | None = None
|
||||
self.moe_block_shape = self.weight_block_size
|
||||
|
||||
# Set weight key and activation key for kernel compatibility
|
||||
if self.block_quant:
|
||||
weight_key = kFp8Static128BlockSym
|
||||
assert self.weight_block_size is not None
|
||||
# TP shards the intermediate dim of the expert weights, so a
|
||||
# per-shard size that is not a multiple of the checkpoint's block
|
||||
# size makes the checkpoint's block scales impossible to shard
|
||||
# exactly. When a finer block size (>= 32) divides both the
|
||||
# checkpoint blocks and all involved dims, the weight scales are
|
||||
# refined to that granularity at load time (a lossless upsampling,
|
||||
# since the refined block divides the checkpoint block). The
|
||||
# refined block shape is encoded in the weight key, so the oracle
|
||||
# only selects kernels that support it (e.g. Triton, which takes
|
||||
# the block shape as a runtime argument).
|
||||
refined_shape = refine_fp8_moe_block_shape(self.moe, self.weight_block_size)
|
||||
if refined_shape is not None:
|
||||
block_n, block_k = self.weight_block_size
|
||||
self.weight_scale_refine = (
|
||||
block_n // refined_shape[0],
|
||||
block_k // refined_shape[1],
|
||||
)
|
||||
self.moe_block_shape = refined_shape
|
||||
logger.info_once(
|
||||
"FP8 MoE block scales refined from %s to %s to fit "
|
||||
"the TP-sharded intermediate size %d.",
|
||||
str(self.weight_block_size),
|
||||
str(refined_shape),
|
||||
self.moe.intermediate_size_per_partition,
|
||||
)
|
||||
assert self.moe_block_shape is not None
|
||||
weight_key = create_fp8_quant_key(
|
||||
static=True, group_shape=GroupShape(*self.moe_block_shape)
|
||||
)
|
||||
activation_key = kFp8Dynamic128Sym
|
||||
else:
|
||||
weight_key = kFp8StaticTensorSym
|
||||
@@ -519,6 +551,8 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
|
||||
if self.block_quant:
|
||||
assert self.weight_block_size is not None
|
||||
assert self.moe_block_shape is not None
|
||||
moe_block_shape = self.moe_block_shape
|
||||
layer.weight_block_size = self.weight_block_size
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
block_n, block_k = (
|
||||
@@ -530,18 +564,24 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
# layers must be divisible by block_n.
|
||||
# Required by column parallel or enabling merged weights
|
||||
if intermediate_size_per_partition % block_n != 0:
|
||||
raise ValueError(
|
||||
f"The output_size of gate's and up's weight = "
|
||||
f"{intermediate_size_per_partition} is not divisible by "
|
||||
f"weight quantization block_n = {block_n}."
|
||||
)
|
||||
if self.weight_scale_refine is None:
|
||||
raise ValueError(
|
||||
f"The output_size of gate's and up's weight = "
|
||||
f"{intermediate_size_per_partition} is not divisible by "
|
||||
f"weight quantization block_n = {block_n}."
|
||||
)
|
||||
# Use the refined block grid for the scale parameters; the
|
||||
# loader upsamples the checkpoint scales accordingly.
|
||||
block_n, block_k = moe_block_shape
|
||||
if tp_size > 1 and intermediate_size_per_partition % block_k != 0:
|
||||
# Required by row parallel
|
||||
raise ValueError(
|
||||
f"The input_size of down's weight = "
|
||||
f"{intermediate_size_per_partition} is not divisible by "
|
||||
f"weight quantization block_k = {block_k}."
|
||||
)
|
||||
if self.weight_scale_refine is None:
|
||||
raise ValueError(
|
||||
f"The input_size of down's weight = "
|
||||
f"{intermediate_size_per_partition} is not divisible by "
|
||||
f"weight quantization block_k = {block_k}."
|
||||
)
|
||||
block_n, block_k = moe_block_shape
|
||||
|
||||
# WEIGHTS
|
||||
w13_weight = torch.nn.Parameter(
|
||||
@@ -747,7 +787,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
w2_scale=w2_scale,
|
||||
a1_scale=a1_scale,
|
||||
a2_scale=a2_scale,
|
||||
block_shape=self.weight_block_size,
|
||||
block_shape=self.moe_block_shape,
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
|
||||
@@ -161,6 +161,16 @@ class QuarkOCP_MX(QuarkScheme):
|
||||
weight_loader: Callable,
|
||||
**kwargs,
|
||||
):
|
||||
if input_size_per_partition % OCP_MX_BLOCK_SIZE != 0:
|
||||
layer_name = getattr(layer, "prefix", "") or type(layer).__name__
|
||||
raise ValueError(
|
||||
f"OCP MX linear layer {layer_name!r} has an input size per "
|
||||
f"partition of {input_size_per_partition}, which must be "
|
||||
f"divisible by the OCP MX group size {OCP_MX_BLOCK_SIZE}. "
|
||||
"Choose a compatible tensor-parallel size or avoid "
|
||||
"tensor-parallel sharding for this layer."
|
||||
)
|
||||
|
||||
if self.dynamic_mxfp4_quant:
|
||||
weight = ModelWeightParameter(
|
||||
data=torch.empty(
|
||||
|
||||
@@ -9,6 +9,41 @@ if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
|
||||
|
||||
def get_quark_ocp_mx_group_size(
|
||||
quant_config: "QuantizationConfig | None",
|
||||
layer_name: str,
|
||||
) -> int | None:
|
||||
"""Return the OCP MX group size for a quantized Quark linear layer."""
|
||||
if quant_config is None:
|
||||
return None
|
||||
|
||||
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
|
||||
from vllm.model_executor.layers.quantization.quark.utils import should_ignore_layer
|
||||
|
||||
if not isinstance(quant_config, QuarkConfig):
|
||||
return None
|
||||
|
||||
if should_ignore_layer(
|
||||
layer_name,
|
||||
ignore=quant_config.quant_config.get("exclude") or [],
|
||||
fused_mapping=quant_config.packed_modules_mapping,
|
||||
):
|
||||
return None
|
||||
|
||||
layer_quant_config = (
|
||||
quant_config.get_layer_quant_config_from_name(layer_name)
|
||||
or quant_config.quant_config.get("global_quant_config")
|
||||
or {}
|
||||
)
|
||||
weight_quant = layer_quant_config.get("weight")
|
||||
input_quant = layer_quant_config.get("input_tensors")
|
||||
if not quant_config._is_w_ocp_mx_a_x(weight_quant, input_quant):
|
||||
return None
|
||||
|
||||
assert weight_quant is not None
|
||||
return int(weight_quant["group_size"])
|
||||
|
||||
|
||||
def is_shared_expert_quant_fse_compatible(
|
||||
quant_config: "QuantizationConfig | None",
|
||||
expert_prefix: str,
|
||||
|
||||
@@ -232,7 +232,16 @@ def align_fp4_moe_weights_for_fi(
|
||||
|
||||
# Pad w13 and w2 along its intermediate dimension.
|
||||
padded_w13 = w13.new_zeros((num_experts, padded_gate_up_dim, hidden_size // 2))
|
||||
padded_w13[:, : w13.shape[1], :] = w13
|
||||
if is_act_and_mul:
|
||||
# Keep the two logical projections independently aligned. Copying the
|
||||
# fused [gate, up] tensor contiguously would move the up projection into
|
||||
# the padded tail of gate when intermediate is rounded up.
|
||||
padded_w13[:, :intermediate, :] = w13[:, :intermediate, :]
|
||||
padded_w13[:, padded_intermediate : padded_intermediate + intermediate, :] = (
|
||||
w13[:, intermediate:, :]
|
||||
)
|
||||
else:
|
||||
padded_w13[:, : w13.shape[1], :] = w13
|
||||
|
||||
padded_w2 = w2.new_zeros((num_experts, hidden_size, padded_intermediate // 2))
|
||||
padded_w2[:, :, : w2.shape[2]] = w2
|
||||
@@ -240,7 +249,13 @@ def align_fp4_moe_weights_for_fi(
|
||||
padded_w13_scale = w13_scale.new_zeros(
|
||||
(num_experts, padded_gate_up_dim, hidden_size // 16)
|
||||
)
|
||||
padded_w13_scale[:, : w13_scale.shape[1], :] = w13_scale
|
||||
if is_act_and_mul:
|
||||
padded_w13_scale[:, :intermediate, :] = w13_scale[:, :intermediate, :]
|
||||
padded_w13_scale[
|
||||
:, padded_intermediate : padded_intermediate + intermediate, :
|
||||
] = w13_scale[:, intermediate:, :]
|
||||
else:
|
||||
padded_w13_scale[:, : w13_scale.shape[1], :] = w13_scale
|
||||
|
||||
padded_w2_scale = w2_scale.new_zeros(
|
||||
(num_experts, hidden_size, padded_intermediate // 16)
|
||||
@@ -322,7 +337,13 @@ def align_moe_weights_for_fi(
|
||||
|
||||
# Pad w13 and w2 along its intermediate dimension.
|
||||
padded_w13 = w13.new_zeros((num_experts, padded_gate_up_dim, hidden_size))
|
||||
padded_w13[:, : w13.shape[1], :] = w13
|
||||
if is_act_and_mul:
|
||||
padded_w13[:, :intermediate, :] = w13[:, :intermediate, :]
|
||||
padded_w13[:, padded_intermediate : padded_intermediate + intermediate, :] = (
|
||||
w13[:, intermediate:, :]
|
||||
)
|
||||
else:
|
||||
padded_w13[:, :intermediate, :] = w13
|
||||
|
||||
padded_w2 = w2.new_zeros((num_experts, hidden_size, padded_intermediate))
|
||||
padded_w2[:, :, :intermediate] = w2
|
||||
|
||||
@@ -233,6 +233,7 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
quant_config: quant config for the layer
|
||||
prefix: full name of the layer in the state dict
|
||||
disable_tp: If true, tensor parallelism will be disabled for this layer.
|
||||
quant_method: Preselected quantization method for model-specific layers.
|
||||
""" # noqa: E501
|
||||
|
||||
# --8<-- [end:vocab_parallel_embedding]
|
||||
@@ -248,6 +249,7 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
prefix: str = "",
|
||||
*,
|
||||
disable_tp: bool = False,
|
||||
quant_method: QuantizeMethodBase | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
@@ -281,8 +283,9 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
)
|
||||
self.embedding_dim = embedding_dim
|
||||
|
||||
quant_method = None
|
||||
if quant_config is not None:
|
||||
# Avoid overriding a preselected model-specific method with generic
|
||||
# config-based dispatch.
|
||||
if quant_method is None and quant_config is not None:
|
||||
quant_method = quant_config.get_quant_method(self, prefix=prefix)
|
||||
if quant_method is None:
|
||||
quant_method = UnquantizedEmbeddingMethod()
|
||||
@@ -501,8 +504,16 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
output_parallel = self.quant_method.embedding(self, masked_input.long())
|
||||
# Mask the output embedding.
|
||||
if self.tp_size > 1:
|
||||
if output_parallel.dtype in (
|
||||
torch.float8_e4m3fn,
|
||||
torch.float8_e5m2,
|
||||
):
|
||||
# Each vocab token has one owner, so FP8 bytes can use int8 SUM.
|
||||
comm_output = output_parallel.view(torch.int8)
|
||||
comm_output.masked_fill_(input_mask.unsqueeze(-1), 0)
|
||||
output = tensor_model_parallel_all_reduce(comm_output)
|
||||
return output.view(output_parallel.dtype)
|
||||
output_parallel.masked_fill_(input_mask.unsqueeze(-1), 0)
|
||||
# Reduce across all the model parallel GPUs.
|
||||
return tensor_model_parallel_all_reduce(output_parallel)
|
||||
return output_parallel
|
||||
|
||||
|
||||
@@ -289,12 +289,17 @@ def get_quant_config(
|
||||
# if hf_quant_config is None, we will try to get config from
|
||||
# hf_overrides
|
||||
hf_overrides = model_config.hf_overrides
|
||||
if callable(hf_overrides):
|
||||
# A callable hf_overrides is a config-to-config transform (e.g. the
|
||||
# one SpeculativeConfig installs on draft model configs); it cannot
|
||||
# carry quantization config entries, so treat it as no overrides.
|
||||
hf_overrides = {}
|
||||
if not isinstance(hf_overrides, dict):
|
||||
raise ValueError(
|
||||
"hf_overrides must be a dict for get_quant_config "
|
||||
"to get the quantization config from it."
|
||||
)
|
||||
quantization_config_file = hf_overrides.get("quantization_config_file", None)
|
||||
quantization_config_file = hf_overrides.get("quantization_config_file")
|
||||
if quantization_config_file is not None:
|
||||
if hasattr(quant_cls, "from_config_file"):
|
||||
return quant_cls.from_config_file(quantization_config_file)
|
||||
@@ -304,7 +309,7 @@ def get_quant_config(
|
||||
"but quant_cls.from_config_file is not implemented in "
|
||||
f"{quant_cls}"
|
||||
)
|
||||
quantization_config_json = hf_overrides.get("quantization_config_dict_json", None)
|
||||
quantization_config_json = hf_overrides.get("quantization_config_dict_json")
|
||||
if quantization_config_json is not None:
|
||||
if hasattr(quant_cls, "from_config_dict_json"):
|
||||
return quant_cls.from_config_dict_json(quantization_config_json)
|
||||
|
||||
@@ -839,6 +839,83 @@ class Qwen3_5ForCausalLMConfig(Qwen3_5ForConditionalGenerationConfig):
|
||||
rope_parameters.pop("mrope_interleaved", None)
|
||||
|
||||
|
||||
def _strip_qwen4_exp_mrope(model_config: "ModelConfig") -> None:
|
||||
configs = {
|
||||
id(config): config
|
||||
for config in (
|
||||
getattr(model_config, "hf_config", None),
|
||||
model_config.hf_text_config,
|
||||
)
|
||||
if config is not None
|
||||
}
|
||||
for config in configs.values():
|
||||
rope_parameters = getattr(config, "rope_parameters", None)
|
||||
if rope_parameters is not None:
|
||||
rope_parameters.pop("mrope_section", None)
|
||||
rope_parameters.pop("mrope_interleaved", None)
|
||||
|
||||
|
||||
class Qwen4ExpForConditionalGenerationConfig(Qwen3_5ForConditionalGenerationConfig):
|
||||
"""Apply the Qwen3.5 hybrid-cache contract to Qwen4Exp."""
|
||||
|
||||
@staticmethod
|
||||
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
|
||||
Qwen3_5ForConditionalGenerationConfig.verify_and_update_config(vllm_config)
|
||||
text_config = vllm_config.model_config.hf_text_config
|
||||
if text_config.hc_count <= 1:
|
||||
raise ValueError("Qwen4Exp requires hc_count > 1")
|
||||
parallel_config = vllm_config.parallel_config
|
||||
uses_ple_or_qsa = bool(text_config.ple_layer_ids) or (
|
||||
getattr(text_config, "indexer_n_heads", None) is not None
|
||||
)
|
||||
if uses_ple_or_qsa and (
|
||||
parallel_config.enable_dbo or parallel_config.ubatch_size > 1
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp PLE/QSA does not support dual-batch overlap or microbatching"
|
||||
)
|
||||
# Checked again in Qwen4ExpModelState; rejecting it here keeps the
|
||||
# engine from loading weights first.
|
||||
if text_config.ple_layer_ids and parallel_config.pipeline_parallel_size > 1:
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp N-gram PLE embedding requires pipeline_parallel_size=1 "
|
||||
"because non-first pipeline ranks do not receive the raw input_ids "
|
||||
"it needs. Please run with PP=1."
|
||||
)
|
||||
multimodal_config = vllm_config.model_config.multimodal_config
|
||||
if multimodal_config is not None and multimodal_config.language_model_only:
|
||||
_strip_qwen4_exp_mrope(vllm_config.model_config)
|
||||
spec_config = vllm_config.speculative_config
|
||||
if spec_config is not None and spec_config.method not in {
|
||||
"mtp",
|
||||
"ngram",
|
||||
"ngram_gpu",
|
||||
}:
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp speculative decoding supports only its native MTP "
|
||||
"checkpoint and linear n-gram proposers"
|
||||
)
|
||||
|
||||
|
||||
class Qwen4ExpForCausalLMConfig(Qwen4ExpForConditionalGenerationConfig):
|
||||
@staticmethod
|
||||
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
|
||||
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(vllm_config)
|
||||
|
||||
_strip_qwen4_exp_mrope(vllm_config.model_config)
|
||||
|
||||
|
||||
class Qwen4ExpMTPConfig(Qwen4ExpForConditionalGenerationConfig):
|
||||
"""Preserve MRoPE for a VL target and use 1D RoPE for a text target."""
|
||||
|
||||
@staticmethod
|
||||
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
|
||||
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(vllm_config)
|
||||
if hasattr(vllm_config.model_config.hf_config, "vision_config"):
|
||||
return
|
||||
_strip_qwen4_exp_mrope(vllm_config.model_config)
|
||||
|
||||
|
||||
class ColQwen3_5Config(Qwen3_5ForConditionalGenerationConfig):
|
||||
"""Apply the attention contract declared by a ColQwen3.5 checkpoint."""
|
||||
|
||||
@@ -963,6 +1040,9 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
|
||||
"Qwen3_5ForConditionalGeneration": Qwen3_5ForConditionalGenerationConfig,
|
||||
"Qwen3_5MoeForCausalLM": Qwen3_5ForCausalLMConfig,
|
||||
"Qwen3_5MoeForConditionalGeneration": Qwen3_5ForConditionalGenerationConfig,
|
||||
"Qwen4ExpForCausalLM": Qwen4ExpForCausalLMConfig,
|
||||
"Qwen4ExpForConditionalGeneration": (Qwen4ExpForConditionalGenerationConfig),
|
||||
"Qwen4ExpMTP": Qwen4ExpMTPConfig,
|
||||
"UnlimitedOCRForCausalLM": UnlimitedOCRForCausalLMConfig,
|
||||
"VoyageQwen3BidirectionalEmbedModel": VoyageQwen3BidirectionalEmbedModelConfig,
|
||||
"XLMRobertaModel": JinaRobertaModelConfig,
|
||||
|
||||
@@ -49,13 +49,17 @@ if TYPE_CHECKING:
|
||||
from vllm.lora.model_manager import LoRAModelManager
|
||||
from vllm.model_executor.layers.fused_moe import MoERunner
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateCopyFunc,
|
||||
MambaStateCopyFuncsByType,
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalKwargsItem
|
||||
from vllm.multimodal.registry import _ProcessorFactories
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tasks import ScoreType
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import (
|
||||
EncoderCudaGraphCaptureInputs,
|
||||
EncoderCudaGraphConfig,
|
||||
@@ -970,6 +974,14 @@ class IsHybrid(Protocol):
|
||||
"""
|
||||
...
|
||||
|
||||
@classmethod
|
||||
def get_mamba_state_copy_funcs(
|
||||
cls,
|
||||
mamba_types: set["MambaAttentionBackendEnum"],
|
||||
) -> "MambaStateCopyFuncsByType":
|
||||
copy_funcs = cls.get_mamba_state_copy_func()
|
||||
return {mamba_type: copy_funcs for mamba_type in mamba_types}
|
||||
|
||||
|
||||
@overload
|
||||
def is_hybrid(model: type[object]) -> TypeIs[type[IsHybrid]]: ...
|
||||
@@ -1131,6 +1143,20 @@ class SupportsMambaPrefixCaching(Protocol):
|
||||
|
||||
supports_mamba_prefix_caching: ClassVar[Literal[True]] = True
|
||||
|
||||
@classmethod
|
||||
def get_mamba_state_copy_func(cls) -> tuple["MambaStateCopyFunc", ...]:
|
||||
"""Return copy functions for the model's Mamba states."""
|
||||
...
|
||||
|
||||
@classmethod
|
||||
def get_mamba_state_copy_funcs(
|
||||
cls,
|
||||
mamba_types: set["MambaAttentionBackendEnum"],
|
||||
) -> "MambaStateCopyFuncsByType":
|
||||
"""Map legacy copy functions to each requested Mamba backend."""
|
||||
copy_funcs = cls.get_mamba_state_copy_func()
|
||||
return {mamba_type: copy_funcs for mamba_type in mamba_types}
|
||||
|
||||
|
||||
@overload
|
||||
def supports_mamba_prefix_caching(
|
||||
|
||||
@@ -80,15 +80,17 @@ class Qwen2MoeMLP(nn.Module):
|
||||
reduce_results: bool = True,
|
||||
expert_gate: torch.nn.Linear | None = None,
|
||||
is_sequence_parallel: bool = False,
|
||||
disable_tp: bool = False,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
disable_tp = disable_tp or is_sequence_parallel
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
hidden_size,
|
||||
[intermediate_size] * 2,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
disable_tp=is_sequence_parallel,
|
||||
disable_tp=disable_tp,
|
||||
prefix=f"{prefix}.gate_up_proj",
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
@@ -97,7 +99,7 @@ class Qwen2MoeMLP(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
reduce_results=reduce_results,
|
||||
disable_tp=is_sequence_parallel,
|
||||
disable_tp=disable_tp,
|
||||
prefix=f"{prefix}.down_proj",
|
||||
)
|
||||
if hidden_act != "silu":
|
||||
|
||||
@@ -43,6 +43,9 @@ from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateShapeCalculator,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.utils.config_utils import (
|
||||
get_quark_ocp_mx_group_size,
|
||||
)
|
||||
from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding, get_rope
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
@@ -90,6 +93,38 @@ def _should_use_sequence_parallel(vllm_config: VllmConfig) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def _should_replicate_misaligned_shared_expert(
|
||||
intermediate_size: int,
|
||||
tp_size: int,
|
||||
group_size: int | None,
|
||||
enable_expert_parallel: bool,
|
||||
is_sequence_parallel: bool,
|
||||
) -> bool:
|
||||
if intermediate_size <= 0 or group_size is None:
|
||||
return False
|
||||
|
||||
partition_size, remainder = divmod(intermediate_size, tp_size)
|
||||
if remainder == 0 and partition_size % group_size == 0:
|
||||
return False
|
||||
|
||||
if enable_expert_parallel or is_sequence_parallel:
|
||||
return True
|
||||
|
||||
if remainder != 0:
|
||||
raise ValueError(
|
||||
f"Shared-expert intermediate size {intermediate_size} must be "
|
||||
f"divisible by tensor-parallel size {tp_size}."
|
||||
)
|
||||
|
||||
raise ValueError(
|
||||
"The Quark OCP MX shared expert cannot be tensor-parallelized: "
|
||||
f"intermediate size {intermediate_size} with TP size {tp_size} "
|
||||
f"produces a partition of {partition_size}, which is not divisible by "
|
||||
f"the OCP MX group size {group_size}. Choose a compatible "
|
||||
"tensor-parallel size or enable expert parallelism."
|
||||
)
|
||||
|
||||
|
||||
class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
def __init__(self, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
@@ -106,6 +141,17 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
|
||||
self.is_sequence_parallel = parallel_config.use_sequence_parallel_moe
|
||||
|
||||
shared_expert_group_size = get_quark_ocp_mx_group_size(
|
||||
quant_config,
|
||||
f"{prefix}.shared_expert.down_proj",
|
||||
)
|
||||
self.replicate_shared_expert = _should_replicate_misaligned_shared_expert(
|
||||
config.shared_expert_intermediate_size,
|
||||
self.tp_size,
|
||||
shared_expert_group_size,
|
||||
parallel_config.enable_expert_parallel,
|
||||
self.is_sequence_parallel,
|
||||
)
|
||||
if self.tp_size > config.num_experts:
|
||||
raise ValueError(
|
||||
f"Tensor parallel size {self.tp_size} is greater than "
|
||||
@@ -138,7 +184,10 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
)
|
||||
|
||||
self.is_fused_shared_expert_enabled = False
|
||||
if config.shared_expert_intermediate_size > 0:
|
||||
if (
|
||||
config.shared_expert_intermediate_size > 0
|
||||
and not self.replicate_shared_expert
|
||||
):
|
||||
self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert(
|
||||
quant_config,
|
||||
prefix,
|
||||
@@ -159,11 +208,14 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
reduce_results=False,
|
||||
expert_gate=self.shared_expert_gate,
|
||||
is_sequence_parallel=self.is_sequence_parallel,
|
||||
disable_tp=self.replicate_shared_expert,
|
||||
prefix=f"{prefix}.shared_expert",
|
||||
)
|
||||
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_expert,
|
||||
shared_experts=(
|
||||
None if self.replicate_shared_expert else self.shared_expert
|
||||
),
|
||||
gate=self.gate,
|
||||
num_experts=self.n_routed_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
@@ -195,9 +247,16 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
if self.is_sequence_parallel and not already_sequence_parallel:
|
||||
hidden_states = sequence_parallel_chunk(hidden_states)
|
||||
|
||||
replicated_shared_output = (
|
||||
self.shared_expert(hidden_states)
|
||||
if self.replicate_shared_expert and self.shared_expert is not None
|
||||
else None
|
||||
)
|
||||
final_hidden_states = self.experts(
|
||||
hidden_states=hidden_states, router_logits=hidden_states
|
||||
)
|
||||
if replicated_shared_output is not None:
|
||||
final_hidden_states += replicated_shared_output
|
||||
|
||||
if self.is_sequence_parallel and not already_sequence_parallel:
|
||||
final_hidden_states = tensor_model_parallel_all_gather(
|
||||
@@ -488,7 +547,7 @@ class Qwen3NextDecoderLayer(nn.Module):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None,
|
||||
positions: torch.Tensor = None,
|
||||
positions: torch.Tensor,
|
||||
**kwargs: object,
|
||||
):
|
||||
full_num_tokens = positions.shape[-1]
|
||||
|
||||
@@ -111,6 +111,10 @@ _TEXT_GENERATION_MODELS = {
|
||||
"Gemma3nForCausalLM": ("gemma3n", "Gemma3nForCausalLM"),
|
||||
"Gemma4ForCausalLM": ("gemma4", "Gemma4ForCausalLM"),
|
||||
"Qwen3NextForCausalLM": ("qwen3_next", "Qwen3NextForCausalLM"),
|
||||
"Qwen4ExpForCausalLM": (
|
||||
"vllm.models.qwen4_exp",
|
||||
"Qwen4ExpForCausalLM",
|
||||
),
|
||||
"GlmForCausalLM": ("glm", "GlmForCausalLM"),
|
||||
"Glm4ForCausalLM": ("glm4", "Glm4ForCausalLM"),
|
||||
"Glm4MoeForCausalLM": ("glm4_moe", "Glm4MoeForCausalLM"),
|
||||
@@ -573,6 +577,10 @@ _MULTIMODAL_MODELS = {
|
||||
"qwen3_5",
|
||||
"Qwen3_5MoeForConditionalGeneration",
|
||||
),
|
||||
"Qwen4ExpForConditionalGeneration": (
|
||||
"vllm.models.qwen4_exp",
|
||||
"Qwen4ExpForConditionalGeneration",
|
||||
),
|
||||
"RForConditionalGeneration": ("rvl", "RForConditionalGeneration"),
|
||||
"SkyworkR1VChatModel": ("skyworkr1v", "SkyworkR1VChatModel"),
|
||||
"SmolVLMForConditionalGeneration": ("smolvlm", "SmolVLMForConditionalGeneration"),
|
||||
@@ -659,6 +667,7 @@ _SPECULATIVE_DECODING_MODELS = {
|
||||
"MedusaModel": ("medusa", "Medusa"),
|
||||
"OpenPanguMTPModel": ("openpangu_mtp", "OpenPanguMTP"),
|
||||
"Qwen3NextMTP": ("qwen3_next_mtp", "Qwen3NextMTP"),
|
||||
"Qwen4ExpMTP": ("vllm.models.qwen4_exp", "Qwen4ExpMTP"),
|
||||
"Step3p5MTP": ("step3p5_mtp", "Step3p5MTP"),
|
||||
"Qwen3_5MTP": ("qwen3_5_mtp", "Qwen3_5MTP"),
|
||||
"Qwen3_5MoeMTP": ("qwen3_5_mtp", "Qwen3_5MoeMTP"),
|
||||
|
||||
Reference in New Issue
Block a user