[Model] Support Qwen3.8-Flash-Next (#53896)

Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com>
Signed-off-by: andyluo7 <andy.luo@amd.com>
Signed-off-by: lishunyang12 <lishunyang03@gmail.com>
Signed-off-by: Harry Huang <huanghaoyan.hhy@alibaba-inc.com>
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: KungYork <firelikest@163.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: andyluo7 <43718156+andyluo7@users.noreply.github.com>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: andyluo7 <andy.luo@amd.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: lishunyang12 <lishunyang03@gmail.com>
Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Harry Huang
2026-08-31 05:57:56 +00:00
committed by GitHub
co-authored by zjy0516 KungYork Thien Tran Yifan Qiao andyluo7 Isotr0py andyluo7 OpenAI Codex lishunyang12 Vadim Gimpelson Isotr0py Lucas Wilkinson Claude Opus 5 OpenAI Codex
parent 687db59744
commit e126687a9a
124 changed files with 19424 additions and 450 deletions
@@ -39,6 +39,7 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
is_deep_gemm_e8m0_used,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
FP8_DTYPE,
QuantKey,
kFp8Dynamic128Sym,
kFp8DynamicTensorSym,
@@ -153,6 +154,19 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
(kFp8StaticTensorSym, kFp8StaticTensorSym),
(kFp8StaticTensorSym, kFp8DynamicTensorSym),
]
# Block-quantized FP8 with arbitrary block shape: the Triton
# kernels take the block shape as a runtime argument.
if (
weight_key is not None
and activation_key == kFp8Dynamic128Sym
and weight_key.dtype == FP8_DTYPE
and weight_key.symmetric
and weight_key.scale.static
and weight_key.scale.dtype == torch.float32
and weight_key.scale.group_shape.row > 1
and weight_key.scale.group_shape.col > 1
):
return True
return (weight_key, activation_key) in supported
@staticmethod
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from enum import Enum
from typing import Any
@@ -268,6 +269,39 @@ def map_fp8_backend(runner_backend: MoEBackend) -> Fp8MoeBackend:
)
def refine_fp8_moe_block_shape(
config: FusedMoEConfig,
weight_block_size: list[int],
) -> list[int] | None:
"""
Compute a refined block shape for block-quantized FP8 MoE weights whose
checkpoint blocks cannot be sharded exactly across TP ranks.
TP shards the intermediate dim of the expert weights, so a per-shard size
that is not a multiple of the checkpoint's block size makes the
checkpoint's block scales impossible to shard exactly. When a finer block
size (>= 32) divides both the checkpoint blocks and all involved dims,
the weight scales can be refined to that granularity at load time (a
lossless upsampling, since the refined block divides the checkpoint
block). Only Triton-based kernels can consume the refined block shape:
they take it as a runtime argument, while the other backends require the
native 128x128 blocks. The refined shape is encoded in the QuantKey used
for backend selection, so backends that only support 128x128 blocks are
rejected by the oracle automatically.
Returns the refined [block_n, block_k] shape, or None if no refinement
is needed or possible.
"""
block_n, block_k = weight_block_size
ispp = config.intermediate_size_per_partition
if ispp % block_n == 0 and (config.tp_size == 1 or ispp % block_k == 0):
return None
refine = math.gcd(block_n, block_k, ispp, config.hidden_dim)
if refine < 32:
return None
return [refine, refine]
def select_fp8_moe_backend(
config: FusedMoEConfig,
weight_key: QuantKey | None,
@@ -474,6 +474,39 @@ def _get_requested_backends(
return _filter_by_activation(backends, requested_activation_key)
def _requires_qwen38_tep8_emulation(
config: FusedMoEConfig,
activation_key: QuantKey | None,
) -> bool:
"""Avoid the inaccurate native gfx950 kernel for Qwen3.8 TEP8.
Qwen3.8 Flash Next's routed experts use the distinctive
``E=512, H=2560, N=640`` W4A4 shape. With eight-way expert parallelism,
AITER operates on 64 local experts and pads ``N`` to 768. That native path
is not numerically reliable on gfx950, while OCP MX emulation preserves
model accuracy. Keep this guard exact so other MXFP4 shapes and smaller EP
configurations retain the native backend.
"""
parallel = config.moe_parallel_config
if not (
activation_key == kMxfp4Dynamic
and parallel.use_ep
and parallel.ep_size == 8
and config.num_experts == 512
and config.num_local_experts == 64
and config.hidden_dim == 2560
and config.intermediate_size == 640
):
return False
if not current_platform.is_rocm():
return False
from vllm.platforms.rocm import on_gfx950
return on_gfx950()
def select_mxfp4_moe_backend(
config: FusedMoEConfig,
activation_key: QuantKey | None = None,
@@ -532,6 +565,21 @@ def select_mxfp4_moe_backend(
assert last_error is not None
raise last_error
if _requires_qwen38_tep8_emulation(config, requested_activation_key):
backend = Mxfp4MoeBackend.EMULATION
logger.warning_once(
"Using OCP MX emulation for the Qwen3.8 Flash Next TEP8 routed "
"experts on gfx950 because the native AITER W4A4 kernel is not "
"numerically reliable for this shape. Performance will be lower."
)
return _return_or_raise(
backend,
config,
kMxfp4Static,
requested_activation_key,
activation_format,
)
# Select kernels in order of backend.
AVAILABLE_BACKENDS = _filter_by_activation(
_get_priority_backends_for_gpt_oss(), requested_activation_key
@@ -345,14 +345,13 @@ def convert_to_unquantized_kernel_format(
elif unquantized_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM:
is_act_and_mul = moe_config.is_act_and_mul
if not is_act_and_mul:
# Kernel requires intermediate_size_per_partition % 128 == 0 (BlockMajorK
# weight layout uses block_k=128). Pad along the intermediate dim when
# the model + TP split don't satisfy the constraint.
w13_weight, w2_weight, padded_intermediate = align_moe_weights_for_fi(
w13_weight, w2_weight, is_act_and_mul, min_alignment=128
)
moe_config.intermediate_size_per_partition = padded_intermediate
# Kernel requires intermediate_size_per_partition % 128 == 0 (BlockMajorK
# weight layout uses block_k=128). Pad along the intermediate dim when
# the model + TP split don't satisfy the constraint.
w13_weight, w2_weight, padded_intermediate = align_moe_weights_for_fi(
w13_weight, w2_weight, is_act_and_mul, min_alignment=128
)
moe_config.intermediate_size_per_partition = padded_intermediate
_cache_permute_indices: dict[torch.Size, torch.Tensor] = {}
w13_weight, w2_weight = convert_moe_weights_to_flashinfer_trtllm_block_layout(
@@ -832,6 +832,20 @@ class RoutedExperts(PluggableLayer):
FusedMoeWeightScaleSupported.GROUP.value,
FusedMoeWeightScaleSupported.BLOCK.value,
]:
scale_refine = getattr(self.quant_method, "weight_scale_refine", None)
if (
quant_method == FusedMoeWeightScaleSupported.BLOCK.value
and scale_refine is not None
):
# FP8 block scales are stored per (block_n, block_k) tile
# of the unsharded weight, while the TP-sharded parameters
# use a refined block grid (see Fp8MoEMethod). Upsample the
# scales to the refined grid (lossless: the refined block
# divides the checkpoint block) so per-rank slicing stays
# exact. Dim -2 is the weight's N dim, dim -1 is K.
loaded_weight = loaded_weight.repeat_interleave(
scale_refine[0], dim=-2
).repeat_interleave(scale_refine[1], dim=-1)
self._load_model_weight_or_group_weight_scale(
shard_id=shard_id,
shard_dim=shard_dim,
@@ -56,6 +56,10 @@ class MambaBase(AttentionLayerBase):
def mamba_type(self) -> MambaAttentionBackendEnum:
pass
@property
def is_kv_cache_tp_replicated(self) -> bool:
return False
@abstractmethod
def get_state_dtype(self) -> tuple[torch.dtype, ...]:
pass
@@ -70,6 +74,7 @@ class MambaBase(AttentionLayerBase):
block_size=mamba_block_size,
page_size_padded=page_size_padded,
mamba_type=self.mamba_type,
tp_replicated=self.is_kv_cache_tp_replicated,
mamba_cache_mode=vllm_config.cache_config.mamba_cache_mode,
# RecoverSSM verifies the whole window off one checkpoint, so it
# never writes the baseline's per-draft-token state slots.
@@ -519,7 +519,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
self.gqa_interleaved_layout
or self.head_k_dim != 128
or self.head_v_dim != 128
or self.norm.activation != "silu"
or self.norm.activation not in ("silu", "sigmoid")
or vllm_config.model_config.dtype != torch.bfloat16
or conv_state_dtype != torch.bfloat16
or recurrent_state_dtype not in FUSED_GDN_STATE_DTYPES
@@ -527,9 +527,9 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
):
return (
"the fused CUDA kernel requires a BF16 GDN model with "
"K=V=128, SiLU gating, non-interleaved GQA layout, BF16 "
"convolution cache, BF16 or FP32 recurrent state, and a "
"GPU with compute capability 8.0+"
"K=V=128, SiLU or sigmoid gating, non-interleaved GQA "
"layout, BF16 convolution cache, BF16 or FP32 recurrent "
"state, and a GPU with compute capability 8.0+"
)
if not hasattr(torch.ops._C, "fused_gdn_decode_post_conv_mtp"):
return "torch.ops._C.fused_gdn_decode_post_conv_mtp is not built"
@@ -1766,6 +1766,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
out=core_attn_out,
scale=self.head_k_dim**-0.5,
norm_eps=self.layer_norm_epsilon,
output_gate_activation=self.norm.activation,
)
def _forward_core_fused_norm_packed(
@@ -17,6 +17,7 @@ from vllm.utils.torch_utils import (
STR_DTYPE_TO_TORCH_DTYPE,
get_kv_cache_torch_dtype,
)
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
logger = init_logger(__name__)
@@ -352,6 +353,10 @@ Parameters:
num_accepted_tokens: int - number of accepted tokens used to compute the copy offset.
Range: 1 .. 1 + num_speculative_tokens (inclusive).
"""
MambaStateCopyFuncs: TypeAlias = tuple[MambaStateCopyFunc, ...]
MambaStateCopyFuncsByType: TypeAlias = dict[
MambaAttentionBackendEnum, MambaStateCopyFuncs
]
def get_conv_copy_spec(
+53 -13
View File
@@ -31,6 +31,7 @@ from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
convert_to_fp8_moe_kernel_format,
make_fp8_moe_kernel,
make_fp8_moe_quant_config,
refine_fp8_moe_block_shape,
select_fp8_moe_backend,
)
from vllm.model_executor.layers.linear import (
@@ -63,7 +64,6 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Dynamic128Sym,
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
kFp8Static128BlockSym,
kFp8StaticTensorSym,
)
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
@@ -481,9 +481,41 @@ class Fp8MoEMethod(FusedMoEMethodBase):
"weight_scale_inv" if self.block_quant else "weight_scale"
)
self.weight_scale_refine: tuple[int, int] | None = None
self.moe_block_shape = self.weight_block_size
# Set weight key and activation key for kernel compatibility
if self.block_quant:
weight_key = kFp8Static128BlockSym
assert self.weight_block_size is not None
# TP shards the intermediate dim of the expert weights, so a
# per-shard size that is not a multiple of the checkpoint's block
# size makes the checkpoint's block scales impossible to shard
# exactly. When a finer block size (>= 32) divides both the
# checkpoint blocks and all involved dims, the weight scales are
# refined to that granularity at load time (a lossless upsampling,
# since the refined block divides the checkpoint block). The
# refined block shape is encoded in the weight key, so the oracle
# only selects kernels that support it (e.g. Triton, which takes
# the block shape as a runtime argument).
refined_shape = refine_fp8_moe_block_shape(self.moe, self.weight_block_size)
if refined_shape is not None:
block_n, block_k = self.weight_block_size
self.weight_scale_refine = (
block_n // refined_shape[0],
block_k // refined_shape[1],
)
self.moe_block_shape = refined_shape
logger.info_once(
"FP8 MoE block scales refined from %s to %s to fit "
"the TP-sharded intermediate size %d.",
str(self.weight_block_size),
str(refined_shape),
self.moe.intermediate_size_per_partition,
)
assert self.moe_block_shape is not None
weight_key = create_fp8_quant_key(
static=True, group_shape=GroupShape(*self.moe_block_shape)
)
activation_key = kFp8Dynamic128Sym
else:
weight_key = kFp8StaticTensorSym
@@ -519,6 +551,8 @@ class Fp8MoEMethod(FusedMoEMethodBase):
if self.block_quant:
assert self.weight_block_size is not None
assert self.moe_block_shape is not None
moe_block_shape = self.moe_block_shape
layer.weight_block_size = self.weight_block_size
tp_size = get_tensor_model_parallel_world_size()
block_n, block_k = (
@@ -530,18 +564,24 @@ class Fp8MoEMethod(FusedMoEMethodBase):
# layers must be divisible by block_n.
# Required by column parallel or enabling merged weights
if intermediate_size_per_partition % block_n != 0:
raise ValueError(
f"The output_size of gate's and up's weight = "
f"{intermediate_size_per_partition} is not divisible by "
f"weight quantization block_n = {block_n}."
)
if self.weight_scale_refine is None:
raise ValueError(
f"The output_size of gate's and up's weight = "
f"{intermediate_size_per_partition} is not divisible by "
f"weight quantization block_n = {block_n}."
)
# Use the refined block grid for the scale parameters; the
# loader upsamples the checkpoint scales accordingly.
block_n, block_k = moe_block_shape
if tp_size > 1 and intermediate_size_per_partition % block_k != 0:
# Required by row parallel
raise ValueError(
f"The input_size of down's weight = "
f"{intermediate_size_per_partition} is not divisible by "
f"weight quantization block_k = {block_k}."
)
if self.weight_scale_refine is None:
raise ValueError(
f"The input_size of down's weight = "
f"{intermediate_size_per_partition} is not divisible by "
f"weight quantization block_k = {block_k}."
)
block_n, block_k = moe_block_shape
# WEIGHTS
w13_weight = torch.nn.Parameter(
@@ -747,7 +787,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
w2_scale=w2_scale,
a1_scale=a1_scale,
a2_scale=a2_scale,
block_shape=self.weight_block_size,
block_shape=self.moe_block_shape,
swiglu_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
@@ -161,6 +161,16 @@ class QuarkOCP_MX(QuarkScheme):
weight_loader: Callable,
**kwargs,
):
if input_size_per_partition % OCP_MX_BLOCK_SIZE != 0:
layer_name = getattr(layer, "prefix", "") or type(layer).__name__
raise ValueError(
f"OCP MX linear layer {layer_name!r} has an input size per "
f"partition of {input_size_per_partition}, which must be "
f"divisible by the OCP MX group size {OCP_MX_BLOCK_SIZE}. "
"Choose a compatible tensor-parallel size or avoid "
"tensor-parallel sharding for this layer."
)
if self.dynamic_mxfp4_quant:
weight = ModelWeightParameter(
data=torch.empty(
@@ -9,6 +9,41 @@ if TYPE_CHECKING:
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
def get_quark_ocp_mx_group_size(
quant_config: "QuantizationConfig | None",
layer_name: str,
) -> int | None:
"""Return the OCP MX group size for a quantized Quark linear layer."""
if quant_config is None:
return None
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
from vllm.model_executor.layers.quantization.quark.utils import should_ignore_layer
if not isinstance(quant_config, QuarkConfig):
return None
if should_ignore_layer(
layer_name,
ignore=quant_config.quant_config.get("exclude") or [],
fused_mapping=quant_config.packed_modules_mapping,
):
return None
layer_quant_config = (
quant_config.get_layer_quant_config_from_name(layer_name)
or quant_config.quant_config.get("global_quant_config")
or {}
)
weight_quant = layer_quant_config.get("weight")
input_quant = layer_quant_config.get("input_tensors")
if not quant_config._is_w_ocp_mx_a_x(weight_quant, input_quant):
return None
assert weight_quant is not None
return int(weight_quant["group_size"])
def is_shared_expert_quant_fse_compatible(
quant_config: "QuantizationConfig | None",
expert_prefix: str,
@@ -232,7 +232,16 @@ def align_fp4_moe_weights_for_fi(
# Pad w13 and w2 along its intermediate dimension.
padded_w13 = w13.new_zeros((num_experts, padded_gate_up_dim, hidden_size // 2))
padded_w13[:, : w13.shape[1], :] = w13
if is_act_and_mul:
# Keep the two logical projections independently aligned. Copying the
# fused [gate, up] tensor contiguously would move the up projection into
# the padded tail of gate when intermediate is rounded up.
padded_w13[:, :intermediate, :] = w13[:, :intermediate, :]
padded_w13[:, padded_intermediate : padded_intermediate + intermediate, :] = (
w13[:, intermediate:, :]
)
else:
padded_w13[:, : w13.shape[1], :] = w13
padded_w2 = w2.new_zeros((num_experts, hidden_size, padded_intermediate // 2))
padded_w2[:, :, : w2.shape[2]] = w2
@@ -240,7 +249,13 @@ def align_fp4_moe_weights_for_fi(
padded_w13_scale = w13_scale.new_zeros(
(num_experts, padded_gate_up_dim, hidden_size // 16)
)
padded_w13_scale[:, : w13_scale.shape[1], :] = w13_scale
if is_act_and_mul:
padded_w13_scale[:, :intermediate, :] = w13_scale[:, :intermediate, :]
padded_w13_scale[
:, padded_intermediate : padded_intermediate + intermediate, :
] = w13_scale[:, intermediate:, :]
else:
padded_w13_scale[:, : w13_scale.shape[1], :] = w13_scale
padded_w2_scale = w2_scale.new_zeros(
(num_experts, hidden_size, padded_intermediate // 16)
@@ -322,7 +337,13 @@ def align_moe_weights_for_fi(
# Pad w13 and w2 along its intermediate dimension.
padded_w13 = w13.new_zeros((num_experts, padded_gate_up_dim, hidden_size))
padded_w13[:, : w13.shape[1], :] = w13
if is_act_and_mul:
padded_w13[:, :intermediate, :] = w13[:, :intermediate, :]
padded_w13[:, padded_intermediate : padded_intermediate + intermediate, :] = (
w13[:, intermediate:, :]
)
else:
padded_w13[:, :intermediate, :] = w13
padded_w2 = w2.new_zeros((num_experts, hidden_size, padded_intermediate))
padded_w2[:, :, :intermediate] = w2
@@ -233,6 +233,7 @@ class VocabParallelEmbedding(PluggableLayer):
quant_config: quant config for the layer
prefix: full name of the layer in the state dict
disable_tp: If true, tensor parallelism will be disabled for this layer.
quant_method: Preselected quantization method for model-specific layers.
""" # noqa: E501
# --8<-- [end:vocab_parallel_embedding]
@@ -248,6 +249,7 @@ class VocabParallelEmbedding(PluggableLayer):
prefix: str = "",
*,
disable_tp: bool = False,
quant_method: QuantizeMethodBase | None = None,
):
super().__init__()
@@ -281,8 +283,9 @@ class VocabParallelEmbedding(PluggableLayer):
)
self.embedding_dim = embedding_dim
quant_method = None
if quant_config is not None:
# Avoid overriding a preselected model-specific method with generic
# config-based dispatch.
if quant_method is None and quant_config is not None:
quant_method = quant_config.get_quant_method(self, prefix=prefix)
if quant_method is None:
quant_method = UnquantizedEmbeddingMethod()
@@ -501,8 +504,16 @@ class VocabParallelEmbedding(PluggableLayer):
output_parallel = self.quant_method.embedding(self, masked_input.long())
# Mask the output embedding.
if self.tp_size > 1:
if output_parallel.dtype in (
torch.float8_e4m3fn,
torch.float8_e5m2,
):
# Each vocab token has one owner, so FP8 bytes can use int8 SUM.
comm_output = output_parallel.view(torch.int8)
comm_output.masked_fill_(input_mask.unsqueeze(-1), 0)
output = tensor_model_parallel_all_reduce(comm_output)
return output.view(output_parallel.dtype)
output_parallel.masked_fill_(input_mask.unsqueeze(-1), 0)
# Reduce across all the model parallel GPUs.
return tensor_model_parallel_all_reduce(output_parallel)
return output_parallel