[Model] Support Qwen3.8-Flash-Next (#53896)

Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com>
Signed-off-by: andyluo7 <andy.luo@amd.com>
Signed-off-by: lishunyang12 <lishunyang03@gmail.com>
Signed-off-by: Harry Huang <huanghaoyan.hhy@alibaba-inc.com>
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: KungYork <firelikest@163.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: andyluo7 <43718156+andyluo7@users.noreply.github.com>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: andyluo7 <andy.luo@amd.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: lishunyang12 <lishunyang03@gmail.com>
Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Harry Huang
2026-08-31 05:57:56 +00:00
committed by GitHub
co-authored by zjy0516 KungYork Thien Tran Yifan Qiao andyluo7 Isotr0py andyluo7 OpenAI Codex lishunyang12 Vadim Gimpelson Isotr0py Lucas Wilkinson Claude Opus 5 OpenAI Codex
parent 687db59744
commit e126687a9a
124 changed files with 19424 additions and 450 deletions
+53 -13
View File
@@ -31,6 +31,7 @@ from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
convert_to_fp8_moe_kernel_format,
make_fp8_moe_kernel,
make_fp8_moe_quant_config,
refine_fp8_moe_block_shape,
select_fp8_moe_backend,
)
from vllm.model_executor.layers.linear import (
@@ -63,7 +64,6 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Dynamic128Sym,
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
kFp8Static128BlockSym,
kFp8StaticTensorSym,
)
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
@@ -481,9 +481,41 @@ class Fp8MoEMethod(FusedMoEMethodBase):
"weight_scale_inv" if self.block_quant else "weight_scale"
)
self.weight_scale_refine: tuple[int, int] | None = None
self.moe_block_shape = self.weight_block_size
# Set weight key and activation key for kernel compatibility
if self.block_quant:
weight_key = kFp8Static128BlockSym
assert self.weight_block_size is not None
# TP shards the intermediate dim of the expert weights, so a
# per-shard size that is not a multiple of the checkpoint's block
# size makes the checkpoint's block scales impossible to shard
# exactly. When a finer block size (>= 32) divides both the
# checkpoint blocks and all involved dims, the weight scales are
# refined to that granularity at load time (a lossless upsampling,
# since the refined block divides the checkpoint block). The
# refined block shape is encoded in the weight key, so the oracle
# only selects kernels that support it (e.g. Triton, which takes
# the block shape as a runtime argument).
refined_shape = refine_fp8_moe_block_shape(self.moe, self.weight_block_size)
if refined_shape is not None:
block_n, block_k = self.weight_block_size
self.weight_scale_refine = (
block_n // refined_shape[0],
block_k // refined_shape[1],
)
self.moe_block_shape = refined_shape
logger.info_once(
"FP8 MoE block scales refined from %s to %s to fit "
"the TP-sharded intermediate size %d.",
str(self.weight_block_size),
str(refined_shape),
self.moe.intermediate_size_per_partition,
)
assert self.moe_block_shape is not None
weight_key = create_fp8_quant_key(
static=True, group_shape=GroupShape(*self.moe_block_shape)
)
activation_key = kFp8Dynamic128Sym
else:
weight_key = kFp8StaticTensorSym
@@ -519,6 +551,8 @@ class Fp8MoEMethod(FusedMoEMethodBase):
if self.block_quant:
assert self.weight_block_size is not None
assert self.moe_block_shape is not None
moe_block_shape = self.moe_block_shape
layer.weight_block_size = self.weight_block_size
tp_size = get_tensor_model_parallel_world_size()
block_n, block_k = (
@@ -530,18 +564,24 @@ class Fp8MoEMethod(FusedMoEMethodBase):
# layers must be divisible by block_n.
# Required by column parallel or enabling merged weights
if intermediate_size_per_partition % block_n != 0:
raise ValueError(
f"The output_size of gate's and up's weight = "
f"{intermediate_size_per_partition} is not divisible by "
f"weight quantization block_n = {block_n}."
)
if self.weight_scale_refine is None:
raise ValueError(
f"The output_size of gate's and up's weight = "
f"{intermediate_size_per_partition} is not divisible by "
f"weight quantization block_n = {block_n}."
)
# Use the refined block grid for the scale parameters; the
# loader upsamples the checkpoint scales accordingly.
block_n, block_k = moe_block_shape
if tp_size > 1 and intermediate_size_per_partition % block_k != 0:
# Required by row parallel
raise ValueError(
f"The input_size of down's weight = "
f"{intermediate_size_per_partition} is not divisible by "
f"weight quantization block_k = {block_k}."
)
if self.weight_scale_refine is None:
raise ValueError(
f"The input_size of down's weight = "
f"{intermediate_size_per_partition} is not divisible by "
f"weight quantization block_k = {block_k}."
)
block_n, block_k = moe_block_shape
# WEIGHTS
w13_weight = torch.nn.Parameter(
@@ -747,7 +787,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
w2_scale=w2_scale,
a1_scale=a1_scale,
a2_scale=a2_scale,
block_shape=self.weight_block_size,
block_shape=self.moe_block_shape,
swiglu_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
@@ -161,6 +161,16 @@ class QuarkOCP_MX(QuarkScheme):
weight_loader: Callable,
**kwargs,
):
if input_size_per_partition % OCP_MX_BLOCK_SIZE != 0:
layer_name = getattr(layer, "prefix", "") or type(layer).__name__
raise ValueError(
f"OCP MX linear layer {layer_name!r} has an input size per "
f"partition of {input_size_per_partition}, which must be "
f"divisible by the OCP MX group size {OCP_MX_BLOCK_SIZE}. "
"Choose a compatible tensor-parallel size or avoid "
"tensor-parallel sharding for this layer."
)
if self.dynamic_mxfp4_quant:
weight = ModelWeightParameter(
data=torch.empty(
@@ -9,6 +9,41 @@ if TYPE_CHECKING:
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
def get_quark_ocp_mx_group_size(
quant_config: "QuantizationConfig | None",
layer_name: str,
) -> int | None:
"""Return the OCP MX group size for a quantized Quark linear layer."""
if quant_config is None:
return None
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
from vllm.model_executor.layers.quantization.quark.utils import should_ignore_layer
if not isinstance(quant_config, QuarkConfig):
return None
if should_ignore_layer(
layer_name,
ignore=quant_config.quant_config.get("exclude") or [],
fused_mapping=quant_config.packed_modules_mapping,
):
return None
layer_quant_config = (
quant_config.get_layer_quant_config_from_name(layer_name)
or quant_config.quant_config.get("global_quant_config")
or {}
)
weight_quant = layer_quant_config.get("weight")
input_quant = layer_quant_config.get("input_tensors")
if not quant_config._is_w_ocp_mx_a_x(weight_quant, input_quant):
return None
assert weight_quant is not None
return int(weight_quant["group_size"])
def is_shared_expert_quant_fse_compatible(
quant_config: "QuantizationConfig | None",
expert_prefix: str,
@@ -232,7 +232,16 @@ def align_fp4_moe_weights_for_fi(
# Pad w13 and w2 along its intermediate dimension.
padded_w13 = w13.new_zeros((num_experts, padded_gate_up_dim, hidden_size // 2))
padded_w13[:, : w13.shape[1], :] = w13
if is_act_and_mul:
# Keep the two logical projections independently aligned. Copying the
# fused [gate, up] tensor contiguously would move the up projection into
# the padded tail of gate when intermediate is rounded up.
padded_w13[:, :intermediate, :] = w13[:, :intermediate, :]
padded_w13[:, padded_intermediate : padded_intermediate + intermediate, :] = (
w13[:, intermediate:, :]
)
else:
padded_w13[:, : w13.shape[1], :] = w13
padded_w2 = w2.new_zeros((num_experts, hidden_size, padded_intermediate // 2))
padded_w2[:, :, : w2.shape[2]] = w2
@@ -240,7 +249,13 @@ def align_fp4_moe_weights_for_fi(
padded_w13_scale = w13_scale.new_zeros(
(num_experts, padded_gate_up_dim, hidden_size // 16)
)
padded_w13_scale[:, : w13_scale.shape[1], :] = w13_scale
if is_act_and_mul:
padded_w13_scale[:, :intermediate, :] = w13_scale[:, :intermediate, :]
padded_w13_scale[
:, padded_intermediate : padded_intermediate + intermediate, :
] = w13_scale[:, intermediate:, :]
else:
padded_w13_scale[:, : w13_scale.shape[1], :] = w13_scale
padded_w2_scale = w2_scale.new_zeros(
(num_experts, hidden_size, padded_intermediate // 16)
@@ -322,7 +337,13 @@ def align_moe_weights_for_fi(
# Pad w13 and w2 along its intermediate dimension.
padded_w13 = w13.new_zeros((num_experts, padded_gate_up_dim, hidden_size))
padded_w13[:, : w13.shape[1], :] = w13
if is_act_and_mul:
padded_w13[:, :intermediate, :] = w13[:, :intermediate, :]
padded_w13[:, padded_intermediate : padded_intermediate + intermediate, :] = (
w13[:, intermediate:, :]
)
else:
padded_w13[:, :intermediate, :] = w13
padded_w2 = w2.new_zeros((num_experts, hidden_size, padded_intermediate))
padded_w2[:, :, :intermediate] = w2