[Model] Support Qwen3.8-Flash-Next (#53896)

Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com>
Signed-off-by: andyluo7 <andy.luo@amd.com>
Signed-off-by: lishunyang12 <lishunyang03@gmail.com>
Signed-off-by: Harry Huang <huanghaoyan.hhy@alibaba-inc.com>
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: KungYork <firelikest@163.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: andyluo7 <43718156+andyluo7@users.noreply.github.com>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: andyluo7 <andy.luo@amd.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: lishunyang12 <lishunyang03@gmail.com>
Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Harry Huang
2026-08-31 05:57:56 +00:00
committed by GitHub
co-authored by zjy0516 KungYork Thien Tran Yifan Qiao andyluo7 Isotr0py andyluo7 OpenAI Codex lishunyang12 Vadim Gimpelson Isotr0py Lucas Wilkinson Claude Opus 5 OpenAI Codex
parent 687db59744
commit e126687a9a
124 changed files with 19424 additions and 450 deletions
+53 -13
View File
@@ -31,6 +31,7 @@ from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
convert_to_fp8_moe_kernel_format,
make_fp8_moe_kernel,
make_fp8_moe_quant_config,
refine_fp8_moe_block_shape,
select_fp8_moe_backend,
)
from vllm.model_executor.layers.linear import (
@@ -63,7 +64,6 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Dynamic128Sym,
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
kFp8Static128BlockSym,
kFp8StaticTensorSym,
)
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
@@ -481,9 +481,41 @@ class Fp8MoEMethod(FusedMoEMethodBase):
"weight_scale_inv" if self.block_quant else "weight_scale"
)
self.weight_scale_refine: tuple[int, int] | None = None
self.moe_block_shape = self.weight_block_size
# Set weight key and activation key for kernel compatibility
if self.block_quant:
weight_key = kFp8Static128BlockSym
assert self.weight_block_size is not None
# TP shards the intermediate dim of the expert weights, so a
# per-shard size that is not a multiple of the checkpoint's block
# size makes the checkpoint's block scales impossible to shard
# exactly. When a finer block size (>= 32) divides both the
# checkpoint blocks and all involved dims, the weight scales are
# refined to that granularity at load time (a lossless upsampling,
# since the refined block divides the checkpoint block). The
# refined block shape is encoded in the weight key, so the oracle
# only selects kernels that support it (e.g. Triton, which takes
# the block shape as a runtime argument).
refined_shape = refine_fp8_moe_block_shape(self.moe, self.weight_block_size)
if refined_shape is not None:
block_n, block_k = self.weight_block_size
self.weight_scale_refine = (
block_n // refined_shape[0],
block_k // refined_shape[1],
)
self.moe_block_shape = refined_shape
logger.info_once(
"FP8 MoE block scales refined from %s to %s to fit "
"the TP-sharded intermediate size %d.",
str(self.weight_block_size),
str(refined_shape),
self.moe.intermediate_size_per_partition,
)
assert self.moe_block_shape is not None
weight_key = create_fp8_quant_key(
static=True, group_shape=GroupShape(*self.moe_block_shape)
)
activation_key = kFp8Dynamic128Sym
else:
weight_key = kFp8StaticTensorSym
@@ -519,6 +551,8 @@ class Fp8MoEMethod(FusedMoEMethodBase):
if self.block_quant:
assert self.weight_block_size is not None
assert self.moe_block_shape is not None
moe_block_shape = self.moe_block_shape
layer.weight_block_size = self.weight_block_size
tp_size = get_tensor_model_parallel_world_size()
block_n, block_k = (
@@ -530,18 +564,24 @@ class Fp8MoEMethod(FusedMoEMethodBase):
# layers must be divisible by block_n.
# Required by column parallel or enabling merged weights
if intermediate_size_per_partition % block_n != 0:
raise ValueError(
f"The output_size of gate's and up's weight = "
f"{intermediate_size_per_partition} is not divisible by "
f"weight quantization block_n = {block_n}."
)
if self.weight_scale_refine is None:
raise ValueError(
f"The output_size of gate's and up's weight = "
f"{intermediate_size_per_partition} is not divisible by "
f"weight quantization block_n = {block_n}."
)
# Use the refined block grid for the scale parameters; the
# loader upsamples the checkpoint scales accordingly.
block_n, block_k = moe_block_shape
if tp_size > 1 and intermediate_size_per_partition % block_k != 0:
# Required by row parallel
raise ValueError(
f"The input_size of down's weight = "
f"{intermediate_size_per_partition} is not divisible by "
f"weight quantization block_k = {block_k}."
)
if self.weight_scale_refine is None:
raise ValueError(
f"The input_size of down's weight = "
f"{intermediate_size_per_partition} is not divisible by "
f"weight quantization block_k = {block_k}."
)
block_n, block_k = moe_block_shape
# WEIGHTS
w13_weight = torch.nn.Parameter(
@@ -747,7 +787,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
w2_scale=w2_scale,
a1_scale=a1_scale,
a2_scale=a2_scale,
block_shape=self.weight_block_size,
block_shape=self.moe_block_shape,
swiglu_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),