[Model] Support Qwen3.8-Flash-Next (#53896)
Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com> Signed-off-by: andyluo7 <andy.luo@amd.com> Signed-off-by: lishunyang12 <lishunyang03@gmail.com> Signed-off-by: Harry Huang <huanghaoyan.hhy@alibaba-inc.com> Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: KungYork <firelikest@163.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: andyluo7 <43718156+andyluo7@users.noreply.github.com> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: andyluo7 <andy.luo@amd.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: lishunyang12 <lishunyang03@gmail.com> Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
zjy0516
KungYork
Thien Tran
Yifan Qiao
andyluo7
Isotr0py
andyluo7
OpenAI Codex
lishunyang12
Vadim Gimpelson
Isotr0py
Lucas Wilkinson
Claude Opus 5
OpenAI Codex
parent
687db59744
commit
e126687a9a
@@ -9,6 +9,41 @@ if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
|
||||
|
||||
def get_quark_ocp_mx_group_size(
|
||||
quant_config: "QuantizationConfig | None",
|
||||
layer_name: str,
|
||||
) -> int | None:
|
||||
"""Return the OCP MX group size for a quantized Quark linear layer."""
|
||||
if quant_config is None:
|
||||
return None
|
||||
|
||||
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
|
||||
from vllm.model_executor.layers.quantization.quark.utils import should_ignore_layer
|
||||
|
||||
if not isinstance(quant_config, QuarkConfig):
|
||||
return None
|
||||
|
||||
if should_ignore_layer(
|
||||
layer_name,
|
||||
ignore=quant_config.quant_config.get("exclude") or [],
|
||||
fused_mapping=quant_config.packed_modules_mapping,
|
||||
):
|
||||
return None
|
||||
|
||||
layer_quant_config = (
|
||||
quant_config.get_layer_quant_config_from_name(layer_name)
|
||||
or quant_config.quant_config.get("global_quant_config")
|
||||
or {}
|
||||
)
|
||||
weight_quant = layer_quant_config.get("weight")
|
||||
input_quant = layer_quant_config.get("input_tensors")
|
||||
if not quant_config._is_w_ocp_mx_a_x(weight_quant, input_quant):
|
||||
return None
|
||||
|
||||
assert weight_quant is not None
|
||||
return int(weight_quant["group_size"])
|
||||
|
||||
|
||||
def is_shared_expert_quant_fse_compatible(
|
||||
quant_config: "QuantizationConfig | None",
|
||||
expert_prefix: str,
|
||||
|
||||
@@ -232,7 +232,16 @@ def align_fp4_moe_weights_for_fi(
|
||||
|
||||
# Pad w13 and w2 along its intermediate dimension.
|
||||
padded_w13 = w13.new_zeros((num_experts, padded_gate_up_dim, hidden_size // 2))
|
||||
padded_w13[:, : w13.shape[1], :] = w13
|
||||
if is_act_and_mul:
|
||||
# Keep the two logical projections independently aligned. Copying the
|
||||
# fused [gate, up] tensor contiguously would move the up projection into
|
||||
# the padded tail of gate when intermediate is rounded up.
|
||||
padded_w13[:, :intermediate, :] = w13[:, :intermediate, :]
|
||||
padded_w13[:, padded_intermediate : padded_intermediate + intermediate, :] = (
|
||||
w13[:, intermediate:, :]
|
||||
)
|
||||
else:
|
||||
padded_w13[:, : w13.shape[1], :] = w13
|
||||
|
||||
padded_w2 = w2.new_zeros((num_experts, hidden_size, padded_intermediate // 2))
|
||||
padded_w2[:, :, : w2.shape[2]] = w2
|
||||
@@ -240,7 +249,13 @@ def align_fp4_moe_weights_for_fi(
|
||||
padded_w13_scale = w13_scale.new_zeros(
|
||||
(num_experts, padded_gate_up_dim, hidden_size // 16)
|
||||
)
|
||||
padded_w13_scale[:, : w13_scale.shape[1], :] = w13_scale
|
||||
if is_act_and_mul:
|
||||
padded_w13_scale[:, :intermediate, :] = w13_scale[:, :intermediate, :]
|
||||
padded_w13_scale[
|
||||
:, padded_intermediate : padded_intermediate + intermediate, :
|
||||
] = w13_scale[:, intermediate:, :]
|
||||
else:
|
||||
padded_w13_scale[:, : w13_scale.shape[1], :] = w13_scale
|
||||
|
||||
padded_w2_scale = w2_scale.new_zeros(
|
||||
(num_experts, hidden_size, padded_intermediate // 16)
|
||||
@@ -322,7 +337,13 @@ def align_moe_weights_for_fi(
|
||||
|
||||
# Pad w13 and w2 along its intermediate dimension.
|
||||
padded_w13 = w13.new_zeros((num_experts, padded_gate_up_dim, hidden_size))
|
||||
padded_w13[:, : w13.shape[1], :] = w13
|
||||
if is_act_and_mul:
|
||||
padded_w13[:, :intermediate, :] = w13[:, :intermediate, :]
|
||||
padded_w13[:, padded_intermediate : padded_intermediate + intermediate, :] = (
|
||||
w13[:, intermediate:, :]
|
||||
)
|
||||
else:
|
||||
padded_w13[:, :intermediate, :] = w13
|
||||
|
||||
padded_w2 = w2.new_zeros((num_experts, hidden_size, padded_intermediate))
|
||||
padded_w2[:, :, :intermediate] = w2
|
||||
|
||||
Reference in New Issue
Block a user