[Model] Support Qwen3.8-Flash-Next (#53896)
Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com> Signed-off-by: andyluo7 <andy.luo@amd.com> Signed-off-by: lishunyang12 <lishunyang03@gmail.com> Signed-off-by: Harry Huang <huanghaoyan.hhy@alibaba-inc.com> Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: KungYork <firelikest@163.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: andyluo7 <43718156+andyluo7@users.noreply.github.com> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: andyluo7 <andy.luo@amd.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: lishunyang12 <lishunyang03@gmail.com> Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
zjy0516
KungYork
Thien Tran
Yifan Qiao
andyluo7
Isotr0py
andyluo7
OpenAI Codex
lishunyang12
Vadim Gimpelson
Isotr0py
Lucas Wilkinson
Claude Opus 5
OpenAI Codex
parent
687db59744
commit
e126687a9a
@@ -18,8 +18,12 @@ from vllm.model_executor.layers.fused_moe.layer import determine_expert_counts
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
|
||||
from vllm.model_executor.layers.quantization.utils.config_utils import (
|
||||
get_quark_ocp_mx_group_size,
|
||||
is_shared_expert_quant_fse_compatible,
|
||||
)
|
||||
from vllm.model_executor.models.qwen3_next import (
|
||||
_should_replicate_misaligned_shared_expert,
|
||||
)
|
||||
from vllm.model_executor.models.utils import PPMissingLayer
|
||||
from vllm.models.deepseek_v4 import quant_config as deepseek_v4_quant_config
|
||||
from vllm.models.minimax_m3.amd import model as minimax_m3_model
|
||||
@@ -605,6 +609,82 @@ def test_quark_shared_expert_fse_compatibility(
|
||||
)
|
||||
|
||||
|
||||
def test_quark_ocp_mx_group_size_for_layer() -> None:
|
||||
quant_config = QuarkConfig({**_QUARK_FSE_CONFIG, "exclude": []})
|
||||
assert (
|
||||
get_quark_ocp_mx_group_size(
|
||||
quant_config,
|
||||
"model.layers.0.mlp.shared_expert.down_proj",
|
||||
)
|
||||
== 32
|
||||
)
|
||||
|
||||
quant_config.quant_config["exclude"] = ["model.layers.0.mlp.shared_expert_gate"]
|
||||
assert (
|
||||
get_quark_ocp_mx_group_size(
|
||||
quant_config,
|
||||
"model.layers.0.mlp.shared_expert.down_proj",
|
||||
)
|
||||
== 32
|
||||
)
|
||||
|
||||
quant_config.quant_config["exclude"] = [
|
||||
"model.layers.0.mlp.shared_expert.down_proj"
|
||||
]
|
||||
assert (
|
||||
get_quark_ocp_mx_group_size(
|
||||
quant_config,
|
||||
"model.layers.0.mlp.shared_expert.down_proj",
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size", [1, 2, 4])
|
||||
def test_quark_ocp_mx_shared_expert_uses_supported_tp_shards(tp_size: int) -> None:
|
||||
assert not _should_replicate_misaligned_shared_expert(
|
||||
intermediate_size=640,
|
||||
tp_size=tp_size,
|
||||
group_size=32,
|
||||
enable_expert_parallel=False,
|
||||
is_sequence_parallel=False,
|
||||
)
|
||||
|
||||
|
||||
def test_quark_ocp_mx_shared_expert_rejects_plain_tp8() -> None:
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="TP size 8 produces a partition of 80",
|
||||
):
|
||||
_should_replicate_misaligned_shared_expert(
|
||||
intermediate_size=640,
|
||||
tp_size=8,
|
||||
group_size=32,
|
||||
enable_expert_parallel=False,
|
||||
is_sequence_parallel=False,
|
||||
)
|
||||
|
||||
|
||||
def test_quark_ocp_mx_shared_expert_is_replicated_for_tep8() -> None:
|
||||
assert _should_replicate_misaligned_shared_expert(
|
||||
intermediate_size=640,
|
||||
tp_size=8,
|
||||
group_size=32,
|
||||
enable_expert_parallel=True,
|
||||
is_sequence_parallel=False,
|
||||
)
|
||||
|
||||
|
||||
def test_quark_ocp_mx_shared_expert_is_replicated_when_not_tp_divisible() -> None:
|
||||
assert _should_replicate_misaligned_shared_expert(
|
||||
intermediate_size=672,
|
||||
tp_size=8,
|
||||
group_size=32,
|
||||
enable_expert_parallel=True,
|
||||
is_sequence_parallel=False,
|
||||
)
|
||||
|
||||
|
||||
def test_quark_shared_expert_fse_requires_matching_layer_quant_configs() -> None:
|
||||
global_quant_config = {"weight": {"dtype": "fp4"}}
|
||||
quant_config = QuarkConfig(
|
||||
|
||||
Reference in New Issue
Block a user