[Model] Support Qwen3.8-Flash-Next (#53896)
Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com> Signed-off-by: andyluo7 <andy.luo@amd.com> Signed-off-by: lishunyang12 <lishunyang03@gmail.com> Signed-off-by: Harry Huang <huanghaoyan.hhy@alibaba-inc.com> Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: KungYork <firelikest@163.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: andyluo7 <43718156+andyluo7@users.noreply.github.com> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: andyluo7 <andy.luo@amd.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: lishunyang12 <lishunyang03@gmail.com> Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
zjy0516
KungYork
Thien Tran
Yifan Qiao
andyluo7
Isotr0py
andyluo7
OpenAI Codex
lishunyang12
Vadim Gimpelson
Isotr0py
Lucas Wilkinson
Claude Opus 5
OpenAI Codex
parent
687db59744
commit
e126687a9a
@@ -6,6 +6,7 @@ Run `pytest tests/quantization/test_fp8.py --forked`.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
@@ -31,6 +32,10 @@ from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import (
|
||||
Fp8PerTensorOnlineLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils import flashinfer_utils
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
prepare_fp8_moe_layer_for_fi,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
process_fp8_input_tensor_strategy_moe,
|
||||
)
|
||||
@@ -50,6 +55,45 @@ MODELS = [
|
||||
]
|
||||
|
||||
|
||||
def test_prepare_gated_trtllm_fp8_moe_weights_pads_each_projection(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
flashinfer_utils,
|
||||
"rotate_weights_for_fi_trtllm_fp8_per_tensor_moe",
|
||||
lambda *args: None,
|
||||
)
|
||||
intermediate = 17
|
||||
padded_intermediate = 32
|
||||
hidden_size = 4
|
||||
gate = torch.ones((1, intermediate, hidden_size), dtype=torch.float8_e4m3fn)
|
||||
up = torch.full_like(gate, 2)
|
||||
w13 = torch.cat((gate, up), dim=1)
|
||||
w2 = torch.ones((1, hidden_size, intermediate), dtype=torch.float8_e4m3fn)
|
||||
layer = SimpleNamespace(
|
||||
activation=SimpleNamespace(is_gated=True),
|
||||
moe_config=SimpleNamespace(
|
||||
is_act_and_mul=True,
|
||||
intermediate_size_per_partition=intermediate,
|
||||
),
|
||||
)
|
||||
|
||||
padded_w31, _, _, _ = prepare_fp8_moe_layer_for_fi(
|
||||
layer,
|
||||
w13,
|
||||
w2,
|
||||
w13_scale=torch.ones(1),
|
||||
w13_input_scale=torch.ones(1),
|
||||
w2_scale=torch.ones(1),
|
||||
w2_input_scale=torch.ones(1),
|
||||
is_trtllm=True,
|
||||
)
|
||||
|
||||
expected = w13.new_zeros((1, 2 * padded_intermediate, hidden_size))
|
||||
expected[:, :intermediate] = up
|
||||
expected[:, padded_intermediate : padded_intermediate + intermediate] = gate
|
||||
assert layer.moe_config.intermediate_size_per_partition == padded_intermediate
|
||||
assert torch.equal(padded_w31, expected)
|
||||
|
||||
|
||||
def test_static_fp8_moe_input_scales_remain_scalar() -> None:
|
||||
a1_scale, a2_scale = process_fp8_input_tensor_strategy_moe(
|
||||
torch.tensor([0.25, 0.5]),
|
||||
|
||||
@@ -16,6 +16,7 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
Mxfp4MoeBackend,
|
||||
_requires_qwen38_tep8_emulation,
|
||||
select_mxfp4_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
@@ -72,6 +73,91 @@ def _make_w4a4_moe_config(moe_backend: str = "auto") -> FusedMoEConfig:
|
||||
)
|
||||
|
||||
|
||||
def _make_qwen38_tep8_moe_config() -> FusedMoEConfig:
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
|
||||
return FusedMoEConfig(
|
||||
num_experts=512,
|
||||
experts_per_token=9,
|
||||
hidden_dim=2560,
|
||||
intermediate_size=640,
|
||||
num_local_experts=64,
|
||||
num_logical_experts=512,
|
||||
moe_parallel_config=FusedMoEParallelConfig(
|
||||
tp_size=1,
|
||||
tp_rank=0,
|
||||
pcp_size=1,
|
||||
pcp_rank=0,
|
||||
dp_size=1,
|
||||
dp_rank=0,
|
||||
ep_size=8,
|
||||
ep_rank=0,
|
||||
sp_size=1,
|
||||
use_ep=True,
|
||||
all2all_backend="allgather_reducescatter",
|
||||
enable_eplb=False,
|
||||
),
|
||||
activation=MoEActivation.SILU,
|
||||
in_dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
routing_method=RoutingMethodType.Renormalize,
|
||||
)
|
||||
|
||||
|
||||
def test_qwen38_tep8_requires_emulation_only_on_gfx950(monkeypatch):
|
||||
import vllm.model_executor.layers.fused_moe.oracle.mxfp4 as mxfp4_oracle
|
||||
|
||||
config = _make_qwen38_tep8_moe_config()
|
||||
monkeypatch.setattr(current_platform, "is_rocm", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: True)
|
||||
|
||||
assert _requires_qwen38_tep8_emulation(config, kMxfp4Dynamic)
|
||||
|
||||
config.moe_parallel_config.ep_size = 4
|
||||
config.num_local_experts = 128
|
||||
assert not _requires_qwen38_tep8_emulation(config, kMxfp4Dynamic)
|
||||
|
||||
config = _make_qwen38_tep8_moe_config()
|
||||
monkeypatch.setattr(mxfp4_oracle.current_platform, "is_rocm", lambda: False)
|
||||
assert not _requires_qwen38_tep8_emulation(config, kMxfp4Dynamic)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"requested_backend,expected_backend",
|
||||
[
|
||||
("auto", Mxfp4MoeBackend.EMULATION),
|
||||
("aiter", Mxfp4MoeBackend.AITER_MXFP4_MXFP4),
|
||||
],
|
||||
)
|
||||
def test_qwen38_tep8_auto_fallback_respects_explicit_backend(
|
||||
requested_backend,
|
||||
expected_backend,
|
||||
monkeypatch,
|
||||
):
|
||||
import vllm.model_executor.layers.fused_moe.oracle.mxfp4 as mxfp4_oracle
|
||||
|
||||
class SupportedExperts:
|
||||
@staticmethod
|
||||
def is_supported_config(*args, **kwargs):
|
||||
return True, None
|
||||
|
||||
config = _make_qwen38_tep8_moe_config()
|
||||
config.moe_backend = requested_backend
|
||||
monkeypatch.setattr(current_platform, "is_rocm", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: True)
|
||||
monkeypatch.setattr(mxfp4_oracle, "_user_moe_activation_override", lambda: None)
|
||||
monkeypatch.setattr(
|
||||
mxfp4_oracle, "backend_to_kernel_cls", lambda backend: [SupportedExperts]
|
||||
)
|
||||
|
||||
backend, experts_cls = select_mxfp4_moe_backend(
|
||||
config, activation_key=kMxfp4Dynamic
|
||||
)
|
||||
|
||||
assert backend == expected_backend
|
||||
assert experts_cls is SupportedExperts
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mxfp4_oracle_config():
|
||||
"""Stub the config the oracle reads (``model_config.quantization_config``)
|
||||
|
||||
@@ -604,3 +604,24 @@ def test_online_mxfp4_dense_matches_quark(
|
||||
keys=("weight", "weight_scale"),
|
||||
packed_weight_keys=("weight",),
|
||||
)
|
||||
|
||||
|
||||
def test_quark_ocp_mx_dense_rejects_unaligned_input_partition():
|
||||
scheme = object.__new__(QuarkOCP_MX)
|
||||
layer = torch.nn.Module()
|
||||
layer.prefix = "model.layers.0.mlp.shared_expert.down_proj"
|
||||
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match=(
|
||||
"input size per partition of 80, which must be divisible by "
|
||||
"the OCP MX group size 32"
|
||||
),
|
||||
):
|
||||
scheme.create_weights(
|
||||
layer=layer,
|
||||
output_partition_sizes=[2560],
|
||||
input_size_per_partition=80,
|
||||
params_dtype=torch.bfloat16,
|
||||
weight_loader=lambda *_: None,
|
||||
)
|
||||
|
||||
@@ -11,6 +11,7 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
|
||||
prepare_nvfp4_moe_layer_for_fi_or_cutlass,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
align_fp4_moe_weights_for_fi,
|
||||
align_trtllm_fp4_moe_hidden_dim_for_fi,
|
||||
)
|
||||
|
||||
@@ -107,3 +108,53 @@ def test_align_trtllm_fp4_moe_hidden_dim_pads_to_256_multiple():
|
||||
assert torch.count_nonzero(out_w13_scale[:, :, hidden_dim // 16 :]) == 0
|
||||
assert torch.count_nonzero(out_w2[:, hidden_dim:, :]) == 0
|
||||
assert torch.count_nonzero(out_w2_scale[:, hidden_dim:, :]) == 0
|
||||
|
||||
|
||||
def test_align_trtllm_fp4_moe_intermediate_pads_gate_and_up_separately():
|
||||
# Qwen3.8-Flash-Next has intermediate_size=640. At TP4 each rank owns
|
||||
# 160 values, which TRTLLM pads to the next 64-value boundary (192).
|
||||
intermediate = 160
|
||||
padded_intermediate = 192
|
||||
hidden_dim = 32
|
||||
|
||||
gate = torch.ones((1, intermediate, hidden_dim // 2), dtype=torch.uint8)
|
||||
up = torch.full_like(gate, 2)
|
||||
w13 = torch.cat((gate, up), dim=1)
|
||||
gate_scale = torch.full((1, intermediate, hidden_dim // 16), 3, dtype=torch.uint8)
|
||||
up_scale = torch.full_like(gate_scale, 4)
|
||||
w13_scale = torch.cat((gate_scale, up_scale), dim=1)
|
||||
w2 = torch.full((1, hidden_dim, intermediate // 2), 5, dtype=torch.uint8)
|
||||
w2_scale = torch.full((1, hidden_dim, intermediate // 16), 6, dtype=torch.uint8)
|
||||
|
||||
out_w13, out_w13_scale, out_w2, out_w2_scale, out_intermediate = (
|
||||
align_fp4_moe_weights_for_fi(
|
||||
w13,
|
||||
w13_scale,
|
||||
w2,
|
||||
w2_scale,
|
||||
is_act_and_mul=True,
|
||||
min_alignment=64,
|
||||
)
|
||||
)
|
||||
|
||||
assert out_intermediate == padded_intermediate
|
||||
torch.testing.assert_close(out_w13[:, :intermediate], gate)
|
||||
torch.testing.assert_close(
|
||||
out_w13[:, padded_intermediate : padded_intermediate + intermediate], up
|
||||
)
|
||||
torch.testing.assert_close(out_w13_scale[:, :intermediate], gate_scale)
|
||||
torch.testing.assert_close(
|
||||
out_w13_scale[:, padded_intermediate : padded_intermediate + intermediate],
|
||||
up_scale,
|
||||
)
|
||||
|
||||
assert torch.count_nonzero(out_w13[:, intermediate:padded_intermediate]) == 0
|
||||
assert torch.count_nonzero(out_w13[:, padded_intermediate + intermediate :]) == 0
|
||||
assert torch.count_nonzero(out_w13_scale[:, intermediate:padded_intermediate]) == 0
|
||||
assert (
|
||||
torch.count_nonzero(out_w13_scale[:, padded_intermediate + intermediate :]) == 0
|
||||
)
|
||||
torch.testing.assert_close(out_w2[:, :, : intermediate // 2], w2)
|
||||
torch.testing.assert_close(out_w2_scale[:, :, : intermediate // 16], w2_scale)
|
||||
assert torch.count_nonzero(out_w2[:, :, intermediate // 2 :]) == 0
|
||||
assert torch.count_nonzero(out_w2_scale[:, :, intermediate // 16 :]) == 0
|
||||
|
||||
Reference in New Issue
Block a user