[Model] Support Qwen3.8-Flash-Next (#53896)

Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com>
Signed-off-by: andyluo7 <andy.luo@amd.com>
Signed-off-by: lishunyang12 <lishunyang03@gmail.com>
Signed-off-by: Harry Huang <huanghaoyan.hhy@alibaba-inc.com>
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: KungYork <firelikest@163.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: andyluo7 <43718156+andyluo7@users.noreply.github.com>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: andyluo7 <andy.luo@amd.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: lishunyang12 <lishunyang03@gmail.com>
Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Harry Huang
2026-08-31 05:57:56 +00:00
committed by GitHub
co-authored by zjy0516 KungYork Thien Tran Yifan Qiao andyluo7 Isotr0py andyluo7 OpenAI Codex lishunyang12 Vadim Gimpelson Isotr0py Lucas Wilkinson Claude Opus 5 OpenAI Codex
parent 687db59744
commit e126687a9a
124 changed files with 19424 additions and 450 deletions
+104
View File
@@ -351,3 +351,107 @@ def test_w8a8_block_fp8_deep_gemm_fused_moe(M, N, K, E, topk, seed, monkeypatch)
torch.accelerator.synchronize()
torch.testing.assert_close(out, ref_out, atol=0.035, rtol=0.035)
def _dequant_block_fp8(
w_fp8: torch.Tensor, w_s: torch.Tensor, block_shape: list[int]
) -> torch.Tensor:
"""Dequantize a [N, K] fp8 weight with [Nb, Kb] block scales to fp32."""
n, k = w_fp8.shape
bn, bk = block_shape
s = w_s.repeat_interleave(bn, dim=0).repeat_interleave(bk, dim=1)
return w_fp8.to(torch.float32) * s[:n, :k]
@pytest.mark.parametrize("tp_rank", [0, 1, 2, 3])
@torch.inference_mode()
def test_w8a8_block_fp8_fused_moe_refined_block_scales(tp_rank, workspace_init):
"""TP-misaligned blockwise FP8 MoE (e.g. Qwen4Exp, intermediate
640 at TP=4 -> 160 per rank, not divisible by the checkpoint's 128 block).
The per-rank scale grid is refined from 128 to 32 (lossless: each 32-block
lies inside one global 128-block) so the Triton kernel can consume exact
per-shard scales. Simulates one TP rank's shard and checks:
1. the refined+sharded scales dequantize exactly like the global scales;
2. the Triton fused MoE kernel with block_shape=[32, 32] matches the
native-torch blockwise reference on the same shard.
"""
torch.manual_seed(0)
dtype = torch.bfloat16
M, E, topk = 8, 4, 2
hidden = 256
inter_full = 640
tp_size = 4
n_shard = inter_full // tp_size # 160
ckpt_block = [128, 128]
refined_block = [32, 32]
factor = ckpt_block[0] // refined_block[0] # 4
a = torch.randn((M, hidden), dtype=dtype) / 10
score = torch.randn((M, E), dtype=dtype)
(_, w1_full, w1_s_full, _), (_, w2_full, w2_s_full, _) = make_test_weights(
E,
inter_full,
hidden,
dtype,
torch.float8_e4m3fn,
per_out_ch_quant=False,
block_shape=ckpt_block,
)
assert w1_s_full is not None
assert w2_s_full is not None
# TP shard the weights (exact slicing, no scale involvement).
lo, hi = tp_rank * n_shard, (tp_rank + 1) * n_shard
w1 = torch.cat(
[w1_full[:, lo:hi], w1_full[:, inter_full + lo : inter_full + hi]], dim=1
)
w2 = w2_full[:, :, lo:hi]
# Refine the global 128-block scales to 32 blocks, then take the shard's
# slice -- mirrors the Fp8MoEMethod/RoutedExperts loading path.
w1_s32 = w1_s_full.repeat_interleave(factor, dim=-2).repeat_interleave(
factor, dim=-1
)
nb = n_shard // refined_block[0] # 5 local 32-blocks per projection
gate_hi = w1_s32.shape[1] // 2
w1_s = torch.cat(
[
w1_s32[:, tp_rank * nb : (tp_rank + 1) * nb],
w1_s32[:, gate_hi + tp_rank * nb : gate_hi + (tp_rank + 1) * nb],
],
dim=1,
)
w2_s32 = w2_s_full.repeat_interleave(factor, dim=-2).repeat_interleave(
factor, dim=-1
)
w2_s = w2_s32[:, :, tp_rank * nb : (tp_rank + 1) * nb]
# The refined per-shard scales must reproduce the global-scale dequant
# exactly (dequant full-width, then slice to the shard).
for e in range(E):
d32 = _dequant_block_fp8(w1[e, :n_shard], w1_s[e, :nb], refined_block)
d128 = _dequant_block_fp8(w1_full[e], w1_s_full[e], ckpt_block)[lo:hi]
assert torch.equal(d32, d128)
d32 = _dequant_block_fp8(w2[e], w2_s[e], refined_block)
d128 = _dequant_block_fp8(w2_full[e], w2_s_full[e], ckpt_block)[:, lo:hi]
assert torch.equal(d32, d128)
quant_config = fp8_w8a8_moe_quant_config(
w1_scale=w1_s,
w2_scale=w2_s,
block_shape=refined_block,
)
topk_weights, topk_ids, _ = fused_topk(a, score.float(), topk, False)
with set_current_vllm_config(vllm_config):
ref_out = torch_w8a8_block_fp8_moe(
a, w1, w2, w1_s, w2_s, topk_weights, topk_ids, refined_block
)
out = fused_experts(
a, w1, w2, topk_weights, topk_ids, quant_config=quant_config
)
torch.testing.assert_close(out, ref_out, atol=0.035, rtol=0.035)
+10 -6
View File
@@ -28,8 +28,9 @@ from vllm.model_executor.layers.fused_moe.config import (
from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import (
TrtLlmBf16ExpertsModular,
)
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
convert_moe_weights_to_flashinfer_trtllm_block_layout,
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
UnquantizedMoeBackend,
convert_to_unquantized_kernel_format,
)
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
@@ -46,9 +47,8 @@ if pytest and (
)
# (m, n, k) = (tokens, intermediate_size_per_partition, hidden_dim).
# Covers larger NvFP4-like shapes while keeping BF16's FlashInfer TRTLLM
# intermediate-size multiple-of-128 requirement.
MNK_FACTORS = [
(2, 160, 2560),
(2, 1024, 1024),
(64, 2048, 1536),
(64, 1024, 4096),
@@ -97,11 +97,15 @@ def test_trtllm_bf16_moe_modular_no_graph(
max_num_tokens=next_power_of_2(m),
)
trtllm_w1, trtllm_w2 = convert_moe_weights_to_flashinfer_trtllm_block_layout(
{},
trtllm_w1, trtllm_w2 = convert_to_unquantized_kernel_format(
UnquantizedMoeBackend.FLASHINFER_TRTLLM,
moe_config,
w1,
w2,
)
expected_n = (n + 127) // 128 * 128
assert moe_config.intermediate_size_per_partition == expected_n
assert trtllm_w2.numel() == e * k * expected_n
trtllm_experts = mk.FusedMoEKernel(
maybe_make_prepare_finalize(