[Model] Support Qwen3.8-Flash-Next (#53896)

Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com>
Signed-off-by: andyluo7 <andy.luo@amd.com>
Signed-off-by: lishunyang12 <lishunyang03@gmail.com>
Signed-off-by: Harry Huang <huanghaoyan.hhy@alibaba-inc.com>
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: KungYork <firelikest@163.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: andyluo7 <43718156+andyluo7@users.noreply.github.com>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: andyluo7 <andy.luo@amd.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: lishunyang12 <lishunyang03@gmail.com>
Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Harry Huang
2026-08-31 05:57:56 +00:00
committed by GitHub
co-authored by zjy0516 KungYork Thien Tran Yifan Qiao andyluo7 Isotr0py andyluo7 OpenAI Codex lishunyang12 Vadim Gimpelson Isotr0py Lucas Wilkinson Claude Opus 5 OpenAI Codex
parent 687db59744
commit e126687a9a
124 changed files with 19424 additions and 450 deletions
+8 -3
View File
@@ -57,13 +57,13 @@ EPS = 1e-6
class _TestGatedNorm:
def __init__(self, weight: torch.Tensor) -> None:
def __init__(self, weight: torch.Tensor, activation: str) -> None:
self.weight = weight
self.bias = None
self.eps = EPS
self.group_size = None
self.norm_before_gate = True
self.activation = "silu"
self.activation = activation
def __call__(self, x: torch.Tensor, z: torch.Tensor) -> torch.Tensor:
return rmsnorm_fn(
@@ -99,6 +99,7 @@ def _build_layer(
dt_bias: torch.Tensor,
conv_weight: torch.Tensor,
norm_weight: torch.Tensor,
output_gate_activation: str,
):
layer = types.SimpleNamespace(
prefix=PREFIX,
@@ -116,7 +117,7 @@ def _build_layer(
dt_bias=dt_bias,
conv1d=types.SimpleNamespace(weight=conv_weight, bias=None),
kv_cache=(conv_state, ssm_state),
norm=_TestGatedNorm(norm_weight),
norm=_TestGatedNorm(norm_weight, output_gate_activation),
layer_norm_epsilon=EPS,
gdn_decode_kernel="cuda",
)
@@ -250,12 +251,14 @@ def test_fused_forward_uses_packed_entrypoint() -> None:
pytest.param([128], [1], [-1], 0, id="pure-decode"),
],
)
@pytest.mark.parametrize("output_gate_activation", ["silu", "sigmoid"])
@torch.inference_mode()
def test_fused_model_path_matches_reference(
seq_lens: list[int],
query_lens: list[int],
draft_tokens: list[int],
expected_fused_calls: int,
output_gate_activation: str,
) -> None:
"""Fused MTP and its mixed/prefill/decode fallbacks match the reference."""
torch.manual_seed(1)
@@ -334,6 +337,7 @@ def test_fused_model_path_matches_reference(
dt_bias,
conv_weight,
norm_weight,
output_gate_activation,
)
reference_out = torch.zeros_like(output_gate)
with patch.object(
@@ -355,6 +359,7 @@ def test_fused_model_path_matches_reference(
dt_bias,
conv_weight,
norm_weight,
output_gate_activation,
)
context.no_compile_layers = {PREFIX: fused_layer}
fused_out = torch.zeros_like(output_gate)
@@ -366,7 +366,7 @@ def test_preprocess_fused_align_matches_scalar_bookkeeping(monkeypatch, token_bi
input_batch=scalar_case[1],
requests=scalar_case[2],
forward_context={},
mamba_state_copy_funcs=(),
mamba_state_copy_funcs={},
copy_bufs=scalar_copy_bufs,
)
@@ -384,7 +384,7 @@ def test_preprocess_fused_align_matches_scalar_bookkeeping(monkeypatch, token_bi
input_batch=fused_case[1],
requests=fused_case[2],
forward_context={},
mamba_state_copy_funcs=(),
mamba_state_copy_funcs={},
copy_bufs=fused_copy_bufs,
align_ctx=ctx,
)
+104
View File
@@ -351,3 +351,107 @@ def test_w8a8_block_fp8_deep_gemm_fused_moe(M, N, K, E, topk, seed, monkeypatch)
torch.accelerator.synchronize()
torch.testing.assert_close(out, ref_out, atol=0.035, rtol=0.035)
def _dequant_block_fp8(
w_fp8: torch.Tensor, w_s: torch.Tensor, block_shape: list[int]
) -> torch.Tensor:
"""Dequantize a [N, K] fp8 weight with [Nb, Kb] block scales to fp32."""
n, k = w_fp8.shape
bn, bk = block_shape
s = w_s.repeat_interleave(bn, dim=0).repeat_interleave(bk, dim=1)
return w_fp8.to(torch.float32) * s[:n, :k]
@pytest.mark.parametrize("tp_rank", [0, 1, 2, 3])
@torch.inference_mode()
def test_w8a8_block_fp8_fused_moe_refined_block_scales(tp_rank, workspace_init):
"""TP-misaligned blockwise FP8 MoE (e.g. Qwen4Exp, intermediate
640 at TP=4 -> 160 per rank, not divisible by the checkpoint's 128 block).
The per-rank scale grid is refined from 128 to 32 (lossless: each 32-block
lies inside one global 128-block) so the Triton kernel can consume exact
per-shard scales. Simulates one TP rank's shard and checks:
1. the refined+sharded scales dequantize exactly like the global scales;
2. the Triton fused MoE kernel with block_shape=[32, 32] matches the
native-torch blockwise reference on the same shard.
"""
torch.manual_seed(0)
dtype = torch.bfloat16
M, E, topk = 8, 4, 2
hidden = 256
inter_full = 640
tp_size = 4
n_shard = inter_full // tp_size # 160
ckpt_block = [128, 128]
refined_block = [32, 32]
factor = ckpt_block[0] // refined_block[0] # 4
a = torch.randn((M, hidden), dtype=dtype) / 10
score = torch.randn((M, E), dtype=dtype)
(_, w1_full, w1_s_full, _), (_, w2_full, w2_s_full, _) = make_test_weights(
E,
inter_full,
hidden,
dtype,
torch.float8_e4m3fn,
per_out_ch_quant=False,
block_shape=ckpt_block,
)
assert w1_s_full is not None
assert w2_s_full is not None
# TP shard the weights (exact slicing, no scale involvement).
lo, hi = tp_rank * n_shard, (tp_rank + 1) * n_shard
w1 = torch.cat(
[w1_full[:, lo:hi], w1_full[:, inter_full + lo : inter_full + hi]], dim=1
)
w2 = w2_full[:, :, lo:hi]
# Refine the global 128-block scales to 32 blocks, then take the shard's
# slice -- mirrors the Fp8MoEMethod/RoutedExperts loading path.
w1_s32 = w1_s_full.repeat_interleave(factor, dim=-2).repeat_interleave(
factor, dim=-1
)
nb = n_shard // refined_block[0] # 5 local 32-blocks per projection
gate_hi = w1_s32.shape[1] // 2
w1_s = torch.cat(
[
w1_s32[:, tp_rank * nb : (tp_rank + 1) * nb],
w1_s32[:, gate_hi + tp_rank * nb : gate_hi + (tp_rank + 1) * nb],
],
dim=1,
)
w2_s32 = w2_s_full.repeat_interleave(factor, dim=-2).repeat_interleave(
factor, dim=-1
)
w2_s = w2_s32[:, :, tp_rank * nb : (tp_rank + 1) * nb]
# The refined per-shard scales must reproduce the global-scale dequant
# exactly (dequant full-width, then slice to the shard).
for e in range(E):
d32 = _dequant_block_fp8(w1[e, :n_shard], w1_s[e, :nb], refined_block)
d128 = _dequant_block_fp8(w1_full[e], w1_s_full[e], ckpt_block)[lo:hi]
assert torch.equal(d32, d128)
d32 = _dequant_block_fp8(w2[e], w2_s[e], refined_block)
d128 = _dequant_block_fp8(w2_full[e], w2_s_full[e], ckpt_block)[:, lo:hi]
assert torch.equal(d32, d128)
quant_config = fp8_w8a8_moe_quant_config(
w1_scale=w1_s,
w2_scale=w2_s,
block_shape=refined_block,
)
topk_weights, topk_ids, _ = fused_topk(a, score.float(), topk, False)
with set_current_vllm_config(vllm_config):
ref_out = torch_w8a8_block_fp8_moe(
a, w1, w2, w1_s, w2_s, topk_weights, topk_ids, refined_block
)
out = fused_experts(
a, w1, w2, topk_weights, topk_ids, quant_config=quant_config
)
torch.testing.assert_close(out, ref_out, atol=0.035, rtol=0.035)
+10 -6
View File
@@ -28,8 +28,9 @@ from vllm.model_executor.layers.fused_moe.config import (
from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import (
TrtLlmBf16ExpertsModular,
)
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
convert_moe_weights_to_flashinfer_trtllm_block_layout,
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
UnquantizedMoeBackend,
convert_to_unquantized_kernel_format,
)
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
@@ -46,9 +47,8 @@ if pytest and (
)
# (m, n, k) = (tokens, intermediate_size_per_partition, hidden_dim).
# Covers larger NvFP4-like shapes while keeping BF16's FlashInfer TRTLLM
# intermediate-size multiple-of-128 requirement.
MNK_FACTORS = [
(2, 160, 2560),
(2, 1024, 1024),
(64, 2048, 1536),
(64, 1024, 4096),
@@ -97,11 +97,15 @@ def test_trtllm_bf16_moe_modular_no_graph(
max_num_tokens=next_power_of_2(m),
)
trtllm_w1, trtllm_w2 = convert_moe_weights_to_flashinfer_trtllm_block_layout(
{},
trtllm_w1, trtllm_w2 = convert_to_unquantized_kernel_format(
UnquantizedMoeBackend.FLASHINFER_TRTLLM,
moe_config,
w1,
w2,
)
expected_n = (n + 127) // 128 * 128
assert moe_config.intermediate_size_per_partition == expected_n
assert trtllm_w2.numel() == e * k * expected_n
trtllm_experts = mk.FusedMoEKernel(
maybe_make_prepare_finalize(
+4 -1
View File
@@ -267,6 +267,7 @@ def test_fused_post_conv_l0():
),
],
)
@pytest.mark.parametrize("output_gate_activation", ["silu", "sigmoid"])
@torch.inference_mode()
def test_fused_gdn_decode_post_conv_mtp_head_ratios(
head_ratio: int,
@@ -274,6 +275,7 @@ def test_fused_gdn_decode_post_conv_mtp_head_ratios(
query_lengths: tuple[int, ...],
state_dtype: torch.dtype,
norm_dtype: torch.dtype,
output_gate_activation: str,
) -> None:
if torch.cuda.get_device_capability() < (8, 0):
pytest.skip("fused GDN decode MTP requires compute capability 8.0+")
@@ -357,7 +359,7 @@ def test_fused_gdn_decode_post_conv_mtp_head_ratios(
z=output_gate,
eps=eps,
norm_before_gate=True,
activation="silu",
activation=output_gate_activation,
)
actual = ops.fused_gdn_decode_post_conv_mtp(
mixed_qkv=mixed_qkv,
@@ -374,6 +376,7 @@ def test_fused_gdn_decode_post_conv_mtp_head_ratios(
out=torch.empty_like(output_gate),
scale=scale,
norm_eps=eps,
output_gate_activation=output_gate_activation,
)
output_error = (actual.float() - expected.float()).norm()