[Model] Support Qwen3.8-Flash-Next (#53896)
Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com> Signed-off-by: andyluo7 <andy.luo@amd.com> Signed-off-by: lishunyang12 <lishunyang03@gmail.com> Signed-off-by: Harry Huang <huanghaoyan.hhy@alibaba-inc.com> Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: KungYork <firelikest@163.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: andyluo7 <43718156+andyluo7@users.noreply.github.com> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: andyluo7 <andy.luo@amd.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: lishunyang12 <lishunyang03@gmail.com> Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
zjy0516
KungYork
Thien Tran
Yifan Qiao
andyluo7
Isotr0py
andyluo7
OpenAI Codex
lishunyang12
Vadim Gimpelson
Isotr0py
Lucas Wilkinson
Claude Opus 5
OpenAI Codex
parent
687db59744
commit
e126687a9a
@@ -57,13 +57,13 @@ EPS = 1e-6
|
||||
|
||||
|
||||
class _TestGatedNorm:
|
||||
def __init__(self, weight: torch.Tensor) -> None:
|
||||
def __init__(self, weight: torch.Tensor, activation: str) -> None:
|
||||
self.weight = weight
|
||||
self.bias = None
|
||||
self.eps = EPS
|
||||
self.group_size = None
|
||||
self.norm_before_gate = True
|
||||
self.activation = "silu"
|
||||
self.activation = activation
|
||||
|
||||
def __call__(self, x: torch.Tensor, z: torch.Tensor) -> torch.Tensor:
|
||||
return rmsnorm_fn(
|
||||
@@ -99,6 +99,7 @@ def _build_layer(
|
||||
dt_bias: torch.Tensor,
|
||||
conv_weight: torch.Tensor,
|
||||
norm_weight: torch.Tensor,
|
||||
output_gate_activation: str,
|
||||
):
|
||||
layer = types.SimpleNamespace(
|
||||
prefix=PREFIX,
|
||||
@@ -116,7 +117,7 @@ def _build_layer(
|
||||
dt_bias=dt_bias,
|
||||
conv1d=types.SimpleNamespace(weight=conv_weight, bias=None),
|
||||
kv_cache=(conv_state, ssm_state),
|
||||
norm=_TestGatedNorm(norm_weight),
|
||||
norm=_TestGatedNorm(norm_weight, output_gate_activation),
|
||||
layer_norm_epsilon=EPS,
|
||||
gdn_decode_kernel="cuda",
|
||||
)
|
||||
@@ -250,12 +251,14 @@ def test_fused_forward_uses_packed_entrypoint() -> None:
|
||||
pytest.param([128], [1], [-1], 0, id="pure-decode"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("output_gate_activation", ["silu", "sigmoid"])
|
||||
@torch.inference_mode()
|
||||
def test_fused_model_path_matches_reference(
|
||||
seq_lens: list[int],
|
||||
query_lens: list[int],
|
||||
draft_tokens: list[int],
|
||||
expected_fused_calls: int,
|
||||
output_gate_activation: str,
|
||||
) -> None:
|
||||
"""Fused MTP and its mixed/prefill/decode fallbacks match the reference."""
|
||||
torch.manual_seed(1)
|
||||
@@ -334,6 +337,7 @@ def test_fused_model_path_matches_reference(
|
||||
dt_bias,
|
||||
conv_weight,
|
||||
norm_weight,
|
||||
output_gate_activation,
|
||||
)
|
||||
reference_out = torch.zeros_like(output_gate)
|
||||
with patch.object(
|
||||
@@ -355,6 +359,7 @@ def test_fused_model_path_matches_reference(
|
||||
dt_bias,
|
||||
conv_weight,
|
||||
norm_weight,
|
||||
output_gate_activation,
|
||||
)
|
||||
context.no_compile_layers = {PREFIX: fused_layer}
|
||||
fused_out = torch.zeros_like(output_gate)
|
||||
|
||||
@@ -366,7 +366,7 @@ def test_preprocess_fused_align_matches_scalar_bookkeeping(monkeypatch, token_bi
|
||||
input_batch=scalar_case[1],
|
||||
requests=scalar_case[2],
|
||||
forward_context={},
|
||||
mamba_state_copy_funcs=(),
|
||||
mamba_state_copy_funcs={},
|
||||
copy_bufs=scalar_copy_bufs,
|
||||
)
|
||||
|
||||
@@ -384,7 +384,7 @@ def test_preprocess_fused_align_matches_scalar_bookkeeping(monkeypatch, token_bi
|
||||
input_batch=fused_case[1],
|
||||
requests=fused_case[2],
|
||||
forward_context={},
|
||||
mamba_state_copy_funcs=(),
|
||||
mamba_state_copy_funcs={},
|
||||
copy_bufs=fused_copy_bufs,
|
||||
align_ctx=ctx,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user