[Model] Support Qwen3.8-Flash-Next (#53896)

Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com>
Signed-off-by: andyluo7 <andy.luo@amd.com>
Signed-off-by: lishunyang12 <lishunyang03@gmail.com>
Signed-off-by: Harry Huang <huanghaoyan.hhy@alibaba-inc.com>
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: KungYork <firelikest@163.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: andyluo7 <43718156+andyluo7@users.noreply.github.com>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: andyluo7 <andy.luo@amd.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: lishunyang12 <lishunyang03@gmail.com>
Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Harry Huang
2026-08-31 05:57:56 +00:00
committed by GitHub
co-authored by zjy0516 KungYork Thien Tran Yifan Qiao andyluo7 Isotr0py andyluo7 OpenAI Codex lishunyang12 Vadim Gimpelson Isotr0py Lucas Wilkinson Claude Opus 5 OpenAI Codex
parent 687db59744
commit e126687a9a
124 changed files with 19424 additions and 450 deletions
+14
View File
@@ -0,0 +1,14 @@
# Qwen4Exp accuracy evaluation
This suite starts a Qwen3.8-Flash-Next-FP8 OpenAI-compatible server once and
uses EvalScope to evaluate GSM8K and AIME25.
```bash
# B200
pytest -s -v tests/evals/qwen4_exp/test_accuracy.py \
--config-list-file=configs/models-b200.txt
# H200
pytest -s -v tests/evals/qwen4_exp/test_accuracy.py \
--config-list-file=configs/models-h200.txt
```
+3
View File
@@ -0,0 +1,3 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,26 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
model_name: "Qwen/Qwen3.8-Flash-Next-FP8"
datasets:
gsm8k:
metric_threshold: 0.98
tolerance: 0.02
aime25:
metric_threshold: 0.90
tolerance: 0.05
eval_batch_size: 32
startup_max_wait_seconds: 1800
generation_config:
seed: 1236
do_sample: true
temperature: 0.7
top_p: 0.8
top_k: 20
repetition_penalty: 1.0
presence_penalty: 1.5
max_tokens: 32768
stream: true
extra_body:
chat_template_kwargs:
enable_thinking: true
server_args: "--tensor-parallel-size 4"
@@ -0,0 +1,2 @@
# B200 configuration for Qwen3.8-Flash-Next-FP8 accuracy evaluation
Qwen3.8-Flash-Next-FP8.yaml
@@ -0,0 +1,2 @@
# H200 configuration for Qwen3.8-Flash-Next-FP8 accuracy evaluation
Qwen3.8-Flash-Next-FP8.yaml
+42
View File
@@ -0,0 +1,42 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from pathlib import Path
def pytest_addoption(parser):
parser.addoption(
"--config-list-file",
required=True,
help="File containing Qwen4Exp evaluation config files",
)
def pytest_generate_tests(metafunc):
if "config_filename" not in metafunc.fixturenames:
return
config_list_path = Path(metafunc.config.getoption("--config-list-file"))
if not config_list_path.is_absolute():
test_relative_path = Path(__file__).parent / config_list_path
config_list_path = (
test_relative_path
if test_relative_path.exists()
else Path.cwd() / config_list_path
)
config_dir = config_list_path.parent
config_files = [
config_dir / line.strip()
for line in config_list_path.read_text(encoding="utf-8").splitlines()
if line.strip() and not line.lstrip().startswith("#")
]
missing = [path for path in config_files if not path.exists()]
if missing:
raise FileNotFoundError(f"Missing evaluation configs: {missing}")
metafunc.parametrize(
"config_filename",
config_files,
ids=[config_file.stem for config_file in config_files],
)
+68
View File
@@ -0,0 +1,68 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""GSM8K and AIME25 accuracy evaluation for Qwen3.8-Flash-Next-FP8."""
import shlex
from pathlib import Path
from typing import Any
import yaml
from tests.utils import RemoteOpenAIServer
def run_evalscope(
eval_config: dict[str, Any], base_url: str, work_dir: Path
) -> dict[str, float]:
from evalscope.run import run_task
reports = run_task(
{
"model": eval_config["model_name"],
"api_url": base_url,
"api_key": "EMPTY_TOKEN",
"datasets": list(eval_config["datasets"]),
"eval_batch_size": eval_config.get("eval_batch_size", 32),
"generation_config": eval_config["generation_config"],
"work_dir": str(work_dir),
"no_timestamp": True,
}
)
if not isinstance(reports, dict):
raise TypeError(f"Expected EvalScope reports to be a dict, got {type(reports)}")
return {dataset: float(report.score) for dataset, report in reports.items()}
def test_qwen4_exp_accuracy(config_filename: Path, tmp_path: Path):
eval_config = yaml.safe_load(config_filename.read_text(encoding="utf-8"))
server_args = shlex.split(eval_config.get("server_args", ""))
server_args.extend(["--trust-remote-code", "--disable-uvicorn-access-log"])
model_name = eval_config["model_name"]
print(f"Starting Qwen4Exp evaluation for model: {model_name}")
print(f"Datasets: {', '.join(eval_config['datasets'])}")
print(f"Server args: {' '.join(server_args)}")
with RemoteOpenAIServer(
model_name,
server_args,
env_dict=eval_config.get("env"),
max_wait_seconds=eval_config.get("startup_max_wait_seconds", 1800),
) as remote_server:
scores = run_evalscope(eval_config, remote_server.url_for("v1"), tmp_path)
for dataset, metric_config in eval_config["datasets"].items():
score = scores[dataset]
threshold = metric_config["metric_threshold"]
tolerance = metric_config.get("tolerance", 0.0)
minimum_score = threshold - tolerance
print(
f"{dataset}: measured={score:.4f}, expected={threshold:.4f}, "
f"tolerance={tolerance:.4f}"
)
assert score >= minimum_score, (
f"{dataset} score too low: {score:.4f} < {threshold:.4f} - "
f"{tolerance:.4f} = {minimum_score:.4f}"
)