[Model] Support Qwen3.8-Flash-Next (#53896)
Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com> Signed-off-by: andyluo7 <andy.luo@amd.com> Signed-off-by: lishunyang12 <lishunyang03@gmail.com> Signed-off-by: Harry Huang <huanghaoyan.hhy@alibaba-inc.com> Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: KungYork <firelikest@163.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: andyluo7 <43718156+andyluo7@users.noreply.github.com> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: andyluo7 <andy.luo@amd.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: lishunyang12 <lishunyang03@gmail.com> Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
zjy0516
KungYork
Thien Tran
Yifan Qiao
andyluo7
Isotr0py
andyluo7
OpenAI Codex
lishunyang12
Vadim Gimpelson
Isotr0py
Lucas Wilkinson
Claude Opus 5
OpenAI Codex
parent
687db59744
commit
e126687a9a
@@ -0,0 +1,14 @@
|
||||
# Qwen4Exp accuracy evaluation
|
||||
|
||||
This suite starts a Qwen3.8-Flash-Next-FP8 OpenAI-compatible server once and
|
||||
uses EvalScope to evaluate GSM8K and AIME25.
|
||||
|
||||
```bash
|
||||
# B200
|
||||
pytest -s -v tests/evals/qwen4_exp/test_accuracy.py \
|
||||
--config-list-file=configs/models-b200.txt
|
||||
|
||||
# H200
|
||||
pytest -s -v tests/evals/qwen4_exp/test_accuracy.py \
|
||||
--config-list-file=configs/models-h200.txt
|
||||
```
|
||||
@@ -0,0 +1,3 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
model_name: "Qwen/Qwen3.8-Flash-Next-FP8"
|
||||
datasets:
|
||||
gsm8k:
|
||||
metric_threshold: 0.98
|
||||
tolerance: 0.02
|
||||
aime25:
|
||||
metric_threshold: 0.90
|
||||
tolerance: 0.05
|
||||
eval_batch_size: 32
|
||||
startup_max_wait_seconds: 1800
|
||||
generation_config:
|
||||
seed: 1236
|
||||
do_sample: true
|
||||
temperature: 0.7
|
||||
top_p: 0.8
|
||||
top_k: 20
|
||||
repetition_penalty: 1.0
|
||||
presence_penalty: 1.5
|
||||
max_tokens: 32768
|
||||
stream: true
|
||||
extra_body:
|
||||
chat_template_kwargs:
|
||||
enable_thinking: true
|
||||
server_args: "--tensor-parallel-size 4"
|
||||
@@ -0,0 +1,2 @@
|
||||
# B200 configuration for Qwen3.8-Flash-Next-FP8 accuracy evaluation
|
||||
Qwen3.8-Flash-Next-FP8.yaml
|
||||
@@ -0,0 +1,2 @@
|
||||
# H200 configuration for Qwen3.8-Flash-Next-FP8 accuracy evaluation
|
||||
Qwen3.8-Flash-Next-FP8.yaml
|
||||
@@ -0,0 +1,42 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def pytest_addoption(parser):
|
||||
parser.addoption(
|
||||
"--config-list-file",
|
||||
required=True,
|
||||
help="File containing Qwen4Exp evaluation config files",
|
||||
)
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc):
|
||||
if "config_filename" not in metafunc.fixturenames:
|
||||
return
|
||||
|
||||
config_list_path = Path(metafunc.config.getoption("--config-list-file"))
|
||||
if not config_list_path.is_absolute():
|
||||
test_relative_path = Path(__file__).parent / config_list_path
|
||||
config_list_path = (
|
||||
test_relative_path
|
||||
if test_relative_path.exists()
|
||||
else Path.cwd() / config_list_path
|
||||
)
|
||||
|
||||
config_dir = config_list_path.parent
|
||||
config_files = [
|
||||
config_dir / line.strip()
|
||||
for line in config_list_path.read_text(encoding="utf-8").splitlines()
|
||||
if line.strip() and not line.lstrip().startswith("#")
|
||||
]
|
||||
missing = [path for path in config_files if not path.exists()]
|
||||
if missing:
|
||||
raise FileNotFoundError(f"Missing evaluation configs: {missing}")
|
||||
|
||||
metafunc.parametrize(
|
||||
"config_filename",
|
||||
config_files,
|
||||
ids=[config_file.stem for config_file in config_files],
|
||||
)
|
||||
@@ -0,0 +1,68 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""GSM8K and AIME25 accuracy evaluation for Qwen3.8-Flash-Next-FP8."""
|
||||
|
||||
import shlex
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
|
||||
def run_evalscope(
|
||||
eval_config: dict[str, Any], base_url: str, work_dir: Path
|
||||
) -> dict[str, float]:
|
||||
from evalscope.run import run_task
|
||||
|
||||
reports = run_task(
|
||||
{
|
||||
"model": eval_config["model_name"],
|
||||
"api_url": base_url,
|
||||
"api_key": "EMPTY_TOKEN",
|
||||
"datasets": list(eval_config["datasets"]),
|
||||
"eval_batch_size": eval_config.get("eval_batch_size", 32),
|
||||
"generation_config": eval_config["generation_config"],
|
||||
"work_dir": str(work_dir),
|
||||
"no_timestamp": True,
|
||||
}
|
||||
)
|
||||
if not isinstance(reports, dict):
|
||||
raise TypeError(f"Expected EvalScope reports to be a dict, got {type(reports)}")
|
||||
|
||||
return {dataset: float(report.score) for dataset, report in reports.items()}
|
||||
|
||||
|
||||
def test_qwen4_exp_accuracy(config_filename: Path, tmp_path: Path):
|
||||
eval_config = yaml.safe_load(config_filename.read_text(encoding="utf-8"))
|
||||
server_args = shlex.split(eval_config.get("server_args", ""))
|
||||
server_args.extend(["--trust-remote-code", "--disable-uvicorn-access-log"])
|
||||
|
||||
model_name = eval_config["model_name"]
|
||||
print(f"Starting Qwen4Exp evaluation for model: {model_name}")
|
||||
print(f"Datasets: {', '.join(eval_config['datasets'])}")
|
||||
print(f"Server args: {' '.join(server_args)}")
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
model_name,
|
||||
server_args,
|
||||
env_dict=eval_config.get("env"),
|
||||
max_wait_seconds=eval_config.get("startup_max_wait_seconds", 1800),
|
||||
) as remote_server:
|
||||
scores = run_evalscope(eval_config, remote_server.url_for("v1"), tmp_path)
|
||||
|
||||
for dataset, metric_config in eval_config["datasets"].items():
|
||||
score = scores[dataset]
|
||||
threshold = metric_config["metric_threshold"]
|
||||
tolerance = metric_config.get("tolerance", 0.0)
|
||||
minimum_score = threshold - tolerance
|
||||
|
||||
print(
|
||||
f"{dataset}: measured={score:.4f}, expected={threshold:.4f}, "
|
||||
f"tolerance={tolerance:.4f}"
|
||||
)
|
||||
assert score >= minimum_score, (
|
||||
f"{dataset} score too low: {score:.4f} < {threshold:.4f} - "
|
||||
f"{tolerance:.4f} = {minimum_score:.4f}"
|
||||
)
|
||||
Reference in New Issue
Block a user