[MoE Refactor] MXFP4 Cutlass Experts to MK (#34542)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
Yongye Zhu
2026-02-25 17:32:39 -08:00
committed by GitHub
parent cbf8f7028c
commit 1976356ee6
19 changed files with 454 additions and 169 deletions
+49
View File
@@ -0,0 +1,49 @@
# GPQA Evaluation using GPT-OSS
This directory contains GPQA evaluation tests using the GPT-OSS evaluation package and vLLM server.
## Usage
### Run tests with pytest (like buildkite)
```bash
# H200
pytest -s -v tests/evals/gpt_oss/test_gpqa_correctness.py \
--config-list-file=configs/models-h200.txt
# B200
pytest -s -v tests/evals/gpt_oss/test_gpqa_correctness.py \
--config-list-file=configs/models-b200.txt
```
## Configuration Format
Model configs in `configs/` directory use this YAML format:
```yaml
model_name: "openai/gpt-oss-20b"
metric_threshold: 0.568 # Minimum expected accuracy
reasoning_effort: "low" # Reasoning effort level (default: "low")
server_args: "--tensor-parallel-size 2" # Server arguments
startup_max_wait_seconds: 1800 # Max wait for server startup (default: 1800)
env: # Environment variables (optional)
SOME_VAR: "value"
```
The `server_args` field accepts any arguments that can be passed to `vllm serve`.
The `env` field accepts a dictionary of environment variables to set for the server process.
## Adding New Models
1. Create a new YAML config file in the `configs/` directory
2. Add the filename to the appropriate `models-*.txt` file
## Tiktoken Encoding Files
The tiktoken encoding files required by the vLLM server are automatically downloaded from OpenAI's public blob storage on first run:
- `cl100k_base.tiktoken`
- `o200k_base.tiktoken`
Files are cached in the `data/` directory. The `TIKTOKEN_ENCODINGS_BASE` environment variable is automatically set to point to this directory when running evaluations.
@@ -0,0 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
model_name: "openai/gpt-oss-20b"
metric_threshold: 0.568
reasoning_effort: "low"
server_args: "--tensor-parallel-size 2"
@@ -0,0 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
model_name: "openai/gpt-oss-20b"
metric_threshold: 0.568
reasoning_effort: "low"
server_args: "--tensor-parallel-size 2"
env:
VLLM_USE_FLASHINFER_MOE_MXFP4_BF16: "1"
@@ -0,0 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
model_name: "openai/gpt-oss-20b"
metric_threshold: 0.568
reasoning_effort: "low"
server_args: "--tensor-parallel-size 2"
env:
VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS: "1"
@@ -0,0 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
model_name: "openai/gpt-oss-20b"
metric_threshold: 0.568
reasoning_effort: "low"
server_args: "--tensor-parallel-size 2"
env:
VLLM_MXFP4_USE_MARLIN: "1"
@@ -0,0 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
model_name: "openai/gpt-oss-20b"
metric_threshold: 0.568
reasoning_effort: "low"
server_args: "--tensor-parallel-size 2"
env:
VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8: "1"
@@ -0,0 +1,5 @@
# B200 model configurations for GPQA evaluation
# Tests different environment variable combinations
gpt-oss-20b-flashinfer-mxfp4-bf16.yaml
gpt-oss-20b-flashinfer-mxfp4-mxfp8-cutlass.yaml
gpt-oss-20b-sm100-fi-mxfp4-mxfp8-trtllm.yaml
@@ -0,0 +1,5 @@
# H100 model configurations for GPQA evaluation
# Tests different environment variable combinations
gpt-oss-20b-baseline.yaml
gpt-oss-20b-flashinfer-mxfp4-bf16.yaml
gpt-oss-20b-marlin.yaml
+54 -6
View File
@@ -4,13 +4,61 @@
Pytest configuration for GPT-OSS evaluation tests.
"""
from pathlib import Path
def pytest_addoption(parser):
"""Add command line options for pytest."""
parser.addoption("--model", action="store", help="Model name to evaluate")
"""Add custom command line options."""
parser.addoption(
"--metric", action="store", type=float, help="Expected metric threshold"
)
parser.addoption(
"--server-args", action="store", default="", help="Additional server arguments"
"--config-list-file",
required=True,
help="File containing list of config files to test",
)
def pytest_generate_tests(metafunc):
"""Generate test parameters from config files."""
if "config_filename" in metafunc.fixturenames:
config_list_file = metafunc.config.getoption("--config-list-file")
# Handle both relative and absolute paths
config_list_path = Path(config_list_file)
if not config_list_path.is_absolute():
# If relative, try relative to test directory first
test_dir_path = Path(__file__).parent / config_list_file
if test_dir_path.exists():
config_list_path = test_dir_path
else:
# Try relative to current working directory
config_list_path = Path.cwd() / config_list_file
print(f"Looking for config list at: {config_list_path}")
config_files = []
if config_list_path.exists():
# Determine config directory (same directory as the list file)
config_dir = config_list_path.parent
with open(config_list_path) as f:
for line in f:
line = line.strip()
if line and not line.startswith("#"):
config_path = config_dir / line
print(f"Checking config file: {config_path}")
if config_path.exists():
config_files.append(config_path)
print(f" Found: {config_path}")
else:
print(f" Missing: {config_path}")
else:
print(f"Config list file not found: {config_list_path}")
# Generate test parameters
if config_files:
metafunc.parametrize(
"config_filename",
config_files,
ids=[config_file.stem for config_file in config_files],
)
else:
print("No config files found, test will be skipped")
+82 -28
View File
@@ -5,22 +5,48 @@ GPQA evaluation using vLLM server and GPT-OSS evaluation package.
Usage:
pytest -s -v tests/evals/gpt_oss/test_gpqa_correctness.py \
--model openai/gpt-oss-20b \
--metric 0.58 \
--server-args "--tensor-parallel-size 2"
--config-list-file=configs/models-h200.txt
"""
import os
import shlex
import subprocess
import sys
import urllib.request
from pathlib import Path
import regex as re
import yaml
from tests.utils import RemoteOpenAIServer
TOL = 0.05 # Absolute tolerance for accuracy comparison
# Path to tiktoken encoding files
TIKTOKEN_DATA_DIR = Path(__file__).parent / "data"
def run_gpqa_eval(model_name: str, base_url: str) -> float:
# Tiktoken encoding files to download
TIKTOKEN_FILES = {
"cl100k_base.tiktoken": "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken",
"o200k_base.tiktoken": "https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken",
}
def ensure_tiktoken_files():
"""Download tiktoken encoding files if they don't exist."""
TIKTOKEN_DATA_DIR.mkdir(parents=True, exist_ok=True)
for filename, url in TIKTOKEN_FILES.items():
filepath = TIKTOKEN_DATA_DIR / filename
if not filepath.exists():
print(f"Downloading {filename} from {url}...")
urllib.request.urlretrieve(url, filepath)
print(f" Downloaded to {filepath}")
else:
print(f" {filename} already exists.")
def run_gpqa_eval(model_name: str, base_url: str, reasoning_effort: str) -> float:
"""Run GPQA evaluation using the gpt-oss evaluation package."""
# Build the command to run the evaluation
@@ -33,7 +59,7 @@ def run_gpqa_eval(model_name: str, base_url: str) -> float:
"--model",
model_name,
"--reasoning-effort",
"low",
reasoning_effort,
"--base-url",
base_url,
"--n-threads",
@@ -41,16 +67,29 @@ def run_gpqa_eval(model_name: str, base_url: str) -> float:
]
try:
# Set up environment for the evaluation subprocess
# Inherit current environment and add required variables
eval_env = os.environ.copy()
eval_env["OPENAI_API_KEY"] = "dummy"
# Run the evaluation
result = subprocess.run(
cmd,
text=True,
capture_output=True,
timeout=1800, # 30 minute timeout
env={"OPENAI_API_KEY": "dummy"},
env=eval_env,
)
print("Evaluation process output:\n", result.stdout)
print("Evaluation process stdout:\n", result.stdout)
print("Evaluation process stderr:\n", result.stderr)
print(f"Evaluation process return code: {result.returncode}")
if result.returncode != 0:
raise RuntimeError(
f"Evaluation failed with exit code {result.returncode}:\n"
f"stdout: {result.stdout}\nstderr: {result.stderr}"
)
# Parse the output to extract the score
match = re.search(r"'metric':\s*([\d.]+)", result.stdout)
@@ -64,47 +103,62 @@ def run_gpqa_eval(model_name: str, base_url: str) -> float:
except subprocess.TimeoutExpired as e:
raise RuntimeError("Evaluation timed out") from e
except subprocess.CalledProcessError as e:
raise RuntimeError(
f"Evaluation failed with exit code {e.returncode}:\n"
f"stdout: {e.stdout}\nstderr: {e.stderr}"
) from e
def test_gpqa_correctness(request):
"""Test GPQA correctness for GPT-OSS model."""
def test_gpqa_correctness(config_filename):
"""Test GPQA correctness for a given model configuration."""
# Ensure tiktoken files are downloaded
ensure_tiktoken_files()
# Get command line arguments
model_name = request.config.getoption("--model")
expected_metric = request.config.getoption("--metric")
server_args_str = request.config.getoption("--server-args")
# Verify tiktoken files exist
for filename in TIKTOKEN_FILES:
filepath = TIKTOKEN_DATA_DIR / filename
assert filepath.exists(), f"Tiktoken file not found: {filepath}"
# Parse server arguments
server_args = []
if server_args_str:
server_args = server_args_str.split()
eval_config = yaml.safe_load(config_filename.read_text(encoding="utf-8"))
# Parse server arguments from config (use shlex to handle quoted strings)
server_args_str = eval_config.get("server_args", "")
server_args = shlex.split(server_args_str) if server_args_str else []
# Add standard server arguments
server_args.extend(
[
"--trust-remote-code",
"--enforce-eager",
"--disable-uvicorn-access-log",
]
)
print(f"Starting GPQA evaluation for model: {model_name}")
print(f"Expected metric threshold: {expected_metric}")
# Build server environment with tiktoken path and any config-specified vars
server_env = {"TIKTOKEN_ENCODINGS_BASE": str(TIKTOKEN_DATA_DIR)}
if eval_config.get("env"):
server_env.update(eval_config["env"])
reasoning_effort = eval_config.get("reasoning_effort", "low")
print(f"Starting GPQA evaluation for model: {eval_config['model_name']}")
print(f"Expected metric threshold: {eval_config['metric_threshold']}")
print(f"Reasoning effort: {reasoning_effort}")
print(f"Server args: {' '.join(server_args)}")
print(f"Server environment variables: {server_env}")
# Launch server and run evaluation
with RemoteOpenAIServer(
model_name, server_args, max_wait_seconds=1800
eval_config["model_name"],
server_args,
env_dict=server_env,
max_wait_seconds=eval_config.get("startup_max_wait_seconds", 1800),
) as remote_server:
base_url = remote_server.url_for("v1")
print(f"Server started at: {base_url}")
measured_metric = run_gpqa_eval(model_name, base_url)
measured_metric = run_gpqa_eval(
eval_config["model_name"], base_url, reasoning_effort
)
expected_metric = eval_config["metric_threshold"]
print(f"GPQA Results for {model_name}:")
print(f"GPQA Results for {eval_config['model_name']}:")
print(f" Measured metric: {measured_metric:.4f}")
print(f" Expected metric: {expected_metric:.4f}")
print(f" Tolerance: {TOL:.4f}")
@@ -115,4 +169,4 @@ def test_gpqa_correctness(request):
f"{expected_metric:.4f} - {TOL:.4f} = {expected_metric - TOL:.4f}"
)
print(f"✅ GPQA test passed for {model_name}")
print(f"GPQA test passed for {eval_config['model_name']}")