Convert formatting to use ruff instead of yapf + isort (#26247)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2025-10-05 07:06:22 -07:00
committed by GitHub
parent 17edd8a807
commit d6953beb91
1508 changed files with 115205 additions and 94107 deletions
+1 -1
View File
@@ -1,2 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+6 -8
View File
@@ -8,11 +8,9 @@ Pytest configuration for GPT-OSS evaluation tests.
def pytest_addoption(parser):
"""Add command line options for pytest."""
parser.addoption("--model", action="store", help="Model name to evaluate")
parser.addoption("--metric",
action="store",
type=float,
help="Expected metric threshold")
parser.addoption("--server-args",
action="store",
default="",
help="Additional server arguments")
parser.addoption(
"--metric", action="store", type=float, help="Expected metric threshold"
)
parser.addoption(
"--server-args", action="store", default="", help="Additional server arguments"
)
+29 -12
View File
@@ -25,9 +25,19 @@ def run_gpqa_eval(model_name: str, base_url: str) -> float:
# Build the command to run the evaluation
cmd = [
sys.executable, "-m", "gpt_oss.evals", "--eval", "gpqa", "--model",
model_name, "--reasoning-effort", "low", "--base-url", base_url,
"--n-threads", "200"
sys.executable,
"-m",
"gpt_oss.evals",
"--eval",
"gpqa",
"--model",
model_name,
"--reasoning-effort",
"low",
"--base-url",
base_url,
"--n-threads",
"200",
]
try:
@@ -37,7 +47,8 @@ def run_gpqa_eval(model_name: str, base_url: str) -> float:
text=True,
capture_output=True,
timeout=1800, # 30 minute timeout
env={"OPENAI_API_KEY": "dummy"})
env={"OPENAI_API_KEY": "dummy"},
)
print("Evaluation process output:\n", result.stdout)
@@ -48,14 +59,16 @@ def run_gpqa_eval(model_name: str, base_url: str) -> float:
# If we still can't find it, raise an error
raise ValueError(
f"Could not parse score from evaluation output:\n{result.stdout}")
f"Could not parse score from evaluation output:\n{result.stdout}"
)
except subprocess.TimeoutExpired as e:
raise RuntimeError("Evaluation timed out") from e
except subprocess.CalledProcessError as e:
raise RuntimeError(
f"Evaluation failed with exit code {e.returncode}:\n"
f"stdout: {e.stdout}\nstderr: {e.stderr}") from e
f"stdout: {e.stdout}\nstderr: {e.stderr}"
) from e
def test_gpqa_correctness(request):
@@ -72,17 +85,20 @@ def test_gpqa_correctness(request):
server_args = server_args_str.split()
# Add standard server arguments
server_args.extend([
"--trust-remote-code",
])
server_args.extend(
[
"--trust-remote-code",
]
)
print(f"Starting GPQA evaluation for model: {model_name}")
print(f"Expected metric threshold: {expected_metric}")
print(f"Server args: {' '.join(server_args)}")
# Launch server and run evaluation
with RemoteOpenAIServer(model_name, server_args,
max_wait_seconds=1800) as remote_server:
with RemoteOpenAIServer(
model_name, server_args, max_wait_seconds=1800
) as remote_server:
base_url = remote_server.url_for("v1")
print(f"Server started at: {base_url}")
@@ -96,6 +112,7 @@ def test_gpqa_correctness(request):
# Verify metric is within tolerance
assert measured_metric >= expected_metric - TOL, (
f"GPQA metric too low: {measured_metric:.4f} < "
f"{expected_metric:.4f} - {TOL:.4f} = {expected_metric - TOL:.4f}")
f"{expected_metric:.4f} - {TOL:.4f} = {expected_metric - TOL:.4f}"
)
print(f"✅ GPQA test passed for {model_name}")
+1 -1
View File
@@ -1,2 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+11 -14
View File
@@ -6,13 +6,12 @@ from pathlib import Path
def pytest_addoption(parser):
"""Add custom command line options."""
parser.addoption("--config-list-file",
default="configs/models-small.txt",
help="File containing list of config files to test")
parser.addoption("--tp-size",
default=1,
type=int,
help="Tensor parallel size")
parser.addoption(
"--config-list-file",
default="configs/models-small.txt",
help="File containing list of config files to test",
)
parser.addoption("--tp-size", default=1, type=int, help="Tensor parallel size")
def pytest_generate_tests(metafunc):
@@ -55,12 +54,10 @@ def pytest_generate_tests(metafunc):
# Generate test parameters
if config_files:
metafunc.parametrize(["config_filename", "tp_size"],
[(config_file, int(tp_size))
for config_file in config_files],
ids=[
f"{config_file.stem}-tp{tp_size}"
for config_file in config_files
])
metafunc.parametrize(
["config_filename", "tp_size"],
[(config_file, int(tp_size)) for config_file in config_files],
ids=[f"{config_file.stem}-tp{tp_size}" for config_file in config_files],
)
else:
print("No config files found, test will be skipped")
+49 -52
View File
@@ -76,13 +76,15 @@ def get_answer_value(answer_str: str) -> int:
return INVALID
async def call_vllm_api(session: aiohttp.ClientSession,
prompt: str,
temperature: float,
max_tokens: int,
stop: Optional[list[str]] = None,
url: Optional[str] = None,
seed: Optional[int] = None) -> str:
async def call_vllm_api(
session: aiohttp.ClientSession,
prompt: str,
temperature: float,
max_tokens: int,
stop: Optional[list[str]] = None,
url: Optional[str] = None,
seed: Optional[int] = None,
) -> str:
"""Call vLLM's OpenAI-compatible completions endpoint."""
data = {
"prompt": prompt,
@@ -94,8 +96,7 @@ async def call_vllm_api(session: aiohttp.ClientSession,
data["seed"] = seed
try:
async with session.post(f"{url}/v1/completions",
json=data) as response:
async with session.post(f"{url}/v1/completions", json=data) as response:
response.raise_for_status()
result = await response.json()
return result["choices"][0]["text"]
@@ -104,16 +105,18 @@ async def call_vllm_api(session: aiohttp.ClientSession,
return ""
def evaluate_gsm8k(num_questions: int = 1319,
num_shots: int = 5,
max_tokens: int = 256,
host: str = "http://127.0.0.1",
port: int = 8000,
temperature: float = 0.0,
seed: Optional[int] = 42) -> dict[str, Union[float, int]]:
def evaluate_gsm8k(
num_questions: int = 1319,
num_shots: int = 5,
max_tokens: int = 256,
host: str = "http://127.0.0.1",
port: int = 8000,
temperature: float = 0.0,
seed: Optional[int] = 42,
) -> dict[str, Union[float, int]]:
"""
Evaluate GSM8K accuracy using vLLM serve endpoint.
Returns dict with accuracy, invalid_rate, latency, etc.
"""
base_url = f"{host}:{port}"
@@ -127,8 +130,10 @@ def evaluate_gsm8k(num_questions: int = 1319,
# Build few-shot examples from train split (like lm-eval does)
few_shot_examples = ""
for i in range(num_shots):
few_shot_examples += (f"Question: {train_data[i]['question']}\n"
f"Answer: {train_data[i]['answer']}\n\n")
few_shot_examples += (
f"Question: {train_data[i]['question']}\n"
f"Answer: {train_data[i]['answer']}\n\n"
)
# Prepare test questions and labels from test split
questions = []
@@ -157,15 +162,15 @@ def evaluate_gsm8k(num_questions: int = 1319,
states[i] = answer
return answer
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(
total=600)) as session:
async with aiohttp.ClientSession(
timeout=aiohttp.ClientTimeout(total=600)
) as session:
tasks = [get_answer(session, i) for i in range(num_questions)]
await tqdm.gather(*tasks, desc="Evaluating")
return states
print(f"Running GSM8K evaluation: {num_questions} questions, "
f"{num_shots}-shot")
print(f"Running GSM8K evaluation: {num_questions} questions, {num_shots}-shot")
tic = time.perf_counter()
states = asyncio.run(run_async_evaluation())
@@ -191,36 +196,28 @@ def evaluate_gsm8k(num_questions: int = 1319,
def main() -> None:
parser = argparse.ArgumentParser(
description="GSM8K evaluation for vLLM serve")
parser.add_argument("--num-shots",
type=int,
default=5,
help="Number of few-shot examples")
parser.add_argument("--num-questions",
type=int,
default=1319,
help="Number of questions to evaluate")
parser.add_argument("--max-tokens",
type=int,
default=256,
help="Max tokens for generation")
parser.add_argument("--host",
type=str,
default="http://127.0.0.1",
help="Host URL")
parser = argparse.ArgumentParser(description="GSM8K evaluation for vLLM serve")
parser.add_argument(
"--num-shots", type=int, default=5, help="Number of few-shot examples"
)
parser.add_argument(
"--num-questions",
type=int,
default=1319,
help="Number of questions to evaluate",
)
parser.add_argument(
"--max-tokens", type=int, default=256, help="Max tokens for generation"
)
parser.add_argument("--host", type=str, default="http://127.0.0.1", help="Host URL")
parser.add_argument("--port", type=int, default=8000, help="Port number")
parser.add_argument("--temperature",
type=float,
default=0.0,
help="Temperature for generation")
parser.add_argument("--seed",
type=int,
default=42,
help="Random seed for reproducibility")
parser.add_argument("--save-results",
type=str,
help="Save results to JSON file")
parser.add_argument(
"--temperature", type=float, default=0.0, help="Temperature for generation"
)
parser.add_argument(
"--seed", type=int, default=42, help="Random seed for reproducibility"
)
parser.add_argument("--save-results", type=str, help="Save results to JSON file")
args = parser.parse_args()
+5 -4
View File
@@ -63,9 +63,9 @@ def test_gsm8k_correctness_param(config_filename, tp_size):
]
# Launch server and run evaluation
with RemoteOpenAIServer(eval_config["model_name"],
server_args,
max_wait_seconds=480) as remote_server:
with RemoteOpenAIServer(
eval_config["model_name"], server_args, max_wait_seconds=480
) as remote_server:
server_url = remote_server.url_for("v1")
results = launch_gsm8k_eval(eval_config, server_url, tp_size)
@@ -85,6 +85,7 @@ def test_gsm8k_correctness_param(config_filename, tp_size):
# Verify accuracy is within tolerance
assert measured_accuracy >= expected_accuracy - RTOL, (
f"Accuracy too low: {measured_accuracy:.3f} < "
f"{expected_accuracy:.3f} - {RTOL:.3f}")
f"{expected_accuracy:.3f} - {RTOL:.3f}"
)
print(f"✅ GSM8K test passed for {eval_config['model_name']}")