Convert formatting to use ruff instead of yapf + isort (#26247)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -1,2 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
@@ -8,11 +8,9 @@ Pytest configuration for GPT-OSS evaluation tests.
|
||||
def pytest_addoption(parser):
|
||||
"""Add command line options for pytest."""
|
||||
parser.addoption("--model", action="store", help="Model name to evaluate")
|
||||
parser.addoption("--metric",
|
||||
action="store",
|
||||
type=float,
|
||||
help="Expected metric threshold")
|
||||
parser.addoption("--server-args",
|
||||
action="store",
|
||||
default="",
|
||||
help="Additional server arguments")
|
||||
parser.addoption(
|
||||
"--metric", action="store", type=float, help="Expected metric threshold"
|
||||
)
|
||||
parser.addoption(
|
||||
"--server-args", action="store", default="", help="Additional server arguments"
|
||||
)
|
||||
|
||||
@@ -25,9 +25,19 @@ def run_gpqa_eval(model_name: str, base_url: str) -> float:
|
||||
|
||||
# Build the command to run the evaluation
|
||||
cmd = [
|
||||
sys.executable, "-m", "gpt_oss.evals", "--eval", "gpqa", "--model",
|
||||
model_name, "--reasoning-effort", "low", "--base-url", base_url,
|
||||
"--n-threads", "200"
|
||||
sys.executable,
|
||||
"-m",
|
||||
"gpt_oss.evals",
|
||||
"--eval",
|
||||
"gpqa",
|
||||
"--model",
|
||||
model_name,
|
||||
"--reasoning-effort",
|
||||
"low",
|
||||
"--base-url",
|
||||
base_url,
|
||||
"--n-threads",
|
||||
"200",
|
||||
]
|
||||
|
||||
try:
|
||||
@@ -37,7 +47,8 @@ def run_gpqa_eval(model_name: str, base_url: str) -> float:
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=1800, # 30 minute timeout
|
||||
env={"OPENAI_API_KEY": "dummy"})
|
||||
env={"OPENAI_API_KEY": "dummy"},
|
||||
)
|
||||
|
||||
print("Evaluation process output:\n", result.stdout)
|
||||
|
||||
@@ -48,14 +59,16 @@ def run_gpqa_eval(model_name: str, base_url: str) -> float:
|
||||
|
||||
# If we still can't find it, raise an error
|
||||
raise ValueError(
|
||||
f"Could not parse score from evaluation output:\n{result.stdout}")
|
||||
f"Could not parse score from evaluation output:\n{result.stdout}"
|
||||
)
|
||||
|
||||
except subprocess.TimeoutExpired as e:
|
||||
raise RuntimeError("Evaluation timed out") from e
|
||||
except subprocess.CalledProcessError as e:
|
||||
raise RuntimeError(
|
||||
f"Evaluation failed with exit code {e.returncode}:\n"
|
||||
f"stdout: {e.stdout}\nstderr: {e.stderr}") from e
|
||||
f"stdout: {e.stdout}\nstderr: {e.stderr}"
|
||||
) from e
|
||||
|
||||
|
||||
def test_gpqa_correctness(request):
|
||||
@@ -72,17 +85,20 @@ def test_gpqa_correctness(request):
|
||||
server_args = server_args_str.split()
|
||||
|
||||
# Add standard server arguments
|
||||
server_args.extend([
|
||||
"--trust-remote-code",
|
||||
])
|
||||
server_args.extend(
|
||||
[
|
||||
"--trust-remote-code",
|
||||
]
|
||||
)
|
||||
|
||||
print(f"Starting GPQA evaluation for model: {model_name}")
|
||||
print(f"Expected metric threshold: {expected_metric}")
|
||||
print(f"Server args: {' '.join(server_args)}")
|
||||
|
||||
# Launch server and run evaluation
|
||||
with RemoteOpenAIServer(model_name, server_args,
|
||||
max_wait_seconds=1800) as remote_server:
|
||||
with RemoteOpenAIServer(
|
||||
model_name, server_args, max_wait_seconds=1800
|
||||
) as remote_server:
|
||||
base_url = remote_server.url_for("v1")
|
||||
print(f"Server started at: {base_url}")
|
||||
|
||||
@@ -96,6 +112,7 @@ def test_gpqa_correctness(request):
|
||||
# Verify metric is within tolerance
|
||||
assert measured_metric >= expected_metric - TOL, (
|
||||
f"GPQA metric too low: {measured_metric:.4f} < "
|
||||
f"{expected_metric:.4f} - {TOL:.4f} = {expected_metric - TOL:.4f}")
|
||||
f"{expected_metric:.4f} - {TOL:.4f} = {expected_metric - TOL:.4f}"
|
||||
)
|
||||
|
||||
print(f"✅ GPQA test passed for {model_name}")
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
@@ -6,13 +6,12 @@ from pathlib import Path
|
||||
|
||||
def pytest_addoption(parser):
|
||||
"""Add custom command line options."""
|
||||
parser.addoption("--config-list-file",
|
||||
default="configs/models-small.txt",
|
||||
help="File containing list of config files to test")
|
||||
parser.addoption("--tp-size",
|
||||
default=1,
|
||||
type=int,
|
||||
help="Tensor parallel size")
|
||||
parser.addoption(
|
||||
"--config-list-file",
|
||||
default="configs/models-small.txt",
|
||||
help="File containing list of config files to test",
|
||||
)
|
||||
parser.addoption("--tp-size", default=1, type=int, help="Tensor parallel size")
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc):
|
||||
@@ -55,12 +54,10 @@ def pytest_generate_tests(metafunc):
|
||||
|
||||
# Generate test parameters
|
||||
if config_files:
|
||||
metafunc.parametrize(["config_filename", "tp_size"],
|
||||
[(config_file, int(tp_size))
|
||||
for config_file in config_files],
|
||||
ids=[
|
||||
f"{config_file.stem}-tp{tp_size}"
|
||||
for config_file in config_files
|
||||
])
|
||||
metafunc.parametrize(
|
||||
["config_filename", "tp_size"],
|
||||
[(config_file, int(tp_size)) for config_file in config_files],
|
||||
ids=[f"{config_file.stem}-tp{tp_size}" for config_file in config_files],
|
||||
)
|
||||
else:
|
||||
print("No config files found, test will be skipped")
|
||||
|
||||
@@ -76,13 +76,15 @@ def get_answer_value(answer_str: str) -> int:
|
||||
return INVALID
|
||||
|
||||
|
||||
async def call_vllm_api(session: aiohttp.ClientSession,
|
||||
prompt: str,
|
||||
temperature: float,
|
||||
max_tokens: int,
|
||||
stop: Optional[list[str]] = None,
|
||||
url: Optional[str] = None,
|
||||
seed: Optional[int] = None) -> str:
|
||||
async def call_vllm_api(
|
||||
session: aiohttp.ClientSession,
|
||||
prompt: str,
|
||||
temperature: float,
|
||||
max_tokens: int,
|
||||
stop: Optional[list[str]] = None,
|
||||
url: Optional[str] = None,
|
||||
seed: Optional[int] = None,
|
||||
) -> str:
|
||||
"""Call vLLM's OpenAI-compatible completions endpoint."""
|
||||
data = {
|
||||
"prompt": prompt,
|
||||
@@ -94,8 +96,7 @@ async def call_vllm_api(session: aiohttp.ClientSession,
|
||||
data["seed"] = seed
|
||||
|
||||
try:
|
||||
async with session.post(f"{url}/v1/completions",
|
||||
json=data) as response:
|
||||
async with session.post(f"{url}/v1/completions", json=data) as response:
|
||||
response.raise_for_status()
|
||||
result = await response.json()
|
||||
return result["choices"][0]["text"]
|
||||
@@ -104,16 +105,18 @@ async def call_vllm_api(session: aiohttp.ClientSession,
|
||||
return ""
|
||||
|
||||
|
||||
def evaluate_gsm8k(num_questions: int = 1319,
|
||||
num_shots: int = 5,
|
||||
max_tokens: int = 256,
|
||||
host: str = "http://127.0.0.1",
|
||||
port: int = 8000,
|
||||
temperature: float = 0.0,
|
||||
seed: Optional[int] = 42) -> dict[str, Union[float, int]]:
|
||||
def evaluate_gsm8k(
|
||||
num_questions: int = 1319,
|
||||
num_shots: int = 5,
|
||||
max_tokens: int = 256,
|
||||
host: str = "http://127.0.0.1",
|
||||
port: int = 8000,
|
||||
temperature: float = 0.0,
|
||||
seed: Optional[int] = 42,
|
||||
) -> dict[str, Union[float, int]]:
|
||||
"""
|
||||
Evaluate GSM8K accuracy using vLLM serve endpoint.
|
||||
|
||||
|
||||
Returns dict with accuracy, invalid_rate, latency, etc.
|
||||
"""
|
||||
base_url = f"{host}:{port}"
|
||||
@@ -127,8 +130,10 @@ def evaluate_gsm8k(num_questions: int = 1319,
|
||||
# Build few-shot examples from train split (like lm-eval does)
|
||||
few_shot_examples = ""
|
||||
for i in range(num_shots):
|
||||
few_shot_examples += (f"Question: {train_data[i]['question']}\n"
|
||||
f"Answer: {train_data[i]['answer']}\n\n")
|
||||
few_shot_examples += (
|
||||
f"Question: {train_data[i]['question']}\n"
|
||||
f"Answer: {train_data[i]['answer']}\n\n"
|
||||
)
|
||||
|
||||
# Prepare test questions and labels from test split
|
||||
questions = []
|
||||
@@ -157,15 +162,15 @@ def evaluate_gsm8k(num_questions: int = 1319,
|
||||
states[i] = answer
|
||||
return answer
|
||||
|
||||
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(
|
||||
total=600)) as session:
|
||||
async with aiohttp.ClientSession(
|
||||
timeout=aiohttp.ClientTimeout(total=600)
|
||||
) as session:
|
||||
tasks = [get_answer(session, i) for i in range(num_questions)]
|
||||
await tqdm.gather(*tasks, desc="Evaluating")
|
||||
|
||||
return states
|
||||
|
||||
print(f"Running GSM8K evaluation: {num_questions} questions, "
|
||||
f"{num_shots}-shot")
|
||||
print(f"Running GSM8K evaluation: {num_questions} questions, {num_shots}-shot")
|
||||
|
||||
tic = time.perf_counter()
|
||||
states = asyncio.run(run_async_evaluation())
|
||||
@@ -191,36 +196,28 @@ def evaluate_gsm8k(num_questions: int = 1319,
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="GSM8K evaluation for vLLM serve")
|
||||
parser.add_argument("--num-shots",
|
||||
type=int,
|
||||
default=5,
|
||||
help="Number of few-shot examples")
|
||||
parser.add_argument("--num-questions",
|
||||
type=int,
|
||||
default=1319,
|
||||
help="Number of questions to evaluate")
|
||||
parser.add_argument("--max-tokens",
|
||||
type=int,
|
||||
default=256,
|
||||
help="Max tokens for generation")
|
||||
parser.add_argument("--host",
|
||||
type=str,
|
||||
default="http://127.0.0.1",
|
||||
help="Host URL")
|
||||
parser = argparse.ArgumentParser(description="GSM8K evaluation for vLLM serve")
|
||||
parser.add_argument(
|
||||
"--num-shots", type=int, default=5, help="Number of few-shot examples"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num-questions",
|
||||
type=int,
|
||||
default=1319,
|
||||
help="Number of questions to evaluate",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-tokens", type=int, default=256, help="Max tokens for generation"
|
||||
)
|
||||
parser.add_argument("--host", type=str, default="http://127.0.0.1", help="Host URL")
|
||||
parser.add_argument("--port", type=int, default=8000, help="Port number")
|
||||
parser.add_argument("--temperature",
|
||||
type=float,
|
||||
default=0.0,
|
||||
help="Temperature for generation")
|
||||
parser.add_argument("--seed",
|
||||
type=int,
|
||||
default=42,
|
||||
help="Random seed for reproducibility")
|
||||
parser.add_argument("--save-results",
|
||||
type=str,
|
||||
help="Save results to JSON file")
|
||||
parser.add_argument(
|
||||
"--temperature", type=float, default=0.0, help="Temperature for generation"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--seed", type=int, default=42, help="Random seed for reproducibility"
|
||||
)
|
||||
parser.add_argument("--save-results", type=str, help="Save results to JSON file")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
@@ -63,9 +63,9 @@ def test_gsm8k_correctness_param(config_filename, tp_size):
|
||||
]
|
||||
|
||||
# Launch server and run evaluation
|
||||
with RemoteOpenAIServer(eval_config["model_name"],
|
||||
server_args,
|
||||
max_wait_seconds=480) as remote_server:
|
||||
with RemoteOpenAIServer(
|
||||
eval_config["model_name"], server_args, max_wait_seconds=480
|
||||
) as remote_server:
|
||||
server_url = remote_server.url_for("v1")
|
||||
|
||||
results = launch_gsm8k_eval(eval_config, server_url, tp_size)
|
||||
@@ -85,6 +85,7 @@ def test_gsm8k_correctness_param(config_filename, tp_size):
|
||||
# Verify accuracy is within tolerance
|
||||
assert measured_accuracy >= expected_accuracy - RTOL, (
|
||||
f"Accuracy too low: {measured_accuracy:.3f} < "
|
||||
f"{expected_accuracy:.3f} - {RTOL:.3f}")
|
||||
f"{expected_accuracy:.3f} - {RTOL:.3f}"
|
||||
)
|
||||
|
||||
print(f"✅ GSM8K test passed for {eval_config['model_name']}")
|
||||
|
||||
Reference in New Issue
Block a user