[Misc] Add chat-completion options to gsm8k_eval.py CLI (#60114)

Signed-off-by: Benjamin Chislett <bchislett@nvidia.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Benjamin Chislett
2026-10-06 20:13:44 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent e43db1f5e2
commit dac35c5b66
2 changed files with 139 additions and 20 deletions
+16
View File
@@ -21,6 +21,22 @@ vllm serve Qwen/Qwen2.5-1.5B-Instruct --port 8000
python tests/evals/gsm8k/gsm8k_eval.py --port 8000 python tests/evals/gsm8k/gsm8k_eval.py --port 8000
``` ```
For chat-only models, send the prompts through `/v1/chat/completions`:
```bash
python tests/evals/gsm8k/gsm8k_eval.py --port 8000 \
--use-chat-completions --max-tokens 1024 \
--chat-template-kwargs '{"enable_thinking": false}'
```
The server applies the chat template and uses its served model, so `--model` is
optional. In chat mode, `temperature` is sent only when `--temperature` is given,
so the server's default sampling applies otherwise; completions mode keeps the
default of 0. Chat mode sends no stop strings; completions mode keeps them.
`--reasoning-effort`, `--top-p`, and `--top-k` set the matching request fields
when given. With a reasoning parser, only the final answer `content` is scored.
`--save-results` records the request settings used.
## Configuration Format ## Configuration Format
Model configs in `configs/` directory use this YAML format: Model configs in `configs/` directory use this YAML format:
+123 -20
View File
@@ -76,14 +76,20 @@ def get_answer_value(answer_str: str) -> int:
return INVALID return INVALID
def _optional_params(**params: object) -> dict[str, object]:
return {k: v for k, v in params.items() if v is not None}
async def call_vllm_api( async def call_vllm_api(
session: aiohttp.ClientSession, session: aiohttp.ClientSession,
prompt: str, prompt: str,
temperature: float, temperature: float | None,
max_tokens: int, max_tokens: int,
stop: list[str] | None = None, stop: list[str] | None = None,
url: str | None = None, url: str | None = None,
seed: int | None = None, seed: int | None = None,
top_p: float | None = None,
top_k: int | None = None,
) -> tuple[str, int]: ) -> tuple[str, int]:
"""Call vLLM's OpenAI-compatible completions endpoint. """Call vLLM's OpenAI-compatible completions endpoint.
@@ -93,12 +99,12 @@ async def call_vllm_api(
""" """
data = { data = {
"prompt": prompt, "prompt": prompt,
"temperature": temperature,
"max_tokens": max_tokens, "max_tokens": max_tokens,
"stop": stop, "stop": stop,
} }
if seed is not None: data.update(
data["seed"] = seed _optional_params(temperature=temperature, seed=seed, top_p=top_p, top_k=top_k)
)
try: try:
async with session.post(f"{url}/v1/completions", json=data) as response: async with session.post(f"{url}/v1/completions", json=data) as response:
@@ -114,24 +120,45 @@ async def call_vllm_api(
async def call_vllm_chat_api( async def call_vllm_chat_api(
session: aiohttp.ClientSession, session: aiohttp.ClientSession,
model: str, model: str | None,
prompt: str, prompt: str,
temperature: float, temperature: float | None,
max_tokens: int, max_tokens: int,
stop: list[str] | None = None, stop: list[str] | None = None,
url: str | None = None, url: str | None = None,
seed: int | None = None, seed: int | None = None,
top_p: float | None = None,
top_k: int | None = None,
reasoning_effort: str | None = None,
chat_template_kwargs: dict[str, object] | None = None,
) -> tuple[str, int]: ) -> tuple[str, int]:
"""Call vLLM's OpenAI-compatible chat completions endpoint.""" """Call vLLM's OpenAI-compatible chat completions endpoint.
``model``, ``temperature``, ``stop`` and the other optional fields are
omitted from the request when ``None``, so the server defaults apply (vLLM
uses its served model when ``model`` is omitted).
Returns:
Tuple of (final answer content, completion_tokens). Reasoning returned
separately by a reasoning parser is not part of the answer.
"""
data = { data = {
"model": model,
"messages": [{"role": "user", "content": prompt}], "messages": [{"role": "user", "content": prompt}],
"temperature": temperature,
"max_tokens": max_tokens, "max_tokens": max_tokens,
"stop": stop,
} }
if seed is not None: data.update(
data["seed"] = seed _optional_params(
stop=stop,
model=model,
temperature=temperature,
seed=seed,
top_p=top_p,
top_k=top_k,
reasoning_effort=reasoning_effort,
chat_template_kwargs=chat_template_kwargs,
)
)
try: try:
async with session.post(f"{url}/v1/chat/completions", json=data) as response: async with session.post(f"{url}/v1/chat/completions", json=data) as response:
@@ -214,16 +241,34 @@ def evaluate_gsm8k(
use_chat_completions: bool = False, use_chat_completions: bool = False,
host: str = "http://127.0.0.1", host: str = "http://127.0.0.1",
port: int = 8000, port: int = 8000,
temperature: float = 0.0, temperature: float | None = 0.0,
seed: int | None = 42, seed: int | None = 42,
request_timeout_seconds: float = 600, request_timeout_seconds: float = 600,
gen_prefix: str = "", gen_prefix: str = "",
max_concurrency: int | None = None, max_concurrency: int | None = None,
top_p: float | None = None,
top_k: int | None = None,
reasoning_effort: str | None = None,
chat_template_kwargs: dict[str, object] | None = None,
) -> dict[str, float | int]: ) -> dict[str, float | int]:
"""Evaluate GSM8K accuracy using vLLM serve endpoint. """Evaluate GSM8K accuracy using vLLM serve endpoint.
``temperature``, ``top_p`` and ``top_k`` are sent only when not ``None``;
otherwise the server default applies. ``model`` is used only in chat mode,
where it may be ``None`` (the server uses its served model).
``reasoning_effort`` and ``chat_template_kwargs`` require
``use_chat_completions=True``. Stop strings are sent only in completions
mode.
Returns dict with accuracy, invalid_rate, latency, etc. Returns dict with accuracy, invalid_rate, latency, etc.
""" """
if not use_chat_completions and (
reasoning_effort is not None or chat_template_kwargs is not None
):
raise ValueError(
"reasoning_effort and chat_template_kwargs require "
"use_chat_completions=True"
)
base_url = f"{host}:{port}" base_url = f"{host}:{port}"
prompts, labels = _build_gsm8k_prompts(num_questions, num_shots, gen_prefix) prompts, labels = _build_gsm8k_prompts(num_questions, num_shots, gen_prefix)
num_questions = len(prompts) num_questions = len(prompts)
@@ -233,19 +278,21 @@ def evaluate_gsm8k(
output_tokens: list[int] = [0] * num_questions output_tokens: list[int] = [0] * num_questions
async def get_answer(session: aiohttp.ClientSession, i: int) -> tuple[str, int]: async def get_answer(session: aiohttp.ClientSession, i: int) -> tuple[str, int]:
stop = ["Question", "Assistant:", "<|separator|>"]
if use_chat_completions: if use_chat_completions:
if model is None: # No stop strings: the chat template delimits the turn, and they
raise ValueError("model is required for chat completions") # would also cut off reasoning.
answer, tokens = await call_vllm_chat_api( answer, tokens = await call_vllm_chat_api(
session=session, session=session,
model=model, model=model,
prompt=prompts[i], prompt=prompts[i],
temperature=temperature, temperature=temperature,
max_tokens=max_tokens, max_tokens=max_tokens,
stop=stop,
url=base_url, url=base_url,
seed=seed, seed=seed,
top_p=top_p,
top_k=top_k,
reasoning_effort=reasoning_effort,
chat_template_kwargs=chat_template_kwargs,
) )
else: else:
answer, tokens = await call_vllm_api( answer, tokens = await call_vllm_api(
@@ -253,9 +300,11 @@ def evaluate_gsm8k(
prompt=prompts[i], prompt=prompts[i],
temperature=temperature, temperature=temperature,
max_tokens=max_tokens, max_tokens=max_tokens,
stop=stop, stop=["Question", "Assistant:", "<|separator|>"],
url=base_url, url=base_url,
seed=seed, seed=seed,
top_p=top_p,
top_k=top_k,
) )
states[i] = answer states[i] = answer
output_tokens[i] = tokens output_tokens[i] = tokens
@@ -352,7 +401,38 @@ def main() -> None:
parser.add_argument("--host", type=str, default="http://127.0.0.1", help="Host URL") parser.add_argument("--host", type=str, default="http://127.0.0.1", help="Host URL")
parser.add_argument("--port", type=int, default=8000, help="Port number") parser.add_argument("--port", type=int, default=8000, help="Port number")
parser.add_argument( parser.add_argument(
"--temperature", type=float, default=0.0, help="Temperature for generation" "--temperature",
type=float,
help="Temperature for generation (default: 0 for completions; "
"server default with --use-chat-completions)",
)
parser.add_argument(
"--top-p", type=float, help="Top-p for generation (server default if unset)"
)
parser.add_argument(
"--top-k", type=int, help="Top-k for generation (server default if unset)"
)
parser.add_argument(
"--use-chat-completions",
action="store_true",
help="Send each few-shot prompt as a user message to /v1/chat/completions "
"instead of /v1/completions (for chat-only models)",
)
parser.add_argument(
"--model",
type=str,
help="Model name to send with chat requests (default: omitted, so the "
"server uses its served model)",
)
parser.add_argument(
"--reasoning-effort",
type=str,
help="reasoning_effort for chat completions, e.g. none, low, high",
)
parser.add_argument(
"--chat-template-kwargs",
type=json.loads,
help="JSON chat_template_kwargs, e.g. '{\"enable_thinking\": false}'",
) )
parser.add_argument( parser.add_argument(
"--seed", type=int, default=42, help="Random seed for reproducibility" "--seed", type=int, default=42, help="Random seed for reproducibility"
@@ -371,6 +451,9 @@ def main() -> None:
parser.add_argument("--save-results", type=str, help="Save results to JSON file") parser.add_argument("--save-results", type=str, help="Save results to JSON file")
args = parser.parse_args() args = parser.parse_args()
temperature = args.temperature
if temperature is None and not args.use_chat_completions:
temperature = 0.0
result = evaluate_gsm8k( result = evaluate_gsm8k(
num_questions=args.num_questions, num_questions=args.num_questions,
@@ -378,10 +461,16 @@ def main() -> None:
max_tokens=args.max_tokens, max_tokens=args.max_tokens,
host=args.host, host=args.host,
port=args.port, port=args.port,
temperature=args.temperature, temperature=temperature,
seed=args.seed, seed=args.seed,
max_concurrency=args.max_concurrency, max_concurrency=args.max_concurrency,
request_timeout_seconds=args.request_timeout_seconds, request_timeout_seconds=args.request_timeout_seconds,
model=args.model,
use_chat_completions=args.use_chat_completions,
top_p=args.top_p,
top_k=args.top_k,
reasoning_effort=args.reasoning_effort,
chat_template_kwargs=args.chat_template_kwargs,
) )
# Print results to terminal # Print results to terminal
@@ -395,6 +484,20 @@ def main() -> None:
# Optional file saving # Optional file saving
if args.save_results: if args.save_results:
# None means the field was not sent and the server default applied.
result["request_params"] = {
"endpoint": "chat" if args.use_chat_completions else "completions",
"model": args.model,
"temperature": temperature,
"top_p": args.top_p,
"top_k": args.top_k,
"seed": args.seed,
"stop": None
if args.use_chat_completions
else ["Question", "Assistant:", "<|separator|>"],
"reasoning_effort": args.reasoning_effort,
"chat_template_kwargs": args.chat_template_kwargs,
}
with open(args.save_results, "w") as f: with open(args.save_results, "w") as f:
json.dump(result, f, indent=2) json.dump(result, f, indent=2)
print(f"Results saved to {args.save_results}") print(f"Results saved to {args.save_results}")