From dac35c5b66efb0ddabb6f67af5d1817b2c74edc0 Mon Sep 17 00:00:00 2001 From: Benjamin Chislett Date: Tue, 6 Oct 2026 16:13:44 -0400 Subject: [PATCH] [Misc] Add chat-completion options to gsm8k_eval.py CLI (#60114) Signed-off-by: Benjamin Chislett Co-authored-by: Claude Opus 5.5 --- tests/evals/gsm8k/README.md | 16 ++++ tests/evals/gsm8k/gsm8k_eval.py | 143 +++++++++++++++++++++++++++----- 2 files changed, 139 insertions(+), 20 deletions(-) diff --git a/tests/evals/gsm8k/README.md b/tests/evals/gsm8k/README.md index db37d2e224..ea61b6bf45 100644 --- a/tests/evals/gsm8k/README.md +++ b/tests/evals/gsm8k/README.md @@ -21,6 +21,22 @@ vllm serve Qwen/Qwen2.5-1.5B-Instruct --port 8000 python tests/evals/gsm8k/gsm8k_eval.py --port 8000 ``` +For chat-only models, send the prompts through `/v1/chat/completions`: + +```bash +python tests/evals/gsm8k/gsm8k_eval.py --port 8000 \ + --use-chat-completions --max-tokens 1024 \ + --chat-template-kwargs '{"enable_thinking": false}' +``` + +The server applies the chat template and uses its served model, so `--model` is +optional. In chat mode, `temperature` is sent only when `--temperature` is given, +so the server's default sampling applies otherwise; completions mode keeps the +default of 0. Chat mode sends no stop strings; completions mode keeps them. +`--reasoning-effort`, `--top-p`, and `--top-k` set the matching request fields +when given. With a reasoning parser, only the final answer `content` is scored. +`--save-results` records the request settings used. + ## Configuration Format Model configs in `configs/` directory use this YAML format: diff --git a/tests/evals/gsm8k/gsm8k_eval.py b/tests/evals/gsm8k/gsm8k_eval.py index 7e2534b55a..4eec87945b 100644 --- a/tests/evals/gsm8k/gsm8k_eval.py +++ b/tests/evals/gsm8k/gsm8k_eval.py @@ -76,14 +76,20 @@ def get_answer_value(answer_str: str) -> int: return INVALID +def _optional_params(**params: object) -> dict[str, object]: + return {k: v for k, v in params.items() if v is not None} + + async def call_vllm_api( session: aiohttp.ClientSession, prompt: str, - temperature: float, + temperature: float | None, max_tokens: int, stop: list[str] | None = None, url: str | None = None, seed: int | None = None, + top_p: float | None = None, + top_k: int | None = None, ) -> tuple[str, int]: """Call vLLM's OpenAI-compatible completions endpoint. @@ -93,12 +99,12 @@ async def call_vllm_api( """ data = { "prompt": prompt, - "temperature": temperature, "max_tokens": max_tokens, "stop": stop, } - if seed is not None: - data["seed"] = seed + data.update( + _optional_params(temperature=temperature, seed=seed, top_p=top_p, top_k=top_k) + ) try: async with session.post(f"{url}/v1/completions", json=data) as response: @@ -114,24 +120,45 @@ async def call_vllm_api( async def call_vllm_chat_api( session: aiohttp.ClientSession, - model: str, + model: str | None, prompt: str, - temperature: float, + temperature: float | None, max_tokens: int, stop: list[str] | None = None, url: str | None = None, seed: int | None = None, + top_p: float | None = None, + top_k: int | None = None, + reasoning_effort: str | None = None, + chat_template_kwargs: dict[str, object] | None = None, ) -> tuple[str, int]: - """Call vLLM's OpenAI-compatible chat completions endpoint.""" + """Call vLLM's OpenAI-compatible chat completions endpoint. + + ``model``, ``temperature``, ``stop`` and the other optional fields are + omitted from the request when ``None``, so the server defaults apply (vLLM + uses its served model when ``model`` is omitted). + + Returns: + Tuple of (final answer content, completion_tokens). Reasoning returned + separately by a reasoning parser is not part of the answer. + + """ data = { - "model": model, "messages": [{"role": "user", "content": prompt}], - "temperature": temperature, "max_tokens": max_tokens, - "stop": stop, } - if seed is not None: - data["seed"] = seed + data.update( + _optional_params( + stop=stop, + model=model, + temperature=temperature, + seed=seed, + top_p=top_p, + top_k=top_k, + reasoning_effort=reasoning_effort, + chat_template_kwargs=chat_template_kwargs, + ) + ) try: async with session.post(f"{url}/v1/chat/completions", json=data) as response: @@ -214,16 +241,34 @@ def evaluate_gsm8k( use_chat_completions: bool = False, host: str = "http://127.0.0.1", port: int = 8000, - temperature: float = 0.0, + temperature: float | None = 0.0, seed: int | None = 42, request_timeout_seconds: float = 600, gen_prefix: str = "", max_concurrency: int | None = None, + top_p: float | None = None, + top_k: int | None = None, + reasoning_effort: str | None = None, + chat_template_kwargs: dict[str, object] | None = None, ) -> dict[str, float | int]: """Evaluate GSM8K accuracy using vLLM serve endpoint. + ``temperature``, ``top_p`` and ``top_k`` are sent only when not ``None``; + otherwise the server default applies. ``model`` is used only in chat mode, + where it may be ``None`` (the server uses its served model). + ``reasoning_effort`` and ``chat_template_kwargs`` require + ``use_chat_completions=True``. Stop strings are sent only in completions + mode. + Returns dict with accuracy, invalid_rate, latency, etc. """ + if not use_chat_completions and ( + reasoning_effort is not None or chat_template_kwargs is not None + ): + raise ValueError( + "reasoning_effort and chat_template_kwargs require " + "use_chat_completions=True" + ) base_url = f"{host}:{port}" prompts, labels = _build_gsm8k_prompts(num_questions, num_shots, gen_prefix) num_questions = len(prompts) @@ -233,19 +278,21 @@ def evaluate_gsm8k( output_tokens: list[int] = [0] * num_questions async def get_answer(session: aiohttp.ClientSession, i: int) -> tuple[str, int]: - stop = ["Question", "Assistant:", "<|separator|>"] if use_chat_completions: - if model is None: - raise ValueError("model is required for chat completions") + # No stop strings: the chat template delimits the turn, and they + # would also cut off reasoning. answer, tokens = await call_vllm_chat_api( session=session, model=model, prompt=prompts[i], temperature=temperature, max_tokens=max_tokens, - stop=stop, url=base_url, seed=seed, + top_p=top_p, + top_k=top_k, + reasoning_effort=reasoning_effort, + chat_template_kwargs=chat_template_kwargs, ) else: answer, tokens = await call_vllm_api( @@ -253,9 +300,11 @@ def evaluate_gsm8k( prompt=prompts[i], temperature=temperature, max_tokens=max_tokens, - stop=stop, + stop=["Question", "Assistant:", "<|separator|>"], url=base_url, seed=seed, + top_p=top_p, + top_k=top_k, ) states[i] = answer output_tokens[i] = tokens @@ -352,7 +401,38 @@ def main() -> None: parser.add_argument("--host", type=str, default="http://127.0.0.1", help="Host URL") parser.add_argument("--port", type=int, default=8000, help="Port number") parser.add_argument( - "--temperature", type=float, default=0.0, help="Temperature for generation" + "--temperature", + type=float, + help="Temperature for generation (default: 0 for completions; " + "server default with --use-chat-completions)", + ) + parser.add_argument( + "--top-p", type=float, help="Top-p for generation (server default if unset)" + ) + parser.add_argument( + "--top-k", type=int, help="Top-k for generation (server default if unset)" + ) + parser.add_argument( + "--use-chat-completions", + action="store_true", + help="Send each few-shot prompt as a user message to /v1/chat/completions " + "instead of /v1/completions (for chat-only models)", + ) + parser.add_argument( + "--model", + type=str, + help="Model name to send with chat requests (default: omitted, so the " + "server uses its served model)", + ) + parser.add_argument( + "--reasoning-effort", + type=str, + help="reasoning_effort for chat completions, e.g. none, low, high", + ) + parser.add_argument( + "--chat-template-kwargs", + type=json.loads, + help="JSON chat_template_kwargs, e.g. '{\"enable_thinking\": false}'", ) parser.add_argument( "--seed", type=int, default=42, help="Random seed for reproducibility" @@ -371,6 +451,9 @@ def main() -> None: parser.add_argument("--save-results", type=str, help="Save results to JSON file") args = parser.parse_args() + temperature = args.temperature + if temperature is None and not args.use_chat_completions: + temperature = 0.0 result = evaluate_gsm8k( num_questions=args.num_questions, @@ -378,10 +461,16 @@ def main() -> None: max_tokens=args.max_tokens, host=args.host, port=args.port, - temperature=args.temperature, + temperature=temperature, seed=args.seed, max_concurrency=args.max_concurrency, request_timeout_seconds=args.request_timeout_seconds, + model=args.model, + use_chat_completions=args.use_chat_completions, + top_p=args.top_p, + top_k=args.top_k, + reasoning_effort=args.reasoning_effort, + chat_template_kwargs=args.chat_template_kwargs, ) # Print results to terminal @@ -395,6 +484,20 @@ def main() -> None: # Optional file saving if args.save_results: + # None means the field was not sent and the server default applied. + result["request_params"] = { + "endpoint": "chat" if args.use_chat_completions else "completions", + "model": args.model, + "temperature": temperature, + "top_p": args.top_p, + "top_k": args.top_k, + "seed": args.seed, + "stop": None + if args.use_chat_completions + else ["Question", "Assistant:", "<|separator|>"], + "reasoning_effort": args.reasoning_effort, + "chat_template_kwargs": args.chat_template_kwargs, + } with open(args.save_results, "w") as f: json.dump(result, f, indent=2) print(f"Results saved to {args.save_results}")