[Misc] Add chat-completion options to gsm8k_eval.py CLI (#60114)
Signed-off-by: Benjamin Chislett <bchislett@nvidia.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
e43db1f5e2
commit
dac35c5b66
@@ -21,6 +21,22 @@ vllm serve Qwen/Qwen2.5-1.5B-Instruct --port 8000
|
|||||||
python tests/evals/gsm8k/gsm8k_eval.py --port 8000
|
python tests/evals/gsm8k/gsm8k_eval.py --port 8000
|
||||||
```
|
```
|
||||||
|
|
||||||
|
For chat-only models, send the prompts through `/v1/chat/completions`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python tests/evals/gsm8k/gsm8k_eval.py --port 8000 \
|
||||||
|
--use-chat-completions --max-tokens 1024 \
|
||||||
|
--chat-template-kwargs '{"enable_thinking": false}'
|
||||||
|
```
|
||||||
|
|
||||||
|
The server applies the chat template and uses its served model, so `--model` is
|
||||||
|
optional. In chat mode, `temperature` is sent only when `--temperature` is given,
|
||||||
|
so the server's default sampling applies otherwise; completions mode keeps the
|
||||||
|
default of 0. Chat mode sends no stop strings; completions mode keeps them.
|
||||||
|
`--reasoning-effort`, `--top-p`, and `--top-k` set the matching request fields
|
||||||
|
when given. With a reasoning parser, only the final answer `content` is scored.
|
||||||
|
`--save-results` records the request settings used.
|
||||||
|
|
||||||
## Configuration Format
|
## Configuration Format
|
||||||
|
|
||||||
Model configs in `configs/` directory use this YAML format:
|
Model configs in `configs/` directory use this YAML format:
|
||||||
|
|||||||
+123
-20
@@ -76,14 +76,20 @@ def get_answer_value(answer_str: str) -> int:
|
|||||||
return INVALID
|
return INVALID
|
||||||
|
|
||||||
|
|
||||||
|
def _optional_params(**params: object) -> dict[str, object]:
|
||||||
|
return {k: v for k, v in params.items() if v is not None}
|
||||||
|
|
||||||
|
|
||||||
async def call_vllm_api(
|
async def call_vllm_api(
|
||||||
session: aiohttp.ClientSession,
|
session: aiohttp.ClientSession,
|
||||||
prompt: str,
|
prompt: str,
|
||||||
temperature: float,
|
temperature: float | None,
|
||||||
max_tokens: int,
|
max_tokens: int,
|
||||||
stop: list[str] | None = None,
|
stop: list[str] | None = None,
|
||||||
url: str | None = None,
|
url: str | None = None,
|
||||||
seed: int | None = None,
|
seed: int | None = None,
|
||||||
|
top_p: float | None = None,
|
||||||
|
top_k: int | None = None,
|
||||||
) -> tuple[str, int]:
|
) -> tuple[str, int]:
|
||||||
"""Call vLLM's OpenAI-compatible completions endpoint.
|
"""Call vLLM's OpenAI-compatible completions endpoint.
|
||||||
|
|
||||||
@@ -93,12 +99,12 @@ async def call_vllm_api(
|
|||||||
"""
|
"""
|
||||||
data = {
|
data = {
|
||||||
"prompt": prompt,
|
"prompt": prompt,
|
||||||
"temperature": temperature,
|
|
||||||
"max_tokens": max_tokens,
|
"max_tokens": max_tokens,
|
||||||
"stop": stop,
|
"stop": stop,
|
||||||
}
|
}
|
||||||
if seed is not None:
|
data.update(
|
||||||
data["seed"] = seed
|
_optional_params(temperature=temperature, seed=seed, top_p=top_p, top_k=top_k)
|
||||||
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
async with session.post(f"{url}/v1/completions", json=data) as response:
|
async with session.post(f"{url}/v1/completions", json=data) as response:
|
||||||
@@ -114,24 +120,45 @@ async def call_vllm_api(
|
|||||||
|
|
||||||
async def call_vllm_chat_api(
|
async def call_vllm_chat_api(
|
||||||
session: aiohttp.ClientSession,
|
session: aiohttp.ClientSession,
|
||||||
model: str,
|
model: str | None,
|
||||||
prompt: str,
|
prompt: str,
|
||||||
temperature: float,
|
temperature: float | None,
|
||||||
max_tokens: int,
|
max_tokens: int,
|
||||||
stop: list[str] | None = None,
|
stop: list[str] | None = None,
|
||||||
url: str | None = None,
|
url: str | None = None,
|
||||||
seed: int | None = None,
|
seed: int | None = None,
|
||||||
|
top_p: float | None = None,
|
||||||
|
top_k: int | None = None,
|
||||||
|
reasoning_effort: str | None = None,
|
||||||
|
chat_template_kwargs: dict[str, object] | None = None,
|
||||||
) -> tuple[str, int]:
|
) -> tuple[str, int]:
|
||||||
"""Call vLLM's OpenAI-compatible chat completions endpoint."""
|
"""Call vLLM's OpenAI-compatible chat completions endpoint.
|
||||||
|
|
||||||
|
``model``, ``temperature``, ``stop`` and the other optional fields are
|
||||||
|
omitted from the request when ``None``, so the server defaults apply (vLLM
|
||||||
|
uses its served model when ``model`` is omitted).
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Tuple of (final answer content, completion_tokens). Reasoning returned
|
||||||
|
separately by a reasoning parser is not part of the answer.
|
||||||
|
|
||||||
|
"""
|
||||||
data = {
|
data = {
|
||||||
"model": model,
|
|
||||||
"messages": [{"role": "user", "content": prompt}],
|
"messages": [{"role": "user", "content": prompt}],
|
||||||
"temperature": temperature,
|
|
||||||
"max_tokens": max_tokens,
|
"max_tokens": max_tokens,
|
||||||
"stop": stop,
|
|
||||||
}
|
}
|
||||||
if seed is not None:
|
data.update(
|
||||||
data["seed"] = seed
|
_optional_params(
|
||||||
|
stop=stop,
|
||||||
|
model=model,
|
||||||
|
temperature=temperature,
|
||||||
|
seed=seed,
|
||||||
|
top_p=top_p,
|
||||||
|
top_k=top_k,
|
||||||
|
reasoning_effort=reasoning_effort,
|
||||||
|
chat_template_kwargs=chat_template_kwargs,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
async with session.post(f"{url}/v1/chat/completions", json=data) as response:
|
async with session.post(f"{url}/v1/chat/completions", json=data) as response:
|
||||||
@@ -214,16 +241,34 @@ def evaluate_gsm8k(
|
|||||||
use_chat_completions: bool = False,
|
use_chat_completions: bool = False,
|
||||||
host: str = "http://127.0.0.1",
|
host: str = "http://127.0.0.1",
|
||||||
port: int = 8000,
|
port: int = 8000,
|
||||||
temperature: float = 0.0,
|
temperature: float | None = 0.0,
|
||||||
seed: int | None = 42,
|
seed: int | None = 42,
|
||||||
request_timeout_seconds: float = 600,
|
request_timeout_seconds: float = 600,
|
||||||
gen_prefix: str = "",
|
gen_prefix: str = "",
|
||||||
max_concurrency: int | None = None,
|
max_concurrency: int | None = None,
|
||||||
|
top_p: float | None = None,
|
||||||
|
top_k: int | None = None,
|
||||||
|
reasoning_effort: str | None = None,
|
||||||
|
chat_template_kwargs: dict[str, object] | None = None,
|
||||||
) -> dict[str, float | int]:
|
) -> dict[str, float | int]:
|
||||||
"""Evaluate GSM8K accuracy using vLLM serve endpoint.
|
"""Evaluate GSM8K accuracy using vLLM serve endpoint.
|
||||||
|
|
||||||
|
``temperature``, ``top_p`` and ``top_k`` are sent only when not ``None``;
|
||||||
|
otherwise the server default applies. ``model`` is used only in chat mode,
|
||||||
|
where it may be ``None`` (the server uses its served model).
|
||||||
|
``reasoning_effort`` and ``chat_template_kwargs`` require
|
||||||
|
``use_chat_completions=True``. Stop strings are sent only in completions
|
||||||
|
mode.
|
||||||
|
|
||||||
Returns dict with accuracy, invalid_rate, latency, etc.
|
Returns dict with accuracy, invalid_rate, latency, etc.
|
||||||
"""
|
"""
|
||||||
|
if not use_chat_completions and (
|
||||||
|
reasoning_effort is not None or chat_template_kwargs is not None
|
||||||
|
):
|
||||||
|
raise ValueError(
|
||||||
|
"reasoning_effort and chat_template_kwargs require "
|
||||||
|
"use_chat_completions=True"
|
||||||
|
)
|
||||||
base_url = f"{host}:{port}"
|
base_url = f"{host}:{port}"
|
||||||
prompts, labels = _build_gsm8k_prompts(num_questions, num_shots, gen_prefix)
|
prompts, labels = _build_gsm8k_prompts(num_questions, num_shots, gen_prefix)
|
||||||
num_questions = len(prompts)
|
num_questions = len(prompts)
|
||||||
@@ -233,19 +278,21 @@ def evaluate_gsm8k(
|
|||||||
output_tokens: list[int] = [0] * num_questions
|
output_tokens: list[int] = [0] * num_questions
|
||||||
|
|
||||||
async def get_answer(session: aiohttp.ClientSession, i: int) -> tuple[str, int]:
|
async def get_answer(session: aiohttp.ClientSession, i: int) -> tuple[str, int]:
|
||||||
stop = ["Question", "Assistant:", "<|separator|>"]
|
|
||||||
if use_chat_completions:
|
if use_chat_completions:
|
||||||
if model is None:
|
# No stop strings: the chat template delimits the turn, and they
|
||||||
raise ValueError("model is required for chat completions")
|
# would also cut off reasoning.
|
||||||
answer, tokens = await call_vllm_chat_api(
|
answer, tokens = await call_vllm_chat_api(
|
||||||
session=session,
|
session=session,
|
||||||
model=model,
|
model=model,
|
||||||
prompt=prompts[i],
|
prompt=prompts[i],
|
||||||
temperature=temperature,
|
temperature=temperature,
|
||||||
max_tokens=max_tokens,
|
max_tokens=max_tokens,
|
||||||
stop=stop,
|
|
||||||
url=base_url,
|
url=base_url,
|
||||||
seed=seed,
|
seed=seed,
|
||||||
|
top_p=top_p,
|
||||||
|
top_k=top_k,
|
||||||
|
reasoning_effort=reasoning_effort,
|
||||||
|
chat_template_kwargs=chat_template_kwargs,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
answer, tokens = await call_vllm_api(
|
answer, tokens = await call_vllm_api(
|
||||||
@@ -253,9 +300,11 @@ def evaluate_gsm8k(
|
|||||||
prompt=prompts[i],
|
prompt=prompts[i],
|
||||||
temperature=temperature,
|
temperature=temperature,
|
||||||
max_tokens=max_tokens,
|
max_tokens=max_tokens,
|
||||||
stop=stop,
|
stop=["Question", "Assistant:", "<|separator|>"],
|
||||||
url=base_url,
|
url=base_url,
|
||||||
seed=seed,
|
seed=seed,
|
||||||
|
top_p=top_p,
|
||||||
|
top_k=top_k,
|
||||||
)
|
)
|
||||||
states[i] = answer
|
states[i] = answer
|
||||||
output_tokens[i] = tokens
|
output_tokens[i] = tokens
|
||||||
@@ -352,7 +401,38 @@ def main() -> None:
|
|||||||
parser.add_argument("--host", type=str, default="http://127.0.0.1", help="Host URL")
|
parser.add_argument("--host", type=str, default="http://127.0.0.1", help="Host URL")
|
||||||
parser.add_argument("--port", type=int, default=8000, help="Port number")
|
parser.add_argument("--port", type=int, default=8000, help="Port number")
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--temperature", type=float, default=0.0, help="Temperature for generation"
|
"--temperature",
|
||||||
|
type=float,
|
||||||
|
help="Temperature for generation (default: 0 for completions; "
|
||||||
|
"server default with --use-chat-completions)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--top-p", type=float, help="Top-p for generation (server default if unset)"
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--top-k", type=int, help="Top-k for generation (server default if unset)"
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--use-chat-completions",
|
||||||
|
action="store_true",
|
||||||
|
help="Send each few-shot prompt as a user message to /v1/chat/completions "
|
||||||
|
"instead of /v1/completions (for chat-only models)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--model",
|
||||||
|
type=str,
|
||||||
|
help="Model name to send with chat requests (default: omitted, so the "
|
||||||
|
"server uses its served model)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--reasoning-effort",
|
||||||
|
type=str,
|
||||||
|
help="reasoning_effort for chat completions, e.g. none, low, high",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--chat-template-kwargs",
|
||||||
|
type=json.loads,
|
||||||
|
help="JSON chat_template_kwargs, e.g. '{\"enable_thinking\": false}'",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--seed", type=int, default=42, help="Random seed for reproducibility"
|
"--seed", type=int, default=42, help="Random seed for reproducibility"
|
||||||
@@ -371,6 +451,9 @@ def main() -> None:
|
|||||||
parser.add_argument("--save-results", type=str, help="Save results to JSON file")
|
parser.add_argument("--save-results", type=str, help="Save results to JSON file")
|
||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
temperature = args.temperature
|
||||||
|
if temperature is None and not args.use_chat_completions:
|
||||||
|
temperature = 0.0
|
||||||
|
|
||||||
result = evaluate_gsm8k(
|
result = evaluate_gsm8k(
|
||||||
num_questions=args.num_questions,
|
num_questions=args.num_questions,
|
||||||
@@ -378,10 +461,16 @@ def main() -> None:
|
|||||||
max_tokens=args.max_tokens,
|
max_tokens=args.max_tokens,
|
||||||
host=args.host,
|
host=args.host,
|
||||||
port=args.port,
|
port=args.port,
|
||||||
temperature=args.temperature,
|
temperature=temperature,
|
||||||
seed=args.seed,
|
seed=args.seed,
|
||||||
max_concurrency=args.max_concurrency,
|
max_concurrency=args.max_concurrency,
|
||||||
request_timeout_seconds=args.request_timeout_seconds,
|
request_timeout_seconds=args.request_timeout_seconds,
|
||||||
|
model=args.model,
|
||||||
|
use_chat_completions=args.use_chat_completions,
|
||||||
|
top_p=args.top_p,
|
||||||
|
top_k=args.top_k,
|
||||||
|
reasoning_effort=args.reasoning_effort,
|
||||||
|
chat_template_kwargs=args.chat_template_kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Print results to terminal
|
# Print results to terminal
|
||||||
@@ -395,6 +484,20 @@ def main() -> None:
|
|||||||
|
|
||||||
# Optional file saving
|
# Optional file saving
|
||||||
if args.save_results:
|
if args.save_results:
|
||||||
|
# None means the field was not sent and the server default applied.
|
||||||
|
result["request_params"] = {
|
||||||
|
"endpoint": "chat" if args.use_chat_completions else "completions",
|
||||||
|
"model": args.model,
|
||||||
|
"temperature": temperature,
|
||||||
|
"top_p": args.top_p,
|
||||||
|
"top_k": args.top_k,
|
||||||
|
"seed": args.seed,
|
||||||
|
"stop": None
|
||||||
|
if args.use_chat_completions
|
||||||
|
else ["Question", "Assistant:", "<|separator|>"],
|
||||||
|
"reasoning_effort": args.reasoning_effort,
|
||||||
|
"chat_template_kwargs": args.chat_template_kwargs,
|
||||||
|
}
|
||||||
with open(args.save_results, "w") as f:
|
with open(args.save_results, "w") as f:
|
||||||
json.dump(result, f, indent=2)
|
json.dump(result, f, indent=2)
|
||||||
print(f"Results saved to {args.save_results}")
|
print(f"Results saved to {args.save_results}")
|
||||||
|
|||||||
Reference in New Issue
Block a user