[Eval][CI] Add basic mrcr eval to tests/evals/ (#40164)

Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Michael Goin
2026-05-01 12:00:38 -04:00
committed by GitHub
parent 529c671e80
commit 3ccc1ff495
8 changed files with 531 additions and 0 deletions
+7
View File
@@ -137,3 +137,10 @@ steps:
commands:
- uv pip install --system 'gpt-oss[eval]==0.0.5'
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-b200.txt
- label: MRCR Eval Small Models
timeout_in_minutes: 30
source_file_dependencies:
- tests/evals/mrcr/
commands:
- pytest -s -v evals/mrcr/test_mrcr_correctness.py --config-list-file=evals/mrcr/configs/models-small.txt
+44
View File
@@ -0,0 +1,44 @@
# MRCR Long-Context Accuracy Evaluation
Smoke test for long-context behavior using OpenAI's public [`openai/mrcr`](https://huggingface.co/datasets/openai/mrcr) dataset. The model sees a long chat with several near-duplicate "needles" and must reproduce a specific earlier assistant turn verbatim, prepended with a random anti-guessing string.
**Scoring:** if the response doesn't start with `random_string_to_prepend`, score is 0; otherwise the prefix is stripped and the mean `SequenceMatcher.ratio()` against the reference answer is reported.
## Usage
```bash
# Pytest (spawns the server)
pytest -s -v tests/evals/mrcr/test_mrcr_correctness.py \
--config-list-file=configs/models-small.txt
# Standalone (server already running; model and context auto-discovered)
vllm serve Qwen/Qwen3-0.6B --reasoning-parser qwen3 --port 8000
python tests/evals/mrcr/mrcr_eval.py --port 8000
```
## Configuration
```yaml
model_name: "Qwen/Qwen3-0.6B"
# Per-needle thresholds catch bucket-specific regressions (sliding window,
# chunked prefill, prefix cache) that an aggregate can hide. A scalar
# (e.g. `match_ratio_threshold: 0.20`) is also accepted and checked against
# the mean match ratio.
match_ratio_threshold:
2: 0.30
4: 0.15
8: 0.10
num_samples: 30
needles: [2, 4, 8]
# max_prompt_tokens: 32768 # Optional; defaults to server max_model_len - max_tokens - 256
max_tokens: 2048
concurrency: 8
server_args: "--max-model-len 32768 --reasoning-parser qwen3"
```
## Notes
- Samples stream from three parquet shards (`{N}needle/{N}needle_0.parquet`); only the first few row groups are fetched, not the full 1.4 GB repo.
- `max_prompt_tokens` defaults to `max_model_len - max_tokens - 256`, i.e. fills whatever context the server advertises. Set `--max-model-len` on the server to control the smoke-test context length; override `--max-prompt-tokens` on the client to cap below that.
- Sample length is pre-filtered by `n_chars × 4 ≤ max_prompt_tokens`, then verified via the server's `/tokenize` endpoint under the actual chat template.
- Reasoning models: start the server with `--reasoning-parser <name>` (e.g. `qwen3`, `deepseek_r1`) so `<think>` goes to `message.reasoning_content` and doesn't contaminate the scored answer.
+2
View File
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+7
View File
@@ -0,0 +1,7 @@
model_name: "Qwen/Qwen3.5-4B"
needles: [2, 4, 8]
match_ratio_threshold:
2: 0.99
4: 0.84
8: 0.76
server_args: "--max-model-len 128K --reasoning-parser qwen3"
@@ -0,0 +1 @@
Qwen3.5-4B.yaml
+54
View File
@@ -0,0 +1,54 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from pathlib import Path
def pytest_addoption(parser):
"""Add custom command line options."""
parser.addoption(
"--config-list-file",
default="configs/models-small.txt",
help="File containing list of config files to test",
)
def pytest_generate_tests(metafunc):
"""Generate test parameters from config files."""
if "config_filename" in metafunc.fixturenames:
config_list_file = metafunc.config.getoption("--config-list-file")
config_list_path = Path(config_list_file)
if not config_list_path.is_absolute():
test_dir_path = Path(__file__).parent / config_list_file
if test_dir_path.exists():
config_list_path = test_dir_path
else:
config_list_path = Path.cwd() / config_list_file
print(f"Looking for config list at: {config_list_path}")
config_files = []
if config_list_path.exists():
config_dir = config_list_path.parent
with open(config_list_path) as f:
for line in f:
line = line.strip()
if line and not line.startswith("#"):
config_path = config_dir / line
if config_path.exists():
config_files.append(config_path)
print(f" ✓ Found: {config_path}")
else:
print(f" ✗ Missing: {config_path}")
else:
print(f"Config list file not found: {config_list_path}")
if config_files:
metafunc.parametrize(
"config_filename",
config_files,
ids=[config_file.stem for config_file in config_files],
)
else:
print("No config files found, test will be skipped")
+333
View File
@@ -0,0 +1,333 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MRCR long-context evaluation for vLLM's OpenAI-compatible server.
Streams samples from `openai/mrcr` on HuggingFace, sends chat completions to
the server, and scores each response with a prefix-gated SequenceMatcher ratio
against the reference answer.
"""
import argparse
import asyncio
import json
import time
from difflib import SequenceMatcher
import aiohttp
import numpy as np
import requests
from tqdm.asyncio import tqdm
DATASET_REPO = "openai/mrcr"
NEEDLE_SHARDS = {
2: "2needle/2needle_0.parquet",
4: "4needle/4needle_0.parquet",
8: "8needle/8needle_0.parquet",
}
# Reserve headroom for chat-template tokens on top of the messages.
PROMPT_SAFETY_BUFFER = 256
# Pre-filter heuristic before the authoritative /tokenize check.
CHARS_PER_TOKEN = 4
# Skip chain-of-thought on reasoning models; ignored by non-reasoning templates.
DEFAULT_EXTRA_BODY: dict = {"chat_template_kwargs": {"enable_thinking": False}}
def discover_server_model(base_url: str) -> tuple[str, int | None]:
"""Return (model_id, max_model_len) from /v1/models."""
resp = requests.get(f"{base_url}/v1/models", timeout=30)
resp.raise_for_status()
data = resp.json().get("data", [])
if not data:
raise RuntimeError(f"No models advertised at {base_url}/v1/models")
entry = data[0]
return entry["id"], entry.get("max_model_len")
def count_chat_tokens(base_url: str, model: str, messages: list[dict]) -> int:
"""Return the chat-template-rendered token count via /tokenize."""
resp = requests.post(
f"{base_url}/tokenize",
json={"model": model, "messages": messages, "add_generation_prompt": True},
timeout=120,
)
resp.raise_for_status()
return int(resp.json()["count"])
def _load_mrcr_samples(
needles: list[int],
max_prompt_tokens: int,
num_samples: int,
seed: int,
base_url: str,
model_name: str,
) -> list[dict]:
"""Stream MRCR samples balanced across needle buckets, token-verified."""
try:
from datasets import load_dataset
except ImportError as e:
raise ImportError(
"MRCR eval requires `datasets`. Install with: uv pip install datasets"
) from e
max_chars = max_prompt_tokens * CHARS_PER_TOKEN
per_bucket = num_samples // len(needles)
leftover = num_samples - per_bucket * len(needles)
samples: list[dict] = []
for idx, n in enumerate(needles):
if n not in NEEDLE_SHARDS:
raise ValueError(f"Unsupported needle count {n}")
target = per_bucket + (1 if idx < leftover else 0)
if target == 0:
continue
ds = load_dataset(
DATASET_REPO,
data_files=NEEDLE_SHARDS[n],
split="train",
streaming=True,
).shuffle(seed=seed + n, buffer_size=16)
taken = 0
for row in ds:
if int(row.get("n_chars", 0)) > max_chars:
continue
prompt = row["prompt"]
messages = json.loads(prompt) if isinstance(prompt, str) else list(prompt)
n_tokens = count_chat_tokens(base_url, model_name, messages)
if n_tokens > max_prompt_tokens:
continue
samples.append(
{
"messages": messages,
"answer": row["answer"],
"random_string_to_prepend": row["random_string_to_prepend"],
"n_needles": int(row["n_needles"]),
"n_tokens": n_tokens,
}
)
taken += 1
if taken >= target:
break
if taken < target:
print(f"Warning: only {taken}/{target} samples for n_needles={n}")
if not samples:
raise RuntimeError("No MRCR samples fit; loosen max_prompt_tokens.")
return samples
def score_mrcr(response: str, answer: str, random_prefix: str) -> float:
"""Prefix-gated SequenceMatcher ratio; 0 if the prefix is missing."""
if not response.startswith(random_prefix):
return 0.0
stripped = response[len(random_prefix) :]
return SequenceMatcher(a=answer, b=stripped, autojunk=False).ratio()
async def _call_chat(
session: aiohttp.ClientSession,
url: str,
model: str,
messages: list[dict],
max_tokens: int,
temperature: float,
seed: int | None,
extra_body: dict,
) -> tuple[str, int]:
data = {
"model": model,
"messages": messages,
"temperature": temperature,
"max_tokens": max_tokens,
**extra_body,
}
if seed is not None:
data["seed"] = seed
try:
async with session.post(f"{url}/v1/chat/completions", json=data) as resp:
resp.raise_for_status()
result = await resp.json()
text = result["choices"][0]["message"]["content"] or ""
return text, result.get("usage", {}).get("completion_tokens", 0)
except Exception as e:
print(f"chat request failed: {e}")
return "", 0
def evaluate_mrcr(
model_name: str | None = None,
num_samples: int = 40,
needles: list[int] | None = None,
max_prompt_tokens: int | None = None,
max_tokens: int = 2048,
host: str = "http://127.0.0.1",
port: int = 8000,
temperature: float = 0.0,
seed: int | None = 42,
concurrency: int = 8,
extra_body: dict | None = None,
) -> dict:
"""Run MRCR against a vLLM server; auto-discovers model and context."""
needles = needles or [2, 4, 8]
extra_body = DEFAULT_EXTRA_BODY if extra_body is None else extra_body
base_url = f"{host}:{port}"
discovered_model, server_max_len = discover_server_model(base_url)
if model_name is None:
model_name = discovered_model
if max_prompt_tokens is None:
if server_max_len is None:
raise RuntimeError(
"Server did not advertise max_model_len; pass --max-prompt-tokens."
)
max_prompt_tokens = max(512, server_max_len - max_tokens - PROMPT_SAFETY_BUFFER)
print(
f"Model: {model_name} | max_prompt_tokens={max_prompt_tokens} "
f"(server max_model_len={server_max_len}, max_tokens={max_tokens})"
)
samples = _load_mrcr_samples(
needles=needles,
max_prompt_tokens=max_prompt_tokens,
num_samples=num_samples,
seed=seed or 0,
base_url=base_url,
model_name=model_name,
)
tok_counts = [s["n_tokens"] for s in samples]
print(
f"Loaded {len(samples)} samples (needles={needles}, "
f"tokens={min(tok_counts)}-{max(tok_counts)})"
)
async def run():
sem = asyncio.Semaphore(concurrency)
responses = [""] * len(samples)
out_tokens = [0] * len(samples)
async def one(session, i):
async with sem:
text, toks = await _call_chat(
session=session,
url=base_url,
model=model_name,
messages=samples[i]["messages"],
max_tokens=max_tokens,
temperature=temperature,
seed=seed,
extra_body=extra_body,
)
responses[i] = text
out_tokens[i] = toks
timeout = aiohttp.ClientTimeout(total=1800)
async with aiohttp.ClientSession(timeout=timeout) as session:
await tqdm.gather(
*[one(session, i) for i in range(len(samples))], desc="MRCR"
)
return responses, out_tokens
tic = time.perf_counter()
responses, out_tokens = asyncio.run(run())
latency = time.perf_counter() - tic
scores = np.array(
[
score_mrcr(r, s["answer"], s["random_string_to_prepend"])
for r, s in zip(responses, samples)
]
)
prefix_hits = np.array(
[
r.startswith(s["random_string_to_prepend"])
for r, s in zip(responses, samples)
]
)
per_needle = {
f"match_ratio_n{n}": float(
scores[np.array([s["n_needles"] == n for s in samples])].mean()
)
for n in needles
if any(s["n_needles"] == n for s in samples)
}
total_out = int(sum(out_tokens))
return {
"model": model_name,
"match_ratio": float(scores.mean()),
"prefix_hit_rate": float(prefix_hits.mean()),
"per_needle": per_needle,
"num_samples": len(samples),
"latency": latency,
"total_output_tokens": total_out,
"tokens_per_second": total_out / latency if latency > 0 else 0.0,
"max_tokens": max_tokens,
"needles": needles,
"max_prompt_tokens": max_prompt_tokens,
}
def main() -> None:
p = argparse.ArgumentParser(description="MRCR evaluation for vLLM serve")
p.add_argument("--model", default=None, help="Default: discovered from /v1/models")
p.add_argument("--num-samples", type=int, default=40)
p.add_argument(
"--needles", type=int, nargs="+", default=[2, 4, 8], choices=[2, 4, 8]
)
p.add_argument(
"--max-prompt-tokens",
type=int,
default=None,
help="Default: server max_model_len - max_tokens - buffer",
)
p.add_argument("--max-tokens", type=int, default=2048)
p.add_argument("--host", default="http://127.0.0.1")
p.add_argument("--port", type=int, default=8000)
p.add_argument("--temperature", type=float, default=0.0)
p.add_argument("--seed", type=int, default=42)
p.add_argument("--concurrency", type=int, default=8)
p.add_argument(
"--extra-body",
default=None,
help="JSON merged into each request. "
"Pass '{}' to disable the default enable_thinking=false.",
)
p.add_argument("--save-results", default=None)
args = p.parse_args()
extra_body = json.loads(args.extra_body) if args.extra_body else None
result = evaluate_mrcr(
model_name=args.model,
num_samples=args.num_samples,
needles=args.needles,
max_prompt_tokens=args.max_prompt_tokens,
max_tokens=args.max_tokens,
host=args.host,
port=args.port,
temperature=args.temperature,
seed=args.seed,
concurrency=args.concurrency,
extra_body=extra_body,
)
print("\nResults:")
print(f" match_ratio: {result['match_ratio']:.4f}")
print(f" prefix_hit_rate: {result['prefix_hit_rate']:.4f}")
for k, v in result["per_needle"].items():
print(f" {k}: {v:.4f}")
print(f" samples: {result['num_samples']}")
print(f" latency: {result['latency']:.1f}s")
print(f" output tok/s: {result['tokens_per_second']:.1f}")
if args.save_results:
with open(args.save_results, "w") as f:
json.dump(result, f, indent=2)
if __name__ == "__main__":
main()
+83
View File
@@ -0,0 +1,83 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
MRCR long-context accuracy test.
Usage:
pytest -s -v tests/evals/mrcr/test_mrcr_correctness.py \
--config-list-file=configs/models-small.txt
"""
import shlex
import yaml
from tests.utils import RemoteOpenAIServer
from .mrcr_eval import evaluate_mrcr
def _split_host_port(url: str, default_port: int = 8000) -> tuple[str, int]:
if "://" in url:
url = url.split("://", 1)[1]
host_port = url.split("/", 1)[0]
if ":" in host_port:
host, p = host_port.split(":", 1)
return f"http://{host}", int(p)
return f"http://{host_port}", default_port
def test_mrcr_correctness(config_filename):
cfg = yaml.safe_load(config_filename.read_text(encoding="utf-8"))
server_args = shlex.split(cfg.get("server_args", ""))
server_args += ["--trust-remote-code", "--disable-uvicorn-access-log"]
print(
f"MRCR eval for {cfg['model_name']} (threshold {cfg['match_ratio_threshold']})"
)
with RemoteOpenAIServer(
cfg["model_name"],
server_args,
env_dict=cfg.get("env"),
max_wait_seconds=cfg.get("startup_max_wait_seconds", 600),
) as server:
host, port = _split_host_port(server.url_for("v1"))
results = evaluate_mrcr(
model_name=cfg.get("model_name"),
num_samples=cfg.get("num_samples", 40),
needles=cfg.get("needles", [2, 4, 8]),
max_prompt_tokens=cfg.get("max_prompt_tokens"),
max_tokens=cfg.get("max_tokens", 2048),
host=host,
port=port,
concurrency=cfg.get("concurrency", 8),
extra_body=cfg.get("extra_body"),
)
threshold = cfg["match_ratio_threshold"]
tol = cfg.get("tolerance", 0.05)
print(f" match_ratio: {results['match_ratio']:.4f}")
print(f" prefix_hit_rate: {results['prefix_hit_rate']:.4f}")
for k, v in results["per_needle"].items():
print(f" {k}: {v:.4f}")
failures: list[str] = []
if isinstance(threshold, dict):
for n, expected in threshold.items():
key = f"match_ratio_n{int(n)}"
measured = results["per_needle"].get(key)
if measured is None:
failures.append(f"{key}: no samples collected")
elif measured < expected - tol:
failures.append(f"{key}: {measured:.4f} < {expected:.4f} - {tol:.4f}")
else:
measured = results["match_ratio"]
if measured < threshold - tol:
failures.append(
f"match_ratio: {measured:.4f} < {threshold:.4f} - {tol:.4f}"
)
assert not failures, "MRCR thresholds failed: " + "; ".join(failures)