From 3ccc1ff4958dd07dbffeaa1c48463325c892b518 Mon Sep 17 00:00:00 2001 From: Michael Goin Date: Fri, 1 May 2026 12:00:38 -0400 Subject: [PATCH] [Eval][CI] Add basic mrcr eval to tests/evals/ (#40164) Signed-off-by: mgoin --- .buildkite/test_areas/lm_eval.yaml | 7 + tests/evals/mrcr/README.md | 44 +++ tests/evals/mrcr/__init__.py | 2 + tests/evals/mrcr/configs/Qwen3.5-4B.yaml | 7 + tests/evals/mrcr/configs/models-small.txt | 1 + tests/evals/mrcr/conftest.py | 54 ++++ tests/evals/mrcr/mrcr_eval.py | 333 ++++++++++++++++++++++ tests/evals/mrcr/test_mrcr_correctness.py | 83 ++++++ 8 files changed, 531 insertions(+) create mode 100644 tests/evals/mrcr/README.md create mode 100644 tests/evals/mrcr/__init__.py create mode 100644 tests/evals/mrcr/configs/Qwen3.5-4B.yaml create mode 100644 tests/evals/mrcr/configs/models-small.txt create mode 100644 tests/evals/mrcr/conftest.py create mode 100644 tests/evals/mrcr/mrcr_eval.py create mode 100644 tests/evals/mrcr/test_mrcr_correctness.py diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index 38b64b163e..67b87ab692 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -137,3 +137,10 @@ steps: commands: - uv pip install --system 'gpt-oss[eval]==0.0.5' - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-b200.txt + +- label: MRCR Eval Small Models + timeout_in_minutes: 30 + source_file_dependencies: + - tests/evals/mrcr/ + commands: + - pytest -s -v evals/mrcr/test_mrcr_correctness.py --config-list-file=evals/mrcr/configs/models-small.txt diff --git a/tests/evals/mrcr/README.md b/tests/evals/mrcr/README.md new file mode 100644 index 0000000000..59acc11ac4 --- /dev/null +++ b/tests/evals/mrcr/README.md @@ -0,0 +1,44 @@ +# MRCR Long-Context Accuracy Evaluation + +Smoke test for long-context behavior using OpenAI's public [`openai/mrcr`](https://huggingface.co/datasets/openai/mrcr) dataset. The model sees a long chat with several near-duplicate "needles" and must reproduce a specific earlier assistant turn verbatim, prepended with a random anti-guessing string. + +**Scoring:** if the response doesn't start with `random_string_to_prepend`, score is 0; otherwise the prefix is stripped and the mean `SequenceMatcher.ratio()` against the reference answer is reported. + +## Usage + +```bash +# Pytest (spawns the server) +pytest -s -v tests/evals/mrcr/test_mrcr_correctness.py \ + --config-list-file=configs/models-small.txt + +# Standalone (server already running; model and context auto-discovered) +vllm serve Qwen/Qwen3-0.6B --reasoning-parser qwen3 --port 8000 +python tests/evals/mrcr/mrcr_eval.py --port 8000 +``` + +## Configuration + +```yaml +model_name: "Qwen/Qwen3-0.6B" +# Per-needle thresholds catch bucket-specific regressions (sliding window, +# chunked prefill, prefix cache) that an aggregate can hide. A scalar +# (e.g. `match_ratio_threshold: 0.20`) is also accepted and checked against +# the mean match ratio. +match_ratio_threshold: + 2: 0.30 + 4: 0.15 + 8: 0.10 +num_samples: 30 +needles: [2, 4, 8] +# max_prompt_tokens: 32768 # Optional; defaults to server max_model_len - max_tokens - 256 +max_tokens: 2048 +concurrency: 8 +server_args: "--max-model-len 32768 --reasoning-parser qwen3" +``` + +## Notes + +- Samples stream from three parquet shards (`{N}needle/{N}needle_0.parquet`); only the first few row groups are fetched, not the full 1.4 GB repo. +- `max_prompt_tokens` defaults to `max_model_len - max_tokens - 256`, i.e. fills whatever context the server advertises. Set `--max-model-len` on the server to control the smoke-test context length; override `--max-prompt-tokens` on the client to cap below that. +- Sample length is pre-filtered by `n_chars × 4 ≤ max_prompt_tokens`, then verified via the server's `/tokenize` endpoint under the actual chat template. +- Reasoning models: start the server with `--reasoning-parser ` (e.g. `qwen3`, `deepseek_r1`) so `` goes to `message.reasoning_content` and doesn't contaminate the scored answer. diff --git a/tests/evals/mrcr/__init__.py b/tests/evals/mrcr/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/tests/evals/mrcr/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/tests/evals/mrcr/configs/Qwen3.5-4B.yaml b/tests/evals/mrcr/configs/Qwen3.5-4B.yaml new file mode 100644 index 0000000000..c2fd438dd3 --- /dev/null +++ b/tests/evals/mrcr/configs/Qwen3.5-4B.yaml @@ -0,0 +1,7 @@ +model_name: "Qwen/Qwen3.5-4B" +needles: [2, 4, 8] +match_ratio_threshold: + 2: 0.99 + 4: 0.84 + 8: 0.76 +server_args: "--max-model-len 128K --reasoning-parser qwen3" diff --git a/tests/evals/mrcr/configs/models-small.txt b/tests/evals/mrcr/configs/models-small.txt new file mode 100644 index 0000000000..b78704fe53 --- /dev/null +++ b/tests/evals/mrcr/configs/models-small.txt @@ -0,0 +1 @@ +Qwen3.5-4B.yaml diff --git a/tests/evals/mrcr/conftest.py b/tests/evals/mrcr/conftest.py new file mode 100644 index 0000000000..46f59a56c2 --- /dev/null +++ b/tests/evals/mrcr/conftest.py @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from pathlib import Path + + +def pytest_addoption(parser): + """Add custom command line options.""" + parser.addoption( + "--config-list-file", + default="configs/models-small.txt", + help="File containing list of config files to test", + ) + + +def pytest_generate_tests(metafunc): + """Generate test parameters from config files.""" + if "config_filename" in metafunc.fixturenames: + config_list_file = metafunc.config.getoption("--config-list-file") + + config_list_path = Path(config_list_file) + if not config_list_path.is_absolute(): + test_dir_path = Path(__file__).parent / config_list_file + if test_dir_path.exists(): + config_list_path = test_dir_path + else: + config_list_path = Path.cwd() / config_list_file + + print(f"Looking for config list at: {config_list_path}") + + config_files = [] + if config_list_path.exists(): + config_dir = config_list_path.parent + with open(config_list_path) as f: + for line in f: + line = line.strip() + if line and not line.startswith("#"): + config_path = config_dir / line + if config_path.exists(): + config_files.append(config_path) + print(f" ✓ Found: {config_path}") + else: + print(f" ✗ Missing: {config_path}") + else: + print(f"Config list file not found: {config_list_path}") + + if config_files: + metafunc.parametrize( + "config_filename", + config_files, + ids=[config_file.stem for config_file in config_files], + ) + else: + print("No config files found, test will be skipped") diff --git a/tests/evals/mrcr/mrcr_eval.py b/tests/evals/mrcr/mrcr_eval.py new file mode 100644 index 0000000000..3ab87a57d1 --- /dev/null +++ b/tests/evals/mrcr/mrcr_eval.py @@ -0,0 +1,333 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""MRCR long-context evaluation for vLLM's OpenAI-compatible server. + +Streams samples from `openai/mrcr` on HuggingFace, sends chat completions to +the server, and scores each response with a prefix-gated SequenceMatcher ratio +against the reference answer. +""" + +import argparse +import asyncio +import json +import time +from difflib import SequenceMatcher + +import aiohttp +import numpy as np +import requests +from tqdm.asyncio import tqdm + +DATASET_REPO = "openai/mrcr" +NEEDLE_SHARDS = { + 2: "2needle/2needle_0.parquet", + 4: "4needle/4needle_0.parquet", + 8: "8needle/8needle_0.parquet", +} +# Reserve headroom for chat-template tokens on top of the messages. +PROMPT_SAFETY_BUFFER = 256 +# Pre-filter heuristic before the authoritative /tokenize check. +CHARS_PER_TOKEN = 4 +# Skip chain-of-thought on reasoning models; ignored by non-reasoning templates. +DEFAULT_EXTRA_BODY: dict = {"chat_template_kwargs": {"enable_thinking": False}} + + +def discover_server_model(base_url: str) -> tuple[str, int | None]: + """Return (model_id, max_model_len) from /v1/models.""" + resp = requests.get(f"{base_url}/v1/models", timeout=30) + resp.raise_for_status() + data = resp.json().get("data", []) + if not data: + raise RuntimeError(f"No models advertised at {base_url}/v1/models") + entry = data[0] + return entry["id"], entry.get("max_model_len") + + +def count_chat_tokens(base_url: str, model: str, messages: list[dict]) -> int: + """Return the chat-template-rendered token count via /tokenize.""" + resp = requests.post( + f"{base_url}/tokenize", + json={"model": model, "messages": messages, "add_generation_prompt": True}, + timeout=120, + ) + resp.raise_for_status() + return int(resp.json()["count"]) + + +def _load_mrcr_samples( + needles: list[int], + max_prompt_tokens: int, + num_samples: int, + seed: int, + base_url: str, + model_name: str, +) -> list[dict]: + """Stream MRCR samples balanced across needle buckets, token-verified.""" + try: + from datasets import load_dataset + except ImportError as e: + raise ImportError( + "MRCR eval requires `datasets`. Install with: uv pip install datasets" + ) from e + + max_chars = max_prompt_tokens * CHARS_PER_TOKEN + per_bucket = num_samples // len(needles) + leftover = num_samples - per_bucket * len(needles) + + samples: list[dict] = [] + for idx, n in enumerate(needles): + if n not in NEEDLE_SHARDS: + raise ValueError(f"Unsupported needle count {n}") + target = per_bucket + (1 if idx < leftover else 0) + if target == 0: + continue + + ds = load_dataset( + DATASET_REPO, + data_files=NEEDLE_SHARDS[n], + split="train", + streaming=True, + ).shuffle(seed=seed + n, buffer_size=16) + + taken = 0 + for row in ds: + if int(row.get("n_chars", 0)) > max_chars: + continue + prompt = row["prompt"] + messages = json.loads(prompt) if isinstance(prompt, str) else list(prompt) + n_tokens = count_chat_tokens(base_url, model_name, messages) + if n_tokens > max_prompt_tokens: + continue + samples.append( + { + "messages": messages, + "answer": row["answer"], + "random_string_to_prepend": row["random_string_to_prepend"], + "n_needles": int(row["n_needles"]), + "n_tokens": n_tokens, + } + ) + taken += 1 + if taken >= target: + break + + if taken < target: + print(f"Warning: only {taken}/{target} samples for n_needles={n}") + + if not samples: + raise RuntimeError("No MRCR samples fit; loosen max_prompt_tokens.") + return samples + + +def score_mrcr(response: str, answer: str, random_prefix: str) -> float: + """Prefix-gated SequenceMatcher ratio; 0 if the prefix is missing.""" + if not response.startswith(random_prefix): + return 0.0 + stripped = response[len(random_prefix) :] + return SequenceMatcher(a=answer, b=stripped, autojunk=False).ratio() + + +async def _call_chat( + session: aiohttp.ClientSession, + url: str, + model: str, + messages: list[dict], + max_tokens: int, + temperature: float, + seed: int | None, + extra_body: dict, +) -> tuple[str, int]: + data = { + "model": model, + "messages": messages, + "temperature": temperature, + "max_tokens": max_tokens, + **extra_body, + } + if seed is not None: + data["seed"] = seed + try: + async with session.post(f"{url}/v1/chat/completions", json=data) as resp: + resp.raise_for_status() + result = await resp.json() + text = result["choices"][0]["message"]["content"] or "" + return text, result.get("usage", {}).get("completion_tokens", 0) + except Exception as e: + print(f"chat request failed: {e}") + return "", 0 + + +def evaluate_mrcr( + model_name: str | None = None, + num_samples: int = 40, + needles: list[int] | None = None, + max_prompt_tokens: int | None = None, + max_tokens: int = 2048, + host: str = "http://127.0.0.1", + port: int = 8000, + temperature: float = 0.0, + seed: int | None = 42, + concurrency: int = 8, + extra_body: dict | None = None, +) -> dict: + """Run MRCR against a vLLM server; auto-discovers model and context.""" + needles = needles or [2, 4, 8] + extra_body = DEFAULT_EXTRA_BODY if extra_body is None else extra_body + base_url = f"{host}:{port}" + + discovered_model, server_max_len = discover_server_model(base_url) + if model_name is None: + model_name = discovered_model + if max_prompt_tokens is None: + if server_max_len is None: + raise RuntimeError( + "Server did not advertise max_model_len; pass --max-prompt-tokens." + ) + max_prompt_tokens = max(512, server_max_len - max_tokens - PROMPT_SAFETY_BUFFER) + print( + f"Model: {model_name} | max_prompt_tokens={max_prompt_tokens} " + f"(server max_model_len={server_max_len}, max_tokens={max_tokens})" + ) + + samples = _load_mrcr_samples( + needles=needles, + max_prompt_tokens=max_prompt_tokens, + num_samples=num_samples, + seed=seed or 0, + base_url=base_url, + model_name=model_name, + ) + tok_counts = [s["n_tokens"] for s in samples] + print( + f"Loaded {len(samples)} samples (needles={needles}, " + f"tokens={min(tok_counts)}-{max(tok_counts)})" + ) + + async def run(): + sem = asyncio.Semaphore(concurrency) + responses = [""] * len(samples) + out_tokens = [0] * len(samples) + + async def one(session, i): + async with sem: + text, toks = await _call_chat( + session=session, + url=base_url, + model=model_name, + messages=samples[i]["messages"], + max_tokens=max_tokens, + temperature=temperature, + seed=seed, + extra_body=extra_body, + ) + responses[i] = text + out_tokens[i] = toks + + timeout = aiohttp.ClientTimeout(total=1800) + async with aiohttp.ClientSession(timeout=timeout) as session: + await tqdm.gather( + *[one(session, i) for i in range(len(samples))], desc="MRCR" + ) + return responses, out_tokens + + tic = time.perf_counter() + responses, out_tokens = asyncio.run(run()) + latency = time.perf_counter() - tic + + scores = np.array( + [ + score_mrcr(r, s["answer"], s["random_string_to_prepend"]) + for r, s in zip(responses, samples) + ] + ) + prefix_hits = np.array( + [ + r.startswith(s["random_string_to_prepend"]) + for r, s in zip(responses, samples) + ] + ) + per_needle = { + f"match_ratio_n{n}": float( + scores[np.array([s["n_needles"] == n for s in samples])].mean() + ) + for n in needles + if any(s["n_needles"] == n for s in samples) + } + + total_out = int(sum(out_tokens)) + return { + "model": model_name, + "match_ratio": float(scores.mean()), + "prefix_hit_rate": float(prefix_hits.mean()), + "per_needle": per_needle, + "num_samples": len(samples), + "latency": latency, + "total_output_tokens": total_out, + "tokens_per_second": total_out / latency if latency > 0 else 0.0, + "max_tokens": max_tokens, + "needles": needles, + "max_prompt_tokens": max_prompt_tokens, + } + + +def main() -> None: + p = argparse.ArgumentParser(description="MRCR evaluation for vLLM serve") + p.add_argument("--model", default=None, help="Default: discovered from /v1/models") + p.add_argument("--num-samples", type=int, default=40) + p.add_argument( + "--needles", type=int, nargs="+", default=[2, 4, 8], choices=[2, 4, 8] + ) + p.add_argument( + "--max-prompt-tokens", + type=int, + default=None, + help="Default: server max_model_len - max_tokens - buffer", + ) + p.add_argument("--max-tokens", type=int, default=2048) + p.add_argument("--host", default="http://127.0.0.1") + p.add_argument("--port", type=int, default=8000) + p.add_argument("--temperature", type=float, default=0.0) + p.add_argument("--seed", type=int, default=42) + p.add_argument("--concurrency", type=int, default=8) + p.add_argument( + "--extra-body", + default=None, + help="JSON merged into each request. " + "Pass '{}' to disable the default enable_thinking=false.", + ) + p.add_argument("--save-results", default=None) + args = p.parse_args() + + extra_body = json.loads(args.extra_body) if args.extra_body else None + + result = evaluate_mrcr( + model_name=args.model, + num_samples=args.num_samples, + needles=args.needles, + max_prompt_tokens=args.max_prompt_tokens, + max_tokens=args.max_tokens, + host=args.host, + port=args.port, + temperature=args.temperature, + seed=args.seed, + concurrency=args.concurrency, + extra_body=extra_body, + ) + + print("\nResults:") + print(f" match_ratio: {result['match_ratio']:.4f}") + print(f" prefix_hit_rate: {result['prefix_hit_rate']:.4f}") + for k, v in result["per_needle"].items(): + print(f" {k}: {v:.4f}") + print(f" samples: {result['num_samples']}") + print(f" latency: {result['latency']:.1f}s") + print(f" output tok/s: {result['tokens_per_second']:.1f}") + + if args.save_results: + with open(args.save_results, "w") as f: + json.dump(result, f, indent=2) + + +if __name__ == "__main__": + main() diff --git a/tests/evals/mrcr/test_mrcr_correctness.py b/tests/evals/mrcr/test_mrcr_correctness.py new file mode 100644 index 0000000000..3adfd4dc87 --- /dev/null +++ b/tests/evals/mrcr/test_mrcr_correctness.py @@ -0,0 +1,83 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +MRCR long-context accuracy test. + +Usage: + pytest -s -v tests/evals/mrcr/test_mrcr_correctness.py \ + --config-list-file=configs/models-small.txt +""" + +import shlex + +import yaml + +from tests.utils import RemoteOpenAIServer + +from .mrcr_eval import evaluate_mrcr + + +def _split_host_port(url: str, default_port: int = 8000) -> tuple[str, int]: + if "://" in url: + url = url.split("://", 1)[1] + host_port = url.split("/", 1)[0] + if ":" in host_port: + host, p = host_port.split(":", 1) + return f"http://{host}", int(p) + return f"http://{host_port}", default_port + + +def test_mrcr_correctness(config_filename): + cfg = yaml.safe_load(config_filename.read_text(encoding="utf-8")) + + server_args = shlex.split(cfg.get("server_args", "")) + server_args += ["--trust-remote-code", "--disable-uvicorn-access-log"] + + print( + f"MRCR eval for {cfg['model_name']} (threshold {cfg['match_ratio_threshold']})" + ) + + with RemoteOpenAIServer( + cfg["model_name"], + server_args, + env_dict=cfg.get("env"), + max_wait_seconds=cfg.get("startup_max_wait_seconds", 600), + ) as server: + host, port = _split_host_port(server.url_for("v1")) + results = evaluate_mrcr( + model_name=cfg.get("model_name"), + num_samples=cfg.get("num_samples", 40), + needles=cfg.get("needles", [2, 4, 8]), + max_prompt_tokens=cfg.get("max_prompt_tokens"), + max_tokens=cfg.get("max_tokens", 2048), + host=host, + port=port, + concurrency=cfg.get("concurrency", 8), + extra_body=cfg.get("extra_body"), + ) + + threshold = cfg["match_ratio_threshold"] + tol = cfg.get("tolerance", 0.05) + + print(f" match_ratio: {results['match_ratio']:.4f}") + print(f" prefix_hit_rate: {results['prefix_hit_rate']:.4f}") + for k, v in results["per_needle"].items(): + print(f" {k}: {v:.4f}") + + failures: list[str] = [] + if isinstance(threshold, dict): + for n, expected in threshold.items(): + key = f"match_ratio_n{int(n)}" + measured = results["per_needle"].get(key) + if measured is None: + failures.append(f"{key}: no samples collected") + elif measured < expected - tol: + failures.append(f"{key}: {measured:.4f} < {expected:.4f} - {tol:.4f}") + else: + measured = results["match_ratio"] + if measured < threshold - tol: + failures.append( + f"match_ratio: {measured:.4f} < {threshold:.4f} - {tol:.4f}" + ) + + assert not failures, "MRCR thresholds failed: " + "; ".join(failures)