From 27d5f78b638d385904b7409ca398272fc08eead7 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Tue, 30 Jun 2026 13:34:25 -0400 Subject: [PATCH] [CI] Move distributed small LM eval to B200 (#47048) Signed-off-by: Lucas Wilkinson Co-authored-by: OpenAI Codex Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .buildkite/test_areas/lm_eval.yaml | 11 +-- ...DiffusionGemma-26B-A4B-it-FP8-dynamic.yaml | 1 + tests/evals/gsm8k/gsm8k_eval.py | 68 ++++++++++++++++--- tests/evals/gsm8k/test_gsm8k_correctness.py | 2 + 4 files changed, 68 insertions(+), 14 deletions(-) diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index 793b9d8913..8f47e7b5c2 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -54,8 +54,8 @@ steps: - export VLLM_USE_DEEP_GEMM=0 # We found Triton is faster than DeepGEMM for H100 - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-hopper.txt --tp-size=4 -- label: LM Eval Small Models (2xB200) - key: lm-eval-small-models-2xb200 +- label: LM Eval Small Models (1xB200) + key: lm-eval-small-models-1xb200 timeout_in_minutes: 120 device: b200-k8s optional: true @@ -65,9 +65,10 @@ steps: commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell.txt -- label: LM Eval Small Models (2xL4) - key: lm-eval-small-models-tp - timeout_in_minutes: 10 +- label: LM Eval Small Models Distributed (2xB200) + key: lm-eval-small-models-distributed-2xb200 + timeout_in_minutes: 120 + device: b200-k8s num_devices: 2 optional: true source_file_dependencies: diff --git a/tests/evals/gsm8k/configs/DiffusionGemma-26B-A4B-it-FP8-dynamic.yaml b/tests/evals/gsm8k/configs/DiffusionGemma-26B-A4B-it-FP8-dynamic.yaml index 3304cbff65..8d412b7c4b 100644 --- a/tests/evals/gsm8k/configs/DiffusionGemma-26B-A4B-it-FP8-dynamic.yaml +++ b/tests/evals/gsm8k/configs/DiffusionGemma-26B-A4B-it-FP8-dynamic.yaml @@ -3,6 +3,7 @@ accuracy_threshold: 0.84 num_questions: 1319 num_fewshot: 5 startup_max_wait_seconds: 1200 +use_chat_completions: true server_args: >- --enforce-eager --max-model-len 4096 diff --git a/tests/evals/gsm8k/gsm8k_eval.py b/tests/evals/gsm8k/gsm8k_eval.py index ff0718cd2a..89d91cb4a9 100644 --- a/tests/evals/gsm8k/gsm8k_eval.py +++ b/tests/evals/gsm8k/gsm8k_eval.py @@ -110,6 +110,39 @@ async def call_vllm_api( return "", 0 +async def call_vllm_chat_api( + session: aiohttp.ClientSession, + model: str, + prompt: str, + temperature: float, + max_tokens: int, + stop: list[str] | None = None, + url: str | None = None, + seed: int | None = None, +) -> tuple[str, int]: + """Call vLLM's OpenAI-compatible chat completions endpoint.""" + data = { + "model": model, + "messages": [{"role": "user", "content": prompt}], + "temperature": temperature, + "max_tokens": max_tokens, + "stop": stop, + } + if seed is not None: + data["seed"] = seed + + try: + async with session.post(f"{url}/v1/chat/completions", json=data) as response: + response.raise_for_status() + result = await response.json() + text = result["choices"][0]["message"]["content"] or "" + completion_tokens = result.get("usage", {}).get("completion_tokens", 0) + return text, completion_tokens + except Exception as e: + print(f"Error calling vLLM chat API ({type(e).__name__}): {e}") + return "", 0 + + def _build_gsm8k_prompts( num_questions: int = 1319, num_shots: int = 5, @@ -173,6 +206,8 @@ def evaluate_gsm8k( num_questions: int = 1319, num_shots: int = 5, max_tokens: int = 256, + model: str | None = None, + use_chat_completions: bool = False, host: str = "http://127.0.0.1", port: int = 8000, temperature: float = 0.0, @@ -193,15 +228,30 @@ def evaluate_gsm8k( output_tokens: list[int] = [0] * num_questions async def get_answer(session: aiohttp.ClientSession, i: int) -> tuple[str, int]: - answer, tokens = await call_vllm_api( - session=session, - prompt=prompts[i], - temperature=temperature, - max_tokens=max_tokens, - stop=["Question", "Assistant:", "<|separator|>"], - url=base_url, - seed=seed, - ) + stop = ["Question", "Assistant:", "<|separator|>"] + if use_chat_completions: + if model is None: + raise ValueError("model is required for chat completions") + answer, tokens = await call_vllm_chat_api( + session=session, + model=model, + prompt=prompts[i], + temperature=temperature, + max_tokens=max_tokens, + stop=stop, + url=base_url, + seed=seed, + ) + else: + answer, tokens = await call_vllm_api( + session=session, + prompt=prompts[i], + temperature=temperature, + max_tokens=max_tokens, + stop=stop, + url=base_url, + seed=seed, + ) states[i] = answer output_tokens[i] = tokens return answer, tokens diff --git a/tests/evals/gsm8k/test_gsm8k_correctness.py b/tests/evals/gsm8k/test_gsm8k_correctness.py index f796f910bb..c9ec5ff66e 100644 --- a/tests/evals/gsm8k/test_gsm8k_correctness.py +++ b/tests/evals/gsm8k/test_gsm8k_correctness.py @@ -65,6 +65,8 @@ def run_gsm8k_eval(eval_config: dict, server_url: str) -> dict: num_questions=eval_config["num_questions"], num_shots=eval_config["num_fewshot"], max_tokens=eval_config.get("max_tokens", 256), + model=eval_config["model_name"], + use_chat_completions=eval_config.get("use_chat_completions", False), host=host, port=port, request_timeout_seconds=request_timeout_seconds,