[CI] Move distributed small LM eval to B200 (#47048)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Lucas Wilkinson
2026-06-30 13:34:25 -04:00
committed by GitHub
co-authored by OpenAI Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 7a341fa109
commit 27d5f78b63
4 changed files with 68 additions and 14 deletions
+6 -5
View File
@@ -54,8 +54,8 @@ steps:
- export VLLM_USE_DEEP_GEMM=0 # We found Triton is faster than DeepGEMM for H100
- pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-hopper.txt --tp-size=4
- label: LM Eval Small Models (2xB200)
key: lm-eval-small-models-2xb200
- label: LM Eval Small Models (1xB200)
key: lm-eval-small-models-1xb200
timeout_in_minutes: 120
device: b200-k8s
optional: true
@@ -65,9 +65,10 @@ steps:
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell.txt
- label: LM Eval Small Models (2xL4)
key: lm-eval-small-models-tp
timeout_in_minutes: 10
- label: LM Eval Small Models Distributed (2xB200)
key: lm-eval-small-models-distributed-2xb200
timeout_in_minutes: 120
device: b200-k8s
num_devices: 2
optional: true
source_file_dependencies:
@@ -3,6 +3,7 @@ accuracy_threshold: 0.84
num_questions: 1319
num_fewshot: 5
startup_max_wait_seconds: 1200
use_chat_completions: true
server_args: >-
--enforce-eager
--max-model-len 4096
+59 -9
View File
@@ -110,6 +110,39 @@ async def call_vllm_api(
return "", 0
async def call_vllm_chat_api(
session: aiohttp.ClientSession,
model: str,
prompt: str,
temperature: float,
max_tokens: int,
stop: list[str] | None = None,
url: str | None = None,
seed: int | None = None,
) -> tuple[str, int]:
"""Call vLLM's OpenAI-compatible chat completions endpoint."""
data = {
"model": model,
"messages": [{"role": "user", "content": prompt}],
"temperature": temperature,
"max_tokens": max_tokens,
"stop": stop,
}
if seed is not None:
data["seed"] = seed
try:
async with session.post(f"{url}/v1/chat/completions", json=data) as response:
response.raise_for_status()
result = await response.json()
text = result["choices"][0]["message"]["content"] or ""
completion_tokens = result.get("usage", {}).get("completion_tokens", 0)
return text, completion_tokens
except Exception as e:
print(f"Error calling vLLM chat API ({type(e).__name__}): {e}")
return "", 0
def _build_gsm8k_prompts(
num_questions: int = 1319,
num_shots: int = 5,
@@ -173,6 +206,8 @@ def evaluate_gsm8k(
num_questions: int = 1319,
num_shots: int = 5,
max_tokens: int = 256,
model: str | None = None,
use_chat_completions: bool = False,
host: str = "http://127.0.0.1",
port: int = 8000,
temperature: float = 0.0,
@@ -193,15 +228,30 @@ def evaluate_gsm8k(
output_tokens: list[int] = [0] * num_questions
async def get_answer(session: aiohttp.ClientSession, i: int) -> tuple[str, int]:
answer, tokens = await call_vllm_api(
session=session,
prompt=prompts[i],
temperature=temperature,
max_tokens=max_tokens,
stop=["Question", "Assistant:", "<|separator|>"],
url=base_url,
seed=seed,
)
stop = ["Question", "Assistant:", "<|separator|>"]
if use_chat_completions:
if model is None:
raise ValueError("model is required for chat completions")
answer, tokens = await call_vllm_chat_api(
session=session,
model=model,
prompt=prompts[i],
temperature=temperature,
max_tokens=max_tokens,
stop=stop,
url=base_url,
seed=seed,
)
else:
answer, tokens = await call_vllm_api(
session=session,
prompt=prompts[i],
temperature=temperature,
max_tokens=max_tokens,
stop=stop,
url=base_url,
seed=seed,
)
states[i] = answer
output_tokens[i] = tokens
return answer, tokens
@@ -65,6 +65,8 @@ def run_gsm8k_eval(eval_config: dict, server_url: str) -> dict:
num_questions=eval_config["num_questions"],
num_shots=eval_config["num_fewshot"],
max_tokens=eval_config.get("max_tokens", 256),
model=eval_config["model_name"],
use_chat_completions=eval_config.get("use_chat_completions", False),
host=host,
port=port,
request_timeout_seconds=request_timeout_seconds,