[CI] Move distributed small LM eval to B200 (#47048)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
7a341fa109
commit
27d5f78b63
@@ -54,8 +54,8 @@ steps:
|
||||
- export VLLM_USE_DEEP_GEMM=0 # We found Triton is faster than DeepGEMM for H100
|
||||
- pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-hopper.txt --tp-size=4
|
||||
|
||||
- label: LM Eval Small Models (2xB200)
|
||||
key: lm-eval-small-models-2xb200
|
||||
- label: LM Eval Small Models (1xB200)
|
||||
key: lm-eval-small-models-1xb200
|
||||
timeout_in_minutes: 120
|
||||
device: b200-k8s
|
||||
optional: true
|
||||
@@ -65,9 +65,10 @@ steps:
|
||||
commands:
|
||||
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell.txt
|
||||
|
||||
- label: LM Eval Small Models (2xL4)
|
||||
key: lm-eval-small-models-tp
|
||||
timeout_in_minutes: 10
|
||||
- label: LM Eval Small Models Distributed (2xB200)
|
||||
key: lm-eval-small-models-distributed-2xb200
|
||||
timeout_in_minutes: 120
|
||||
device: b200-k8s
|
||||
num_devices: 2
|
||||
optional: true
|
||||
source_file_dependencies:
|
||||
|
||||
@@ -3,6 +3,7 @@ accuracy_threshold: 0.84
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
startup_max_wait_seconds: 1200
|
||||
use_chat_completions: true
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 4096
|
||||
|
||||
@@ -110,6 +110,39 @@ async def call_vllm_api(
|
||||
return "", 0
|
||||
|
||||
|
||||
async def call_vllm_chat_api(
|
||||
session: aiohttp.ClientSession,
|
||||
model: str,
|
||||
prompt: str,
|
||||
temperature: float,
|
||||
max_tokens: int,
|
||||
stop: list[str] | None = None,
|
||||
url: str | None = None,
|
||||
seed: int | None = None,
|
||||
) -> tuple[str, int]:
|
||||
"""Call vLLM's OpenAI-compatible chat completions endpoint."""
|
||||
data = {
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"temperature": temperature,
|
||||
"max_tokens": max_tokens,
|
||||
"stop": stop,
|
||||
}
|
||||
if seed is not None:
|
||||
data["seed"] = seed
|
||||
|
||||
try:
|
||||
async with session.post(f"{url}/v1/chat/completions", json=data) as response:
|
||||
response.raise_for_status()
|
||||
result = await response.json()
|
||||
text = result["choices"][0]["message"]["content"] or ""
|
||||
completion_tokens = result.get("usage", {}).get("completion_tokens", 0)
|
||||
return text, completion_tokens
|
||||
except Exception as e:
|
||||
print(f"Error calling vLLM chat API ({type(e).__name__}): {e}")
|
||||
return "", 0
|
||||
|
||||
|
||||
def _build_gsm8k_prompts(
|
||||
num_questions: int = 1319,
|
||||
num_shots: int = 5,
|
||||
@@ -173,6 +206,8 @@ def evaluate_gsm8k(
|
||||
num_questions: int = 1319,
|
||||
num_shots: int = 5,
|
||||
max_tokens: int = 256,
|
||||
model: str | None = None,
|
||||
use_chat_completions: bool = False,
|
||||
host: str = "http://127.0.0.1",
|
||||
port: int = 8000,
|
||||
temperature: float = 0.0,
|
||||
@@ -193,15 +228,30 @@ def evaluate_gsm8k(
|
||||
output_tokens: list[int] = [0] * num_questions
|
||||
|
||||
async def get_answer(session: aiohttp.ClientSession, i: int) -> tuple[str, int]:
|
||||
answer, tokens = await call_vllm_api(
|
||||
session=session,
|
||||
prompt=prompts[i],
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
stop=["Question", "Assistant:", "<|separator|>"],
|
||||
url=base_url,
|
||||
seed=seed,
|
||||
)
|
||||
stop = ["Question", "Assistant:", "<|separator|>"]
|
||||
if use_chat_completions:
|
||||
if model is None:
|
||||
raise ValueError("model is required for chat completions")
|
||||
answer, tokens = await call_vllm_chat_api(
|
||||
session=session,
|
||||
model=model,
|
||||
prompt=prompts[i],
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
stop=stop,
|
||||
url=base_url,
|
||||
seed=seed,
|
||||
)
|
||||
else:
|
||||
answer, tokens = await call_vllm_api(
|
||||
session=session,
|
||||
prompt=prompts[i],
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
stop=stop,
|
||||
url=base_url,
|
||||
seed=seed,
|
||||
)
|
||||
states[i] = answer
|
||||
output_tokens[i] = tokens
|
||||
return answer, tokens
|
||||
|
||||
@@ -65,6 +65,8 @@ def run_gsm8k_eval(eval_config: dict, server_url: str) -> dict:
|
||||
num_questions=eval_config["num_questions"],
|
||||
num_shots=eval_config["num_fewshot"],
|
||||
max_tokens=eval_config.get("max_tokens", 256),
|
||||
model=eval_config["model_name"],
|
||||
use_chat_completions=eval_config.get("use_chat_completions", False),
|
||||
host=host,
|
||||
port=port,
|
||||
request_timeout_seconds=request_timeout_seconds,
|
||||
|
||||
Reference in New Issue
Block a user