[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,507 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Isolated GSM8K evaluation script for vLLM serve endpoint."""
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import time
|
||||
from collections.abc import Generator
|
||||
|
||||
import aiohttp
|
||||
import numpy as np
|
||||
import regex as re
|
||||
import requests
|
||||
from tqdm.asyncio import tqdm
|
||||
|
||||
from vllm.assets.base import VLLM_S3_BUCKET_URL
|
||||
|
||||
INVALID = -9999999
|
||||
|
||||
|
||||
def download_and_cache_file(url: str, filename: str | None = None) -> str:
|
||||
"""Download and cache a file from a URL."""
|
||||
if filename is None:
|
||||
filename = os.path.join(tempfile.gettempdir(), url.split("/")[-1])
|
||||
|
||||
if os.path.exists(filename):
|
||||
return filename
|
||||
|
||||
print(f"Downloading from {url} to {filename}")
|
||||
response = requests.get(url, stream=True)
|
||||
response.raise_for_status()
|
||||
|
||||
with open(filename, "wb") as f:
|
||||
for chunk in response.iter_content(chunk_size=1024):
|
||||
f.write(chunk)
|
||||
|
||||
return filename
|
||||
|
||||
|
||||
def load_gsm8k_data() -> tuple[list[dict], list[dict]]:
|
||||
"""Load GSM8K train and test data."""
|
||||
train_url = f"{VLLM_S3_BUCKET_URL}/ci-datasets/gsm8k/train.jsonl"
|
||||
test_url = f"{VLLM_S3_BUCKET_URL}/ci-datasets/gsm8k/test.jsonl"
|
||||
|
||||
train_file = download_and_cache_file(train_url)
|
||||
test_file = download_and_cache_file(test_url)
|
||||
|
||||
train_data = list(read_jsonl(train_file))
|
||||
test_data = list(read_jsonl(test_file))
|
||||
|
||||
return train_data, test_data
|
||||
|
||||
|
||||
def read_jsonl(filename: str) -> Generator[dict, None, None]:
|
||||
"""Read a JSONL file."""
|
||||
with open(filename) as fin:
|
||||
for line in fin:
|
||||
if not line.startswith("#"):
|
||||
yield json.loads(line)
|
||||
|
||||
|
||||
def get_answer_value(answer_str: str) -> int:
|
||||
"""Extract the numerical answer from the response."""
|
||||
answer_str = answer_str.replace(",", "")
|
||||
numbers = re.findall(r"\d+", answer_str)
|
||||
if len(numbers) < 1:
|
||||
return INVALID
|
||||
try:
|
||||
return ast.literal_eval(numbers[-1])
|
||||
except SyntaxError:
|
||||
return INVALID
|
||||
|
||||
|
||||
def _optional_params(**params: object) -> dict[str, object]:
|
||||
return {k: v for k, v in params.items() if v is not None}
|
||||
|
||||
|
||||
async def call_vllm_api(
|
||||
session: aiohttp.ClientSession,
|
||||
prompt: str,
|
||||
temperature: float | None,
|
||||
max_tokens: int,
|
||||
stop: list[str] | None = None,
|
||||
url: str | None = None,
|
||||
seed: int | None = None,
|
||||
top_p: float | None = None,
|
||||
top_k: int | None = None,
|
||||
) -> tuple[str, int]:
|
||||
"""Call vLLM's OpenAI-compatible completions endpoint.
|
||||
|
||||
Returns:
|
||||
Tuple of (response_text, completion_tokens)
|
||||
|
||||
"""
|
||||
data = {
|
||||
"prompt": prompt,
|
||||
"max_tokens": max_tokens,
|
||||
"stop": stop,
|
||||
}
|
||||
data.update(
|
||||
_optional_params(temperature=temperature, seed=seed, top_p=top_p, top_k=top_k)
|
||||
)
|
||||
|
||||
try:
|
||||
async with session.post(f"{url}/v1/completions", json=data) as response:
|
||||
response.raise_for_status()
|
||||
result = await response.json()
|
||||
text = result["choices"][0]["text"]
|
||||
completion_tokens = result.get("usage", {}).get("completion_tokens", 0)
|
||||
return text, completion_tokens
|
||||
except Exception as e:
|
||||
print(f"Error calling vLLM API ({type(e).__name__}): {e}")
|
||||
return "", 0
|
||||
|
||||
|
||||
async def call_vllm_chat_api(
|
||||
session: aiohttp.ClientSession,
|
||||
model: str | None,
|
||||
prompt: str,
|
||||
temperature: float | None,
|
||||
max_tokens: int,
|
||||
stop: list[str] | None = None,
|
||||
url: str | None = None,
|
||||
seed: int | None = None,
|
||||
top_p: float | None = None,
|
||||
top_k: int | None = None,
|
||||
reasoning_effort: str | None = None,
|
||||
chat_template_kwargs: dict[str, object] | None = None,
|
||||
) -> tuple[str, int]:
|
||||
"""Call vLLM's OpenAI-compatible chat completions endpoint.
|
||||
|
||||
``model``, ``temperature``, ``stop`` and the other optional fields are
|
||||
omitted from the request when ``None``, so the server defaults apply (vLLM
|
||||
uses its served model when ``model`` is omitted).
|
||||
|
||||
Returns:
|
||||
Tuple of (final answer content, completion_tokens). Reasoning returned
|
||||
separately by a reasoning parser is not part of the answer.
|
||||
|
||||
"""
|
||||
data = {
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"max_tokens": max_tokens,
|
||||
}
|
||||
data.update(
|
||||
_optional_params(
|
||||
stop=stop,
|
||||
model=model,
|
||||
temperature=temperature,
|
||||
seed=seed,
|
||||
top_p=top_p,
|
||||
top_k=top_k,
|
||||
reasoning_effort=reasoning_effort,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
)
|
||||
)
|
||||
|
||||
try:
|
||||
async with session.post(f"{url}/v1/chat/completions", json=data) as response:
|
||||
response.raise_for_status()
|
||||
result = await response.json()
|
||||
text = result["choices"][0]["message"]["content"] or ""
|
||||
completion_tokens = result.get("usage", {}).get("completion_tokens", 0)
|
||||
return text, completion_tokens
|
||||
except Exception as e:
|
||||
print(f"Error calling vLLM chat API ({type(e).__name__}): {e}")
|
||||
return "", 0
|
||||
|
||||
|
||||
def _build_gsm8k_prompts(
|
||||
num_questions: int = 1319,
|
||||
num_shots: int = 5,
|
||||
gen_prefix: str = "",
|
||||
) -> tuple[list[str], list[int]]:
|
||||
"""Build few-shot GSM8K completion prompts and ground-truth labels."""
|
||||
if num_questions == 0:
|
||||
return [], []
|
||||
train_data, test_data = load_gsm8k_data()
|
||||
num_questions = min(num_questions, len(test_data))
|
||||
|
||||
few_shot_examples = ""
|
||||
for i in range(num_shots):
|
||||
few_shot_examples += (
|
||||
f"Question: {train_data[i]['question']}\n"
|
||||
f"Answer:{gen_prefix} {train_data[i]['answer']}\n\n"
|
||||
)
|
||||
|
||||
prompts = []
|
||||
labels = []
|
||||
for i in range(num_questions):
|
||||
prompts.append(
|
||||
few_shot_examples
|
||||
+ f"Question: {test_data[i]['question']}\nAnswer:{gen_prefix}"
|
||||
)
|
||||
labels.append(get_answer_value(test_data[i]["answer"]))
|
||||
|
||||
assert all(label != INVALID for label in labels), "Some labels are invalid"
|
||||
return prompts, labels
|
||||
|
||||
|
||||
def _score_gsm8k(
|
||||
states: list[str],
|
||||
output_tokens: list[int],
|
||||
labels: list[int],
|
||||
num_shots: int,
|
||||
max_tokens: int,
|
||||
latency: float,
|
||||
) -> dict[str, float | int]:
|
||||
"""Score GSM8K responses and return a results dict."""
|
||||
num_questions = len(labels)
|
||||
preds = [get_answer_value(state) for state in states]
|
||||
accuracy = np.mean(np.array(preds) == np.array(labels))
|
||||
invalid_rate = np.mean(np.array(preds) == INVALID)
|
||||
total_output_tokens = sum(output_tokens)
|
||||
tokens_per_second = total_output_tokens / latency if latency > 0 else 0.0
|
||||
|
||||
return {
|
||||
"accuracy": accuracy,
|
||||
"invalid_rate": invalid_rate,
|
||||
"latency": latency,
|
||||
"questions_per_second": num_questions / latency if latency > 0 else 0.0,
|
||||
"total_output_tokens": total_output_tokens,
|
||||
"tokens_per_second": tokens_per_second,
|
||||
"num_questions": num_questions,
|
||||
"num_shots": num_shots,
|
||||
"max_tokens": max_tokens,
|
||||
"timestamp": time.time(),
|
||||
}
|
||||
|
||||
|
||||
def evaluate_gsm8k(
|
||||
num_questions: int = 1319,
|
||||
num_shots: int = 5,
|
||||
max_tokens: int = 256,
|
||||
model: str | None = None,
|
||||
use_chat_completions: bool = False,
|
||||
host: str = "http://127.0.0.1",
|
||||
port: int = 8000,
|
||||
temperature: float | None = 0.0,
|
||||
seed: int | None = 42,
|
||||
request_timeout_seconds: float = 600,
|
||||
gen_prefix: str = "",
|
||||
max_concurrency: int | None = None,
|
||||
top_p: float | None = None,
|
||||
top_k: int | None = None,
|
||||
reasoning_effort: str | None = None,
|
||||
chat_template_kwargs: dict[str, object] | None = None,
|
||||
) -> dict[str, float | int]:
|
||||
"""Evaluate GSM8K accuracy using vLLM serve endpoint.
|
||||
|
||||
``temperature``, ``top_p`` and ``top_k`` are sent only when not ``None``;
|
||||
otherwise the server default applies. ``model`` is used only in chat mode,
|
||||
where it may be ``None`` (the server uses its served model).
|
||||
``reasoning_effort`` and ``chat_template_kwargs`` require
|
||||
``use_chat_completions=True``. Stop strings are sent only in completions
|
||||
mode.
|
||||
|
||||
Returns dict with accuracy, invalid_rate, latency, etc.
|
||||
"""
|
||||
if not use_chat_completions and (
|
||||
reasoning_effort is not None or chat_template_kwargs is not None
|
||||
):
|
||||
raise ValueError(
|
||||
"reasoning_effort and chat_template_kwargs require "
|
||||
"use_chat_completions=True"
|
||||
)
|
||||
base_url = f"{host}:{port}"
|
||||
prompts, labels = _build_gsm8k_prompts(num_questions, num_shots, gen_prefix)
|
||||
num_questions = len(prompts)
|
||||
|
||||
async def run_async_evaluation():
|
||||
states: list[str] = [""] * num_questions
|
||||
output_tokens: list[int] = [0] * num_questions
|
||||
|
||||
async def get_answer(session: aiohttp.ClientSession, i: int) -> tuple[str, int]:
|
||||
if use_chat_completions:
|
||||
# No stop strings: the chat template delimits the turn, and they
|
||||
# would also cut off reasoning.
|
||||
answer, tokens = await call_vllm_chat_api(
|
||||
session=session,
|
||||
model=model,
|
||||
prompt=prompts[i],
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
url=base_url,
|
||||
seed=seed,
|
||||
top_p=top_p,
|
||||
top_k=top_k,
|
||||
reasoning_effort=reasoning_effort,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
)
|
||||
else:
|
||||
answer, tokens = await call_vllm_api(
|
||||
session=session,
|
||||
prompt=prompts[i],
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
stop=["Question", "Assistant:", "<|separator|>"],
|
||||
url=base_url,
|
||||
seed=seed,
|
||||
top_p=top_p,
|
||||
top_k=top_k,
|
||||
)
|
||||
states[i] = answer
|
||||
output_tokens[i] = tokens
|
||||
return answer, tokens
|
||||
|
||||
timeout = aiohttp.ClientTimeout(total=request_timeout_seconds)
|
||||
connector = (
|
||||
aiohttp.TCPConnector(limit=max_concurrency)
|
||||
if max_concurrency is not None
|
||||
else None
|
||||
)
|
||||
async with aiohttp.ClientSession(
|
||||
timeout=timeout, connector=connector
|
||||
) as session:
|
||||
tasks = [get_answer(session, i) for i in range(num_questions)]
|
||||
await tqdm.gather(*tasks, desc="Evaluating")
|
||||
|
||||
return states, output_tokens
|
||||
|
||||
print(f"Running GSM8K evaluation: {num_questions} questions, {num_shots}-shot")
|
||||
|
||||
tic = time.perf_counter()
|
||||
states, output_tokens = asyncio.run(run_async_evaluation())
|
||||
latency = time.perf_counter() - tic
|
||||
|
||||
return _score_gsm8k(states, output_tokens, labels, num_shots, max_tokens, latency)
|
||||
|
||||
|
||||
def evaluate_gsm8k_offline(
|
||||
llm,
|
||||
num_questions: int = 1319,
|
||||
num_shots: int = 5,
|
||||
max_tokens: int = 256,
|
||||
temperature: float = 0.0,
|
||||
gen_prefix: str = "",
|
||||
use_chat_completions: bool = False,
|
||||
chat_template_kwargs: dict[str, object] | None = None,
|
||||
) -> dict[str, float | int]:
|
||||
"""Evaluate GSM8K accuracy using an offline vllm.LLM object.
|
||||
|
||||
Same prompts and scoring as evaluate_gsm8k(), but runs generation
|
||||
directly via llm.generate() instead of calling a server over HTTP.
|
||||
|
||||
When ``use_chat_completions=True``, prompts go through the chat template via
|
||||
``llm.chat()`` instead of raw completion (for instruction-tuned models).
|
||||
``chat_template_kwargs`` are forwarded to ``llm.chat()`` when provided.
|
||||
"""
|
||||
from vllm import SamplingParams
|
||||
|
||||
prompts, labels = _build_gsm8k_prompts(num_questions, num_shots, gen_prefix)
|
||||
sampling_params = SamplingParams(
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
stop=["Question", "Assistant:", "<|separator|>"],
|
||||
)
|
||||
mode = "chat" if use_chat_completions else "completion"
|
||||
print(
|
||||
f"Running offline GSM8K evaluation: {len(prompts)} questions, "
|
||||
f"{num_shots}-shot, {mode}"
|
||||
)
|
||||
|
||||
tic = time.perf_counter()
|
||||
if use_chat_completions:
|
||||
conversations = [[{"role": "user", "content": p}] for p in prompts]
|
||||
outputs = llm.chat(
|
||||
conversations,
|
||||
sampling_params,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
)
|
||||
else:
|
||||
outputs = llm.generate(prompts, sampling_params)
|
||||
latency = time.perf_counter() - tic
|
||||
|
||||
states = [o.outputs[0].text for o in outputs]
|
||||
output_tokens = [len(o.outputs[0].token_ids) for o in outputs]
|
||||
|
||||
return _score_gsm8k(states, output_tokens, labels, num_shots, max_tokens, latency)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="GSM8K evaluation for vLLM serve")
|
||||
parser.add_argument(
|
||||
"--num-shots", type=int, default=5, help="Number of few-shot examples"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num-questions",
|
||||
type=int,
|
||||
default=1319,
|
||||
help="Number of questions to evaluate",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-tokens", type=int, default=256, help="Max tokens for generation"
|
||||
)
|
||||
parser.add_argument("--host", type=str, default="http://127.0.0.1", help="Host URL")
|
||||
parser.add_argument("--port", type=int, default=8000, help="Port number")
|
||||
parser.add_argument(
|
||||
"--temperature",
|
||||
type=float,
|
||||
help="Temperature for generation (default: 0 for completions; "
|
||||
"server default with --use-chat-completions)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--top-p", type=float, help="Top-p for generation (server default if unset)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--top-k", type=int, help="Top-k for generation (server default if unset)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--use-chat-completions",
|
||||
action="store_true",
|
||||
help="Send each few-shot prompt as a user message to /v1/chat/completions "
|
||||
"instead of /v1/completions (for chat-only models)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
type=str,
|
||||
help="Model name to send with chat requests (default: omitted, so the "
|
||||
"server uses its served model)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--reasoning-effort",
|
||||
type=str,
|
||||
help="reasoning_effort for chat completions, e.g. none, low, high",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--chat-template-kwargs",
|
||||
type=json.loads,
|
||||
help="JSON chat_template_kwargs, e.g. '{\"enable_thinking\": false}'",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--seed", type=int, default=42, help="Random seed for reproducibility"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-concurrency",
|
||||
type=int,
|
||||
help="Maximum number of concurrent requests",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--request-timeout-seconds",
|
||||
type=float,
|
||||
default=600,
|
||||
help="Timeout for each request, including time waiting for a connection",
|
||||
)
|
||||
parser.add_argument("--save-results", type=str, help="Save results to JSON file")
|
||||
|
||||
args = parser.parse_args()
|
||||
temperature = args.temperature
|
||||
if temperature is None and not args.use_chat_completions:
|
||||
temperature = 0.0
|
||||
|
||||
result = evaluate_gsm8k(
|
||||
num_questions=args.num_questions,
|
||||
num_shots=args.num_shots,
|
||||
max_tokens=args.max_tokens,
|
||||
host=args.host,
|
||||
port=args.port,
|
||||
temperature=temperature,
|
||||
seed=args.seed,
|
||||
max_concurrency=args.max_concurrency,
|
||||
request_timeout_seconds=args.request_timeout_seconds,
|
||||
model=args.model,
|
||||
use_chat_completions=args.use_chat_completions,
|
||||
top_p=args.top_p,
|
||||
top_k=args.top_k,
|
||||
reasoning_effort=args.reasoning_effort,
|
||||
chat_template_kwargs=args.chat_template_kwargs,
|
||||
)
|
||||
|
||||
# Print results to terminal
|
||||
print("\nResults:")
|
||||
print(f"Accuracy: {result['accuracy']:.3f}")
|
||||
print(f"Invalid responses: {result['invalid_rate']:.3f}")
|
||||
print(f"Total latency: {result['latency']:.3f} s")
|
||||
print(f"Questions per second: {result['questions_per_second']:.3f}")
|
||||
print(f"Total output tokens: {result['total_output_tokens']}")
|
||||
print(f"Output tokens per second: {result['tokens_per_second']:.3f}")
|
||||
|
||||
# Optional file saving
|
||||
if args.save_results:
|
||||
# None means the field was not sent and the server default applied.
|
||||
result["request_params"] = {
|
||||
"endpoint": "chat" if args.use_chat_completions else "completions",
|
||||
"model": args.model,
|
||||
"temperature": temperature,
|
||||
"top_p": args.top_p,
|
||||
"top_k": args.top_k,
|
||||
"seed": args.seed,
|
||||
"stop": None
|
||||
if args.use_chat_completions
|
||||
else ["Question", "Assistant:", "<|separator|>"],
|
||||
"reasoning_effort": args.reasoning_effort,
|
||||
"chat_template_kwargs": args.chat_template_kwargs,
|
||||
}
|
||||
with open(args.save_results, "w") as f:
|
||||
json.dump(result, f, indent=2)
|
||||
print(f"Results saved to {args.save_results}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,213 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""GSM8K evaluation using vLLM server and isolated GSM8K script.
|
||||
Replacement for lm-eval-harness with better performance and control.
|
||||
|
||||
Usage:
|
||||
pytest -s -v tests/evals/gsm8k/test_gsm8k_correctness.py \
|
||||
--config-list-file=configs/models-small.txt
|
||||
"""
|
||||
|
||||
import shlex
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
import yaml
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .gsm8k_eval import evaluate_gsm8k
|
||||
|
||||
DEFAULT_STARTUP_MAX_WAIT_SECONDS = 1200
|
||||
|
||||
|
||||
def run_gsm8k_eval(eval_config: dict, server_url: str) -> dict:
|
||||
"""Run GSM8K evaluation using our isolated script."""
|
||||
# Extract host and port from server URL
|
||||
if "://" in server_url:
|
||||
server_url = server_url.split("://")[1]
|
||||
|
||||
host_port = server_url.split("/")[0] # Remove path if present
|
||||
if ":" in host_port:
|
||||
host, p = host_port.split(":")
|
||||
port = int(p)
|
||||
else:
|
||||
host = host_port
|
||||
port = 8000
|
||||
|
||||
# Add http:// prefix if not present
|
||||
if not host.startswith("http"):
|
||||
host = f"http://{host}"
|
||||
|
||||
# Run GSM8K evaluation
|
||||
request_timeout_seconds = eval_config.get("request_timeout_seconds", 600)
|
||||
if current_platform.is_rocm():
|
||||
request_timeout_seconds = eval_config.get(
|
||||
"rocm_request_timeout_seconds", request_timeout_seconds
|
||||
)
|
||||
|
||||
results = evaluate_gsm8k(
|
||||
num_questions=eval_config["num_questions"],
|
||||
num_shots=eval_config["num_fewshot"],
|
||||
max_tokens=eval_config.get("max_tokens", 256),
|
||||
model=eval_config["model_name"],
|
||||
use_chat_completions=eval_config.get("use_chat_completions", False),
|
||||
host=host,
|
||||
port=port,
|
||||
temperature=eval_config.get("temperature", 0.0),
|
||||
seed=eval_config.get("seed", 42),
|
||||
request_timeout_seconds=request_timeout_seconds,
|
||||
gen_prefix=eval_config.get("gen_prefix", ""),
|
||||
max_concurrency=eval_config.get("max_concurrency"),
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def get_acceptance_length(server_url: str) -> float:
|
||||
"""Mean tokens emitted per verification step, from the server's counters.
|
||||
|
||||
1.0 means every draft was rejected (speculation bought nothing); the
|
||||
theoretical maximum is 1 + num_speculative_tokens.
|
||||
"""
|
||||
response = requests.get(f"{server_url.rstrip('/').removesuffix('/v1')}/metrics")
|
||||
response.raise_for_status()
|
||||
counters: dict[str, float] = {}
|
||||
for line in response.text.splitlines():
|
||||
if line.startswith("vllm:spec_decode_num_"):
|
||||
name, _, value = line.partition(" ")
|
||||
counters[name.split("{")[0]] = float(value)
|
||||
|
||||
num_drafts = counters.get("vllm:spec_decode_num_drafts_total", 0.0)
|
||||
num_accepted = counters.get("vllm:spec_decode_num_accepted_tokens_total", 0.0)
|
||||
assert num_drafts > 0, (
|
||||
"no drafts recorded; speculative decoding did not run for this config"
|
||||
)
|
||||
return 1.0 + num_accepted / num_drafts
|
||||
|
||||
|
||||
def test_gsm8k_correctness(config_filename):
|
||||
"""Test GSM8K correctness for a given model configuration."""
|
||||
eval_config = yaml.safe_load(config_filename.read_text(encoding="utf-8"))
|
||||
|
||||
if (
|
||||
not current_platform.is_cuda()
|
||||
and "Qwen3-30B-A3B-MXFP4A16" in eval_config["model_name"]
|
||||
):
|
||||
pytest.skip(
|
||||
"Skipping Qwen3-30B-A3B-MXFP4A16 on non-CUDA platforms. "
|
||||
"Marlin kernels are not supported."
|
||||
)
|
||||
|
||||
if (
|
||||
not current_platform.is_cuda()
|
||||
and "gemma-4-E4B-it-qat-mobile-ct" in eval_config["model_name"]
|
||||
):
|
||||
pytest.skip(
|
||||
"Skipping gemma-4-E4B-it-qat-mobile-ct on non-CUDA platforms. "
|
||||
"Its W2A16 (uint2b2) scheme has no kernel outside CUDA."
|
||||
)
|
||||
|
||||
if current_platform.is_rocm() and (
|
||||
"deepseek-ai/DeepSeek-V3.2" in eval_config["model_name"]
|
||||
or "deepseek-ai/DeepSeek-R1" in eval_config["model_name"]
|
||||
):
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
|
||||
if get_cdna_version() < 4:
|
||||
pytest.skip(
|
||||
"Skipping DeepSeek-V3.2 and DeepSeek-R1 on ROCm platforms below "
|
||||
"CDNA 4 due to agent pool disk space issues and pod evictions."
|
||||
)
|
||||
if current_platform.is_rocm() and (
|
||||
"Qwen3.5-35B-A3B-MXFP4-AITER-TP2" in config_filename.name
|
||||
):
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
if not on_gfx950():
|
||||
pytest.skip(
|
||||
"Skipping Qwen3.5-35B-A3B-MXFP4-AITER-TP2 on non-GFX950 platforms. "
|
||||
"The quantization scheme is not supported on non-GFX950 platforms."
|
||||
)
|
||||
if "--all2all-backend=moonep" in eval_config.get("server_args", ""):
|
||||
from vllm.utils.import_utils import has_moonep
|
||||
|
||||
if not has_moonep():
|
||||
pytest.skip("Skipping MoonEP config: the moonep package is not installed.")
|
||||
|
||||
# Parse server arguments from config (use shlex to handle quoted strings)
|
||||
server_args_str = eval_config.get("server_args", "")
|
||||
server_args = shlex.split(server_args_str) if server_args_str else []
|
||||
|
||||
# Add standard server arguments
|
||||
server_args.extend(
|
||||
[
|
||||
"--trust-remote-code",
|
||||
"--disable-uvicorn-access-log",
|
||||
]
|
||||
)
|
||||
|
||||
startup_max_wait_seconds = eval_config.get(
|
||||
"startup_max_wait_seconds", DEFAULT_STARTUP_MAX_WAIT_SECONDS
|
||||
)
|
||||
env_dict = dict(eval_config.get("env") or {})
|
||||
env_dict["VLLM_ENGINE_READY_TIMEOUT_S"] = str(int(startup_max_wait_seconds))
|
||||
|
||||
print(f"Starting GSM8K evaluation for model: {eval_config['model_name']}")
|
||||
print(f"Expected metric threshold: {eval_config['accuracy_threshold']}")
|
||||
print(f"Number of questions: {eval_config['num_questions']}")
|
||||
print(f"Number of few-shot examples: {eval_config['num_fewshot']}")
|
||||
request_timeout_seconds = eval_config.get("request_timeout_seconds", 600)
|
||||
if current_platform.is_rocm():
|
||||
request_timeout_seconds = eval_config.get(
|
||||
"rocm_request_timeout_seconds", request_timeout_seconds
|
||||
)
|
||||
print(f"Request timeout: {request_timeout_seconds}s")
|
||||
print(f"Startup max wait: {startup_max_wait_seconds}s")
|
||||
print(f"Server args: {' '.join(server_args)}")
|
||||
print(f"Environment variables: {env_dict}")
|
||||
|
||||
# Launch server and run evaluation
|
||||
with RemoteOpenAIServer(
|
||||
eval_config["model_name"],
|
||||
server_args,
|
||||
env_dict=env_dict,
|
||||
max_wait_seconds=startup_max_wait_seconds,
|
||||
) as remote_server:
|
||||
server_url = remote_server.url_for("v1")
|
||||
print(f"Server started at: {server_url}")
|
||||
|
||||
results = run_gsm8k_eval(eval_config, server_url)
|
||||
|
||||
measured_metric = results["accuracy"]
|
||||
expected_metric = eval_config["accuracy_threshold"]
|
||||
tol = eval_config.get("tolerance", 0.08)
|
||||
|
||||
print(f"GSM8K Results for {eval_config['model_name']}:")
|
||||
print(f" Measured metric: {measured_metric:.4f}")
|
||||
print(f" Expected metric: {expected_metric:.4f}")
|
||||
print(f" Tolerance: {tol:.4f}")
|
||||
print(f" Questions: {results['num_questions']}")
|
||||
print(f" Invalid rate: {results['invalid_rate']:.3f}")
|
||||
print(f" Latency: {results['latency']:.1f}s")
|
||||
print(f" QPS: {results['questions_per_second']:.1f}")
|
||||
|
||||
assert measured_metric >= expected_metric - tol, (
|
||||
f"GSM8K metric too low: {measured_metric:.4f} < "
|
||||
f"{expected_metric:.4f} - {tol:.4f} = {expected_metric - tol:.4f}"
|
||||
)
|
||||
|
||||
# Speculative configs additionally assert that drafts are actually
|
||||
# landing: accuracy alone passes even when every draft is rejected.
|
||||
min_acceptance_length = eval_config.get("min_acceptance_length")
|
||||
if min_acceptance_length is not None:
|
||||
acceptance_length = get_acceptance_length(server_url)
|
||||
print(f" Mean acceptance length: {acceptance_length:.3f}")
|
||||
print(f" Minimum acceptance length: {min_acceptance_length:.3f}")
|
||||
assert acceptance_length >= min_acceptance_length, (
|
||||
f"Acceptance length too low: {acceptance_length:.3f} < "
|
||||
f"{min_acceptance_length:.3f}"
|
||||
)
|
||||
|
||||
print(f"✅ GSM8K test passed for {eval_config['model_name']}")
|
||||
@@ -1,290 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""GSM8K correctness test for CPU KV offloading connectors.
|
||||
|
||||
Regression guard for stride computation bugs in the offloading worker
|
||||
(e.g. https://github.com/vllm-project/vllm/pull/46888) and silent KV
|
||||
cache data corruption during CPU offloading. Runs GSM8K twice, dropping
|
||||
the GPU prefix cache (but not the CPU cache) between runs so the second
|
||||
run reloads offloaded KV data from CPU. The reset only succeeds once
|
||||
in-flight offload transfers have released their GPU blocks, so retrying
|
||||
it until success also waits for offloading to complete.
|
||||
|
||||
Correctness is enforced via GSM8K accuracy on both runs: corrupt reloaded
|
||||
KV drops accuracy below threshold. The reload itself is not asserted, so
|
||||
a silently skipped reload (e.g. offloading disabled) would not be flagged.
|
||||
|
||||
Covers both KV offloading connectors (OffloadingConnector and
|
||||
SimpleCPUOffloadConnector) across five architecture families:
|
||||
- Hybrid Mamba (NemotronH: attention + Mamba)
|
||||
- Heterogeneous head dim (Gemma 4)
|
||||
- Hybrid GDN (Qwen 3.5: attention + GatedDeltaNet)
|
||||
- Compressed attention (DeepSeek-V4-Flash: CSA)
|
||||
- Pure MLA (DeepSeek-V2-Lite: TieringOffloadingSpec, TP=2, replicated
|
||||
single-slot host layout)
|
||||
|
||||
Usage:
|
||||
pytest -s -v evals/gsm8k/test_gsm8k_offloading.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .gsm8k_eval import evaluate_gsm8k
|
||||
|
||||
if not current_platform.is_cuda_alike():
|
||||
pytest.skip("Requires CUDA or ROCm", allow_module_level=True)
|
||||
|
||||
NUM_QUESTIONS = 200
|
||||
NUM_FEWSHOT = 5
|
||||
|
||||
_OFFLOAD_SYNC_TIMEOUT = 60
|
||||
|
||||
|
||||
def _kv_transfer_config(
|
||||
connector: str, cpu_gib: int = 4, spec_name: str = "CPUOffloadingSpec"
|
||||
) -> str:
|
||||
if connector == "OffloadingConnector":
|
||||
return json.dumps(
|
||||
{
|
||||
"kv_connector": "OffloadingConnector",
|
||||
"kv_role": "kv_both",
|
||||
"kv_connector_extra_config": {
|
||||
"spec_name": spec_name,
|
||||
"cpu_bytes_to_use": cpu_gib << 30,
|
||||
"eviction_policy": "lru",
|
||||
},
|
||||
}
|
||||
)
|
||||
elif connector == "SimpleCPUOffloadConnector":
|
||||
return json.dumps(
|
||||
{
|
||||
"kv_connector": "SimpleCPUOffloadConnector",
|
||||
"kv_role": "kv_both",
|
||||
"kv_connector_extra_config": {
|
||||
"cpu_bytes_to_use": cpu_gib << 30,
|
||||
},
|
||||
}
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unknown connector: {connector}")
|
||||
|
||||
|
||||
def _force_engine_step(base_url: str) -> None:
|
||||
"""Force an engine step so completed offload transfers get processed."""
|
||||
requests.post(
|
||||
f"{base_url}/v1/completions",
|
||||
json={"prompt": "0", "max_tokens": 1, "temperature": 0},
|
||||
timeout=60,
|
||||
).raise_for_status()
|
||||
|
||||
|
||||
def _reset_gpu_prefix_cache(base_url: str) -> None:
|
||||
"""Drop the GPU prefix cache while keeping the CPU (connector) cache, so
|
||||
the next run must reload KV data through the connector.
|
||||
|
||||
The reset fails while asynchronous offload transfers still hold GPU
|
||||
blocks, so retry until it succeeds. Requires VLLM_SERVER_DEV_MODE=1.
|
||||
"""
|
||||
deadline = time.monotonic() + _OFFLOAD_SYNC_TIMEOUT
|
||||
while True:
|
||||
resp = requests.post(
|
||||
f"{base_url}/reset_prefix_cache",
|
||||
params={"reset_external": "false"},
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
if resp.json().get("success"):
|
||||
return
|
||||
assert time.monotonic() < deadline, (
|
||||
f"prefix cache reset did not succeed within {_OFFLOAD_SYNC_TIMEOUT}s; "
|
||||
"async offload may be stuck"
|
||||
)
|
||||
_force_engine_step(base_url)
|
||||
|
||||
|
||||
@dataclass
|
||||
class OffloadingModelConfig:
|
||||
id: str
|
||||
model: str
|
||||
connector: str
|
||||
accuracy_threshold: float
|
||||
tolerance: float = 0.05
|
||||
extra_server_args: list[str] = field(default_factory=list)
|
||||
env_dict: dict[str, str] = field(default_factory=dict)
|
||||
cpu_offload_gib: int = 4
|
||||
startup_timeout: int = 600
|
||||
spec_name: str = "CPUOffloadingSpec"
|
||||
|
||||
|
||||
MODELS = [
|
||||
# ── OffloadingConnector ──────────────────────────────────────────
|
||||
OffloadingModelConfig(
|
||||
id="offloading-nemotron-h-8b",
|
||||
model="nvidia/Nemotron-H-8B-Base-8K",
|
||||
connector="OffloadingConnector",
|
||||
# Baseline ~0.49 on 200 questions (measured on GB200).
|
||||
accuracy_threshold=0.39,
|
||||
cpu_offload_gib=1,
|
||||
),
|
||||
OffloadingModelConfig(
|
||||
id="offloading-gemma-4-e4b-it",
|
||||
model="google/gemma-4-E4B-it",
|
||||
connector="OffloadingConnector",
|
||||
# Baseline ~0.64 on 200 questions (measured on GB200).
|
||||
accuracy_threshold=0.55,
|
||||
cpu_offload_gib=1,
|
||||
),
|
||||
OffloadingModelConfig(
|
||||
id="offloading-qwen3.5-35b",
|
||||
model="Qwen/Qwen3.5-35B-A3B",
|
||||
connector="OffloadingConnector",
|
||||
accuracy_threshold=0.75,
|
||||
extra_server_args=[
|
||||
"--tensor-parallel-size",
|
||||
"2",
|
||||
"--enable-expert-parallel",
|
||||
],
|
||||
startup_timeout=1200,
|
||||
),
|
||||
OffloadingModelConfig(
|
||||
id="offloading-deepseek-v4-flash",
|
||||
model="deepseek-ai/DeepSeek-V4-Flash",
|
||||
connector="OffloadingConnector",
|
||||
# Baseline ~0.97 on 200 questions (measured on GB200).
|
||||
accuracy_threshold=0.90,
|
||||
extra_server_args=[
|
||||
"--tensor-parallel-size",
|
||||
"4",
|
||||
"--enable-expert-parallel",
|
||||
"--kv-cache-dtype",
|
||||
"fp8",
|
||||
"--block-size",
|
||||
"256",
|
||||
],
|
||||
cpu_offload_gib=16,
|
||||
startup_timeout=1200,
|
||||
),
|
||||
OffloadingModelConfig(
|
||||
id="offloading-deepseek-v2-lite-tiering-tp2",
|
||||
model="deepseek-ai/DeepSeek-V2-Lite",
|
||||
connector="OffloadingConnector",
|
||||
# Baseline 0.360/0.345 over two runs (measured on 2xA100).
|
||||
accuracy_threshold=0.35,
|
||||
extra_server_args=[
|
||||
"--tensor-parallel-size",
|
||||
"2",
|
||||
],
|
||||
cpu_offload_gib=8,
|
||||
startup_timeout=1200,
|
||||
spec_name="TieringOffloadingSpec",
|
||||
),
|
||||
# ── SimpleCPUOffloadConnector ────────────────────────────────────
|
||||
OffloadingModelConfig(
|
||||
id="simple-nemotron-h-8b",
|
||||
model="nvidia/Nemotron-H-8B-Base-8K",
|
||||
connector="SimpleCPUOffloadConnector",
|
||||
accuracy_threshold=0.43 if current_platform.is_rocm() else 0.45,
|
||||
),
|
||||
OffloadingModelConfig(
|
||||
id="simple-gemma-4-e4b-it",
|
||||
model="google/gemma-4-E4B-it",
|
||||
connector="SimpleCPUOffloadConnector",
|
||||
accuracy_threshold=0.55,
|
||||
),
|
||||
OffloadingModelConfig(
|
||||
id="simple-qwen3.5-35b",
|
||||
model="Qwen/Qwen3.5-35B-A3B",
|
||||
connector="SimpleCPUOffloadConnector",
|
||||
accuracy_threshold=0.75,
|
||||
extra_server_args=[
|
||||
"--tensor-parallel-size",
|
||||
"2",
|
||||
"--enable-expert-parallel",
|
||||
],
|
||||
startup_timeout=1200,
|
||||
),
|
||||
OffloadingModelConfig(
|
||||
id="simple-deepseek-v4-flash",
|
||||
model="deepseek-ai/DeepSeek-V4-Flash",
|
||||
connector="SimpleCPUOffloadConnector",
|
||||
accuracy_threshold=0.90,
|
||||
extra_server_args=[
|
||||
"--tensor-parallel-size",
|
||||
"4",
|
||||
"--enable-expert-parallel",
|
||||
"--kv-cache-dtype",
|
||||
"fp8",
|
||||
"--block-size",
|
||||
"256",
|
||||
],
|
||||
cpu_offload_gib=16,
|
||||
startup_timeout=1200,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cfg", MODELS, ids=lambda c: c.id)
|
||||
def test_gsm8k_offloading_correctness(cfg: OffloadingModelConfig):
|
||||
if "--tensor-parallel-size" in cfg.extra_server_args:
|
||||
tp_size = int(
|
||||
cfg.extra_server_args[
|
||||
cfg.extra_server_args.index("--tensor-parallel-size") + 1
|
||||
]
|
||||
)
|
||||
if current_platform.device_count() < tp_size:
|
||||
pytest.skip(f"Requires {tp_size} GPUs")
|
||||
|
||||
# Prefix caching must be explicitly enabled: SimpleCPUOffloadConnector requires it.
|
||||
server_args = [
|
||||
"--enforce-eager",
|
||||
"--max-model-len",
|
||||
"4096",
|
||||
"--enable-prefix-caching",
|
||||
"--no-disable-hybrid-kv-cache-manager",
|
||||
"--kv-transfer-config",
|
||||
_kv_transfer_config(cfg.connector, cfg.cpu_offload_gib, cfg.spec_name),
|
||||
"--trust-remote-code",
|
||||
"--disable-uvicorn-access-log",
|
||||
*cfg.extra_server_args,
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
cfg.model,
|
||||
server_args,
|
||||
# /reset_prefix_cache requires dev mode.
|
||||
env_dict={"VLLM_SERVER_DEV_MODE": "1", **cfg.env_dict},
|
||||
max_wait_seconds=cfg.startup_timeout,
|
||||
) as server:
|
||||
base_url = f"http://{server.host}:{server.port}"
|
||||
|
||||
for run_idx in range(1, 3):
|
||||
results = evaluate_gsm8k(
|
||||
num_questions=NUM_QUESTIONS,
|
||||
num_shots=NUM_FEWSHOT,
|
||||
host=f"http://{server.host}",
|
||||
port=server.port,
|
||||
)
|
||||
|
||||
print(
|
||||
f"GSM8K run {run_idx}/2 + {cfg.connector} ({cfg.id}): "
|
||||
f"accuracy={results['accuracy']:.4f}, "
|
||||
f"invalid_rate={results['invalid_rate']:.3f}, "
|
||||
f"latency={results['latency']:.1f}s"
|
||||
)
|
||||
|
||||
assert results["accuracy"] >= (cfg.accuracy_threshold - cfg.tolerance), (
|
||||
f"GSM8K run {run_idx}/2 accuracy "
|
||||
f"{results['accuracy']:.4f} below "
|
||||
f"{cfg.accuracy_threshold - cfg.tolerance:.4f}"
|
||||
)
|
||||
|
||||
if run_idx == 1:
|
||||
_reset_gpu_prefix_cache(base_url)
|
||||
Reference in New Issue
Block a user