[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests

- Drop dead model modules, keeping files live imports require
- Remove anthropic, cohere, codegen entrypoints and batch STT modes
- Delete unused attention backends, MoE experts, and NPU/XPU util leaves
- Strip dead third_party trees and ROCm leaves with no CUDA path
- Remove mooncake EC connectors: dependency missing since branch base
- Re-audit verified by full import sweep and Qwen3.5 smoke test

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 00:20:48 +07:00
co-authored by Hinata
parent 6a95f62e2f
commit 81074e94a3
940 changed files with 11 additions and 380533 deletions
-507
View File
@@ -1,507 +0,0 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Isolated GSM8K evaluation script for vLLM serve endpoint."""
import argparse
import ast
import asyncio
import json
import os
import tempfile
import time
from collections.abc import Generator
import aiohttp
import numpy as np
import regex as re
import requests
from tqdm.asyncio import tqdm
from vllm.assets.base import VLLM_S3_BUCKET_URL
INVALID = -9999999
def download_and_cache_file(url: str, filename: str | None = None) -> str:
"""Download and cache a file from a URL."""
if filename is None:
filename = os.path.join(tempfile.gettempdir(), url.split("/")[-1])
if os.path.exists(filename):
return filename
print(f"Downloading from {url} to {filename}")
response = requests.get(url, stream=True)
response.raise_for_status()
with open(filename, "wb") as f:
for chunk in response.iter_content(chunk_size=1024):
f.write(chunk)
return filename
def load_gsm8k_data() -> tuple[list[dict], list[dict]]:
"""Load GSM8K train and test data."""
train_url = f"{VLLM_S3_BUCKET_URL}/ci-datasets/gsm8k/train.jsonl"
test_url = f"{VLLM_S3_BUCKET_URL}/ci-datasets/gsm8k/test.jsonl"
train_file = download_and_cache_file(train_url)
test_file = download_and_cache_file(test_url)
train_data = list(read_jsonl(train_file))
test_data = list(read_jsonl(test_file))
return train_data, test_data
def read_jsonl(filename: str) -> Generator[dict, None, None]:
"""Read a JSONL file."""
with open(filename) as fin:
for line in fin:
if not line.startswith("#"):
yield json.loads(line)
def get_answer_value(answer_str: str) -> int:
"""Extract the numerical answer from the response."""
answer_str = answer_str.replace(",", "")
numbers = re.findall(r"\d+", answer_str)
if len(numbers) < 1:
return INVALID
try:
return ast.literal_eval(numbers[-1])
except SyntaxError:
return INVALID
def _optional_params(**params: object) -> dict[str, object]:
return {k: v for k, v in params.items() if v is not None}
async def call_vllm_api(
session: aiohttp.ClientSession,
prompt: str,
temperature: float | None,
max_tokens: int,
stop: list[str] | None = None,
url: str | None = None,
seed: int | None = None,
top_p: float | None = None,
top_k: int | None = None,
) -> tuple[str, int]:
"""Call vLLM's OpenAI-compatible completions endpoint.
Returns:
Tuple of (response_text, completion_tokens)
"""
data = {
"prompt": prompt,
"max_tokens": max_tokens,
"stop": stop,
}
data.update(
_optional_params(temperature=temperature, seed=seed, top_p=top_p, top_k=top_k)
)
try:
async with session.post(f"{url}/v1/completions", json=data) as response:
response.raise_for_status()
result = await response.json()
text = result["choices"][0]["text"]
completion_tokens = result.get("usage", {}).get("completion_tokens", 0)
return text, completion_tokens
except Exception as e:
print(f"Error calling vLLM API ({type(e).__name__}): {e}")
return "", 0
async def call_vllm_chat_api(
session: aiohttp.ClientSession,
model: str | None,
prompt: str,
temperature: float | None,
max_tokens: int,
stop: list[str] | None = None,
url: str | None = None,
seed: int | None = None,
top_p: float | None = None,
top_k: int | None = None,
reasoning_effort: str | None = None,
chat_template_kwargs: dict[str, object] | None = None,
) -> tuple[str, int]:
"""Call vLLM's OpenAI-compatible chat completions endpoint.
``model``, ``temperature``, ``stop`` and the other optional fields are
omitted from the request when ``None``, so the server defaults apply (vLLM
uses its served model when ``model`` is omitted).
Returns:
Tuple of (final answer content, completion_tokens). Reasoning returned
separately by a reasoning parser is not part of the answer.
"""
data = {
"messages": [{"role": "user", "content": prompt}],
"max_tokens": max_tokens,
}
data.update(
_optional_params(
stop=stop,
model=model,
temperature=temperature,
seed=seed,
top_p=top_p,
top_k=top_k,
reasoning_effort=reasoning_effort,
chat_template_kwargs=chat_template_kwargs,
)
)
try:
async with session.post(f"{url}/v1/chat/completions", json=data) as response:
response.raise_for_status()
result = await response.json()
text = result["choices"][0]["message"]["content"] or ""
completion_tokens = result.get("usage", {}).get("completion_tokens", 0)
return text, completion_tokens
except Exception as e:
print(f"Error calling vLLM chat API ({type(e).__name__}): {e}")
return "", 0
def _build_gsm8k_prompts(
num_questions: int = 1319,
num_shots: int = 5,
gen_prefix: str = "",
) -> tuple[list[str], list[int]]:
"""Build few-shot GSM8K completion prompts and ground-truth labels."""
if num_questions == 0:
return [], []
train_data, test_data = load_gsm8k_data()
num_questions = min(num_questions, len(test_data))
few_shot_examples = ""
for i in range(num_shots):
few_shot_examples += (
f"Question: {train_data[i]['question']}\n"
f"Answer:{gen_prefix} {train_data[i]['answer']}\n\n"
)
prompts = []
labels = []
for i in range(num_questions):
prompts.append(
few_shot_examples
+ f"Question: {test_data[i]['question']}\nAnswer:{gen_prefix}"
)
labels.append(get_answer_value(test_data[i]["answer"]))
assert all(label != INVALID for label in labels), "Some labels are invalid"
return prompts, labels
def _score_gsm8k(
states: list[str],
output_tokens: list[int],
labels: list[int],
num_shots: int,
max_tokens: int,
latency: float,
) -> dict[str, float | int]:
"""Score GSM8K responses and return a results dict."""
num_questions = len(labels)
preds = [get_answer_value(state) for state in states]
accuracy = np.mean(np.array(preds) == np.array(labels))
invalid_rate = np.mean(np.array(preds) == INVALID)
total_output_tokens = sum(output_tokens)
tokens_per_second = total_output_tokens / latency if latency > 0 else 0.0
return {
"accuracy": accuracy,
"invalid_rate": invalid_rate,
"latency": latency,
"questions_per_second": num_questions / latency if latency > 0 else 0.0,
"total_output_tokens": total_output_tokens,
"tokens_per_second": tokens_per_second,
"num_questions": num_questions,
"num_shots": num_shots,
"max_tokens": max_tokens,
"timestamp": time.time(),
}
def evaluate_gsm8k(
num_questions: int = 1319,
num_shots: int = 5,
max_tokens: int = 256,
model: str | None = None,
use_chat_completions: bool = False,
host: str = "http://127.0.0.1",
port: int = 8000,
temperature: float | None = 0.0,
seed: int | None = 42,
request_timeout_seconds: float = 600,
gen_prefix: str = "",
max_concurrency: int | None = None,
top_p: float | None = None,
top_k: int | None = None,
reasoning_effort: str | None = None,
chat_template_kwargs: dict[str, object] | None = None,
) -> dict[str, float | int]:
"""Evaluate GSM8K accuracy using vLLM serve endpoint.
``temperature``, ``top_p`` and ``top_k`` are sent only when not ``None``;
otherwise the server default applies. ``model`` is used only in chat mode,
where it may be ``None`` (the server uses its served model).
``reasoning_effort`` and ``chat_template_kwargs`` require
``use_chat_completions=True``. Stop strings are sent only in completions
mode.
Returns dict with accuracy, invalid_rate, latency, etc.
"""
if not use_chat_completions and (
reasoning_effort is not None or chat_template_kwargs is not None
):
raise ValueError(
"reasoning_effort and chat_template_kwargs require "
"use_chat_completions=True"
)
base_url = f"{host}:{port}"
prompts, labels = _build_gsm8k_prompts(num_questions, num_shots, gen_prefix)
num_questions = len(prompts)
async def run_async_evaluation():
states: list[str] = [""] * num_questions
output_tokens: list[int] = [0] * num_questions
async def get_answer(session: aiohttp.ClientSession, i: int) -> tuple[str, int]:
if use_chat_completions:
# No stop strings: the chat template delimits the turn, and they
# would also cut off reasoning.
answer, tokens = await call_vllm_chat_api(
session=session,
model=model,
prompt=prompts[i],
temperature=temperature,
max_tokens=max_tokens,
url=base_url,
seed=seed,
top_p=top_p,
top_k=top_k,
reasoning_effort=reasoning_effort,
chat_template_kwargs=chat_template_kwargs,
)
else:
answer, tokens = await call_vllm_api(
session=session,
prompt=prompts[i],
temperature=temperature,
max_tokens=max_tokens,
stop=["Question", "Assistant:", "<|separator|>"],
url=base_url,
seed=seed,
top_p=top_p,
top_k=top_k,
)
states[i] = answer
output_tokens[i] = tokens
return answer, tokens
timeout = aiohttp.ClientTimeout(total=request_timeout_seconds)
connector = (
aiohttp.TCPConnector(limit=max_concurrency)
if max_concurrency is not None
else None
)
async with aiohttp.ClientSession(
timeout=timeout, connector=connector
) as session:
tasks = [get_answer(session, i) for i in range(num_questions)]
await tqdm.gather(*tasks, desc="Evaluating")
return states, output_tokens
print(f"Running GSM8K evaluation: {num_questions} questions, {num_shots}-shot")
tic = time.perf_counter()
states, output_tokens = asyncio.run(run_async_evaluation())
latency = time.perf_counter() - tic
return _score_gsm8k(states, output_tokens, labels, num_shots, max_tokens, latency)
def evaluate_gsm8k_offline(
llm,
num_questions: int = 1319,
num_shots: int = 5,
max_tokens: int = 256,
temperature: float = 0.0,
gen_prefix: str = "",
use_chat_completions: bool = False,
chat_template_kwargs: dict[str, object] | None = None,
) -> dict[str, float | int]:
"""Evaluate GSM8K accuracy using an offline vllm.LLM object.
Same prompts and scoring as evaluate_gsm8k(), but runs generation
directly via llm.generate() instead of calling a server over HTTP.
When ``use_chat_completions=True``, prompts go through the chat template via
``llm.chat()`` instead of raw completion (for instruction-tuned models).
``chat_template_kwargs`` are forwarded to ``llm.chat()`` when provided.
"""
from vllm import SamplingParams
prompts, labels = _build_gsm8k_prompts(num_questions, num_shots, gen_prefix)
sampling_params = SamplingParams(
temperature=temperature,
max_tokens=max_tokens,
stop=["Question", "Assistant:", "<|separator|>"],
)
mode = "chat" if use_chat_completions else "completion"
print(
f"Running offline GSM8K evaluation: {len(prompts)} questions, "
f"{num_shots}-shot, {mode}"
)
tic = time.perf_counter()
if use_chat_completions:
conversations = [[{"role": "user", "content": p}] for p in prompts]
outputs = llm.chat(
conversations,
sampling_params,
chat_template_kwargs=chat_template_kwargs,
)
else:
outputs = llm.generate(prompts, sampling_params)
latency = time.perf_counter() - tic
states = [o.outputs[0].text for o in outputs]
output_tokens = [len(o.outputs[0].token_ids) for o in outputs]
return _score_gsm8k(states, output_tokens, labels, num_shots, max_tokens, latency)
def main() -> None:
parser = argparse.ArgumentParser(description="GSM8K evaluation for vLLM serve")
parser.add_argument(
"--num-shots", type=int, default=5, help="Number of few-shot examples"
)
parser.add_argument(
"--num-questions",
type=int,
default=1319,
help="Number of questions to evaluate",
)
parser.add_argument(
"--max-tokens", type=int, default=256, help="Max tokens for generation"
)
parser.add_argument("--host", type=str, default="http://127.0.0.1", help="Host URL")
parser.add_argument("--port", type=int, default=8000, help="Port number")
parser.add_argument(
"--temperature",
type=float,
help="Temperature for generation (default: 0 for completions; "
"server default with --use-chat-completions)",
)
parser.add_argument(
"--top-p", type=float, help="Top-p for generation (server default if unset)"
)
parser.add_argument(
"--top-k", type=int, help="Top-k for generation (server default if unset)"
)
parser.add_argument(
"--use-chat-completions",
action="store_true",
help="Send each few-shot prompt as a user message to /v1/chat/completions "
"instead of /v1/completions (for chat-only models)",
)
parser.add_argument(
"--model",
type=str,
help="Model name to send with chat requests (default: omitted, so the "
"server uses its served model)",
)
parser.add_argument(
"--reasoning-effort",
type=str,
help="reasoning_effort for chat completions, e.g. none, low, high",
)
parser.add_argument(
"--chat-template-kwargs",
type=json.loads,
help="JSON chat_template_kwargs, e.g. '{\"enable_thinking\": false}'",
)
parser.add_argument(
"--seed", type=int, default=42, help="Random seed for reproducibility"
)
parser.add_argument(
"--max-concurrency",
type=int,
help="Maximum number of concurrent requests",
)
parser.add_argument(
"--request-timeout-seconds",
type=float,
default=600,
help="Timeout for each request, including time waiting for a connection",
)
parser.add_argument("--save-results", type=str, help="Save results to JSON file")
args = parser.parse_args()
temperature = args.temperature
if temperature is None and not args.use_chat_completions:
temperature = 0.0
result = evaluate_gsm8k(
num_questions=args.num_questions,
num_shots=args.num_shots,
max_tokens=args.max_tokens,
host=args.host,
port=args.port,
temperature=temperature,
seed=args.seed,
max_concurrency=args.max_concurrency,
request_timeout_seconds=args.request_timeout_seconds,
model=args.model,
use_chat_completions=args.use_chat_completions,
top_p=args.top_p,
top_k=args.top_k,
reasoning_effort=args.reasoning_effort,
chat_template_kwargs=args.chat_template_kwargs,
)
# Print results to terminal
print("\nResults:")
print(f"Accuracy: {result['accuracy']:.3f}")
print(f"Invalid responses: {result['invalid_rate']:.3f}")
print(f"Total latency: {result['latency']:.3f} s")
print(f"Questions per second: {result['questions_per_second']:.3f}")
print(f"Total output tokens: {result['total_output_tokens']}")
print(f"Output tokens per second: {result['tokens_per_second']:.3f}")
# Optional file saving
if args.save_results:
# None means the field was not sent and the server default applied.
result["request_params"] = {
"endpoint": "chat" if args.use_chat_completions else "completions",
"model": args.model,
"temperature": temperature,
"top_p": args.top_p,
"top_k": args.top_k,
"seed": args.seed,
"stop": None
if args.use_chat_completions
else ["Question", "Assistant:", "<|separator|>"],
"reasoning_effort": args.reasoning_effort,
"chat_template_kwargs": args.chat_template_kwargs,
}
with open(args.save_results, "w") as f:
json.dump(result, f, indent=2)
print(f"Results saved to {args.save_results}")
if __name__ == "__main__":
main()
-213
View File
@@ -1,213 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""GSM8K evaluation using vLLM server and isolated GSM8K script.
Replacement for lm-eval-harness with better performance and control.
Usage:
pytest -s -v tests/evals/gsm8k/test_gsm8k_correctness.py \
--config-list-file=configs/models-small.txt
"""
import shlex
import pytest
import requests
import yaml
from tests.utils import RemoteOpenAIServer
from vllm.platforms import current_platform
from .gsm8k_eval import evaluate_gsm8k
DEFAULT_STARTUP_MAX_WAIT_SECONDS = 1200
def run_gsm8k_eval(eval_config: dict, server_url: str) -> dict:
"""Run GSM8K evaluation using our isolated script."""
# Extract host and port from server URL
if "://" in server_url:
server_url = server_url.split("://")[1]
host_port = server_url.split("/")[0] # Remove path if present
if ":" in host_port:
host, p = host_port.split(":")
port = int(p)
else:
host = host_port
port = 8000
# Add http:// prefix if not present
if not host.startswith("http"):
host = f"http://{host}"
# Run GSM8K evaluation
request_timeout_seconds = eval_config.get("request_timeout_seconds", 600)
if current_platform.is_rocm():
request_timeout_seconds = eval_config.get(
"rocm_request_timeout_seconds", request_timeout_seconds
)
results = evaluate_gsm8k(
num_questions=eval_config["num_questions"],
num_shots=eval_config["num_fewshot"],
max_tokens=eval_config.get("max_tokens", 256),
model=eval_config["model_name"],
use_chat_completions=eval_config.get("use_chat_completions", False),
host=host,
port=port,
temperature=eval_config.get("temperature", 0.0),
seed=eval_config.get("seed", 42),
request_timeout_seconds=request_timeout_seconds,
gen_prefix=eval_config.get("gen_prefix", ""),
max_concurrency=eval_config.get("max_concurrency"),
)
return results
def get_acceptance_length(server_url: str) -> float:
"""Mean tokens emitted per verification step, from the server's counters.
1.0 means every draft was rejected (speculation bought nothing); the
theoretical maximum is 1 + num_speculative_tokens.
"""
response = requests.get(f"{server_url.rstrip('/').removesuffix('/v1')}/metrics")
response.raise_for_status()
counters: dict[str, float] = {}
for line in response.text.splitlines():
if line.startswith("vllm:spec_decode_num_"):
name, _, value = line.partition(" ")
counters[name.split("{")[0]] = float(value)
num_drafts = counters.get("vllm:spec_decode_num_drafts_total", 0.0)
num_accepted = counters.get("vllm:spec_decode_num_accepted_tokens_total", 0.0)
assert num_drafts > 0, (
"no drafts recorded; speculative decoding did not run for this config"
)
return 1.0 + num_accepted / num_drafts
def test_gsm8k_correctness(config_filename):
"""Test GSM8K correctness for a given model configuration."""
eval_config = yaml.safe_load(config_filename.read_text(encoding="utf-8"))
if (
not current_platform.is_cuda()
and "Qwen3-30B-A3B-MXFP4A16" in eval_config["model_name"]
):
pytest.skip(
"Skipping Qwen3-30B-A3B-MXFP4A16 on non-CUDA platforms. "
"Marlin kernels are not supported."
)
if (
not current_platform.is_cuda()
and "gemma-4-E4B-it-qat-mobile-ct" in eval_config["model_name"]
):
pytest.skip(
"Skipping gemma-4-E4B-it-qat-mobile-ct on non-CUDA platforms. "
"Its W2A16 (uint2b2) scheme has no kernel outside CUDA."
)
if current_platform.is_rocm() and (
"deepseek-ai/DeepSeek-V3.2" in eval_config["model_name"]
or "deepseek-ai/DeepSeek-R1" in eval_config["model_name"]
):
from vllm.platforms.rocm import get_cdna_version
if get_cdna_version() < 4:
pytest.skip(
"Skipping DeepSeek-V3.2 and DeepSeek-R1 on ROCm platforms below "
"CDNA 4 due to agent pool disk space issues and pod evictions."
)
if current_platform.is_rocm() and (
"Qwen3.5-35B-A3B-MXFP4-AITER-TP2" in config_filename.name
):
from vllm.platforms.rocm import on_gfx950
if not on_gfx950():
pytest.skip(
"Skipping Qwen3.5-35B-A3B-MXFP4-AITER-TP2 on non-GFX950 platforms. "
"The quantization scheme is not supported on non-GFX950 platforms."
)
if "--all2all-backend=moonep" in eval_config.get("server_args", ""):
from vllm.utils.import_utils import has_moonep
if not has_moonep():
pytest.skip("Skipping MoonEP config: the moonep package is not installed.")
# Parse server arguments from config (use shlex to handle quoted strings)
server_args_str = eval_config.get("server_args", "")
server_args = shlex.split(server_args_str) if server_args_str else []
# Add standard server arguments
server_args.extend(
[
"--trust-remote-code",
"--disable-uvicorn-access-log",
]
)
startup_max_wait_seconds = eval_config.get(
"startup_max_wait_seconds", DEFAULT_STARTUP_MAX_WAIT_SECONDS
)
env_dict = dict(eval_config.get("env") or {})
env_dict["VLLM_ENGINE_READY_TIMEOUT_S"] = str(int(startup_max_wait_seconds))
print(f"Starting GSM8K evaluation for model: {eval_config['model_name']}")
print(f"Expected metric threshold: {eval_config['accuracy_threshold']}")
print(f"Number of questions: {eval_config['num_questions']}")
print(f"Number of few-shot examples: {eval_config['num_fewshot']}")
request_timeout_seconds = eval_config.get("request_timeout_seconds", 600)
if current_platform.is_rocm():
request_timeout_seconds = eval_config.get(
"rocm_request_timeout_seconds", request_timeout_seconds
)
print(f"Request timeout: {request_timeout_seconds}s")
print(f"Startup max wait: {startup_max_wait_seconds}s")
print(f"Server args: {' '.join(server_args)}")
print(f"Environment variables: {env_dict}")
# Launch server and run evaluation
with RemoteOpenAIServer(
eval_config["model_name"],
server_args,
env_dict=env_dict,
max_wait_seconds=startup_max_wait_seconds,
) as remote_server:
server_url = remote_server.url_for("v1")
print(f"Server started at: {server_url}")
results = run_gsm8k_eval(eval_config, server_url)
measured_metric = results["accuracy"]
expected_metric = eval_config["accuracy_threshold"]
tol = eval_config.get("tolerance", 0.08)
print(f"GSM8K Results for {eval_config['model_name']}:")
print(f" Measured metric: {measured_metric:.4f}")
print(f" Expected metric: {expected_metric:.4f}")
print(f" Tolerance: {tol:.4f}")
print(f" Questions: {results['num_questions']}")
print(f" Invalid rate: {results['invalid_rate']:.3f}")
print(f" Latency: {results['latency']:.1f}s")
print(f" QPS: {results['questions_per_second']:.1f}")
assert measured_metric >= expected_metric - tol, (
f"GSM8K metric too low: {measured_metric:.4f} < "
f"{expected_metric:.4f} - {tol:.4f} = {expected_metric - tol:.4f}"
)
# Speculative configs additionally assert that drafts are actually
# landing: accuracy alone passes even when every draft is rejected.
min_acceptance_length = eval_config.get("min_acceptance_length")
if min_acceptance_length is not None:
acceptance_length = get_acceptance_length(server_url)
print(f" Mean acceptance length: {acceptance_length:.3f}")
print(f" Minimum acceptance length: {min_acceptance_length:.3f}")
assert acceptance_length >= min_acceptance_length, (
f"Acceptance length too low: {acceptance_length:.3f} < "
f"{min_acceptance_length:.3f}"
)
print(f"✅ GSM8K test passed for {eval_config['model_name']}")
-290
View File
@@ -1,290 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""GSM8K correctness test for CPU KV offloading connectors.
Regression guard for stride computation bugs in the offloading worker
(e.g. https://github.com/vllm-project/vllm/pull/46888) and silent KV
cache data corruption during CPU offloading. Runs GSM8K twice, dropping
the GPU prefix cache (but not the CPU cache) between runs so the second
run reloads offloaded KV data from CPU. The reset only succeeds once
in-flight offload transfers have released their GPU blocks, so retrying
it until success also waits for offloading to complete.
Correctness is enforced via GSM8K accuracy on both runs: corrupt reloaded
KV drops accuracy below threshold. The reload itself is not asserted, so
a silently skipped reload (e.g. offloading disabled) would not be flagged.
Covers both KV offloading connectors (OffloadingConnector and
SimpleCPUOffloadConnector) across five architecture families:
- Hybrid Mamba (NemotronH: attention + Mamba)
- Heterogeneous head dim (Gemma 4)
- Hybrid GDN (Qwen 3.5: attention + GatedDeltaNet)
- Compressed attention (DeepSeek-V4-Flash: CSA)
- Pure MLA (DeepSeek-V2-Lite: TieringOffloadingSpec, TP=2, replicated
single-slot host layout)
Usage:
pytest -s -v evals/gsm8k/test_gsm8k_offloading.py
"""
import json
import time
from dataclasses import dataclass, field
import pytest
import requests
from tests.utils import RemoteOpenAIServer
from vllm.platforms import current_platform
from .gsm8k_eval import evaluate_gsm8k
if not current_platform.is_cuda_alike():
pytest.skip("Requires CUDA or ROCm", allow_module_level=True)
NUM_QUESTIONS = 200
NUM_FEWSHOT = 5
_OFFLOAD_SYNC_TIMEOUT = 60
def _kv_transfer_config(
connector: str, cpu_gib: int = 4, spec_name: str = "CPUOffloadingSpec"
) -> str:
if connector == "OffloadingConnector":
return json.dumps(
{
"kv_connector": "OffloadingConnector",
"kv_role": "kv_both",
"kv_connector_extra_config": {
"spec_name": spec_name,
"cpu_bytes_to_use": cpu_gib << 30,
"eviction_policy": "lru",
},
}
)
elif connector == "SimpleCPUOffloadConnector":
return json.dumps(
{
"kv_connector": "SimpleCPUOffloadConnector",
"kv_role": "kv_both",
"kv_connector_extra_config": {
"cpu_bytes_to_use": cpu_gib << 30,
},
}
)
else:
raise ValueError(f"Unknown connector: {connector}")
def _force_engine_step(base_url: str) -> None:
"""Force an engine step so completed offload transfers get processed."""
requests.post(
f"{base_url}/v1/completions",
json={"prompt": "0", "max_tokens": 1, "temperature": 0},
timeout=60,
).raise_for_status()
def _reset_gpu_prefix_cache(base_url: str) -> None:
"""Drop the GPU prefix cache while keeping the CPU (connector) cache, so
the next run must reload KV data through the connector.
The reset fails while asynchronous offload transfers still hold GPU
blocks, so retry until it succeeds. Requires VLLM_SERVER_DEV_MODE=1.
"""
deadline = time.monotonic() + _OFFLOAD_SYNC_TIMEOUT
while True:
resp = requests.post(
f"{base_url}/reset_prefix_cache",
params={"reset_external": "false"},
timeout=30,
)
resp.raise_for_status()
if resp.json().get("success"):
return
assert time.monotonic() < deadline, (
f"prefix cache reset did not succeed within {_OFFLOAD_SYNC_TIMEOUT}s; "
"async offload may be stuck"
)
_force_engine_step(base_url)
@dataclass
class OffloadingModelConfig:
id: str
model: str
connector: str
accuracy_threshold: float
tolerance: float = 0.05
extra_server_args: list[str] = field(default_factory=list)
env_dict: dict[str, str] = field(default_factory=dict)
cpu_offload_gib: int = 4
startup_timeout: int = 600
spec_name: str = "CPUOffloadingSpec"
MODELS = [
# ── OffloadingConnector ──────────────────────────────────────────
OffloadingModelConfig(
id="offloading-nemotron-h-8b",
model="nvidia/Nemotron-H-8B-Base-8K",
connector="OffloadingConnector",
# Baseline ~0.49 on 200 questions (measured on GB200).
accuracy_threshold=0.39,
cpu_offload_gib=1,
),
OffloadingModelConfig(
id="offloading-gemma-4-e4b-it",
model="google/gemma-4-E4B-it",
connector="OffloadingConnector",
# Baseline ~0.64 on 200 questions (measured on GB200).
accuracy_threshold=0.55,
cpu_offload_gib=1,
),
OffloadingModelConfig(
id="offloading-qwen3.5-35b",
model="Qwen/Qwen3.5-35B-A3B",
connector="OffloadingConnector",
accuracy_threshold=0.75,
extra_server_args=[
"--tensor-parallel-size",
"2",
"--enable-expert-parallel",
],
startup_timeout=1200,
),
OffloadingModelConfig(
id="offloading-deepseek-v4-flash",
model="deepseek-ai/DeepSeek-V4-Flash",
connector="OffloadingConnector",
# Baseline ~0.97 on 200 questions (measured on GB200).
accuracy_threshold=0.90,
extra_server_args=[
"--tensor-parallel-size",
"4",
"--enable-expert-parallel",
"--kv-cache-dtype",
"fp8",
"--block-size",
"256",
],
cpu_offload_gib=16,
startup_timeout=1200,
),
OffloadingModelConfig(
id="offloading-deepseek-v2-lite-tiering-tp2",
model="deepseek-ai/DeepSeek-V2-Lite",
connector="OffloadingConnector",
# Baseline 0.360/0.345 over two runs (measured on 2xA100).
accuracy_threshold=0.35,
extra_server_args=[
"--tensor-parallel-size",
"2",
],
cpu_offload_gib=8,
startup_timeout=1200,
spec_name="TieringOffloadingSpec",
),
# ── SimpleCPUOffloadConnector ────────────────────────────────────
OffloadingModelConfig(
id="simple-nemotron-h-8b",
model="nvidia/Nemotron-H-8B-Base-8K",
connector="SimpleCPUOffloadConnector",
accuracy_threshold=0.43 if current_platform.is_rocm() else 0.45,
),
OffloadingModelConfig(
id="simple-gemma-4-e4b-it",
model="google/gemma-4-E4B-it",
connector="SimpleCPUOffloadConnector",
accuracy_threshold=0.55,
),
OffloadingModelConfig(
id="simple-qwen3.5-35b",
model="Qwen/Qwen3.5-35B-A3B",
connector="SimpleCPUOffloadConnector",
accuracy_threshold=0.75,
extra_server_args=[
"--tensor-parallel-size",
"2",
"--enable-expert-parallel",
],
startup_timeout=1200,
),
OffloadingModelConfig(
id="simple-deepseek-v4-flash",
model="deepseek-ai/DeepSeek-V4-Flash",
connector="SimpleCPUOffloadConnector",
accuracy_threshold=0.90,
extra_server_args=[
"--tensor-parallel-size",
"4",
"--enable-expert-parallel",
"--kv-cache-dtype",
"fp8",
"--block-size",
"256",
],
cpu_offload_gib=16,
startup_timeout=1200,
),
]
@pytest.mark.parametrize("cfg", MODELS, ids=lambda c: c.id)
def test_gsm8k_offloading_correctness(cfg: OffloadingModelConfig):
if "--tensor-parallel-size" in cfg.extra_server_args:
tp_size = int(
cfg.extra_server_args[
cfg.extra_server_args.index("--tensor-parallel-size") + 1
]
)
if current_platform.device_count() < tp_size:
pytest.skip(f"Requires {tp_size} GPUs")
# Prefix caching must be explicitly enabled: SimpleCPUOffloadConnector requires it.
server_args = [
"--enforce-eager",
"--max-model-len",
"4096",
"--enable-prefix-caching",
"--no-disable-hybrid-kv-cache-manager",
"--kv-transfer-config",
_kv_transfer_config(cfg.connector, cfg.cpu_offload_gib, cfg.spec_name),
"--trust-remote-code",
"--disable-uvicorn-access-log",
*cfg.extra_server_args,
]
with RemoteOpenAIServer(
cfg.model,
server_args,
# /reset_prefix_cache requires dev mode.
env_dict={"VLLM_SERVER_DEV_MODE": "1", **cfg.env_dict},
max_wait_seconds=cfg.startup_timeout,
) as server:
base_url = f"http://{server.host}:{server.port}"
for run_idx in range(1, 3):
results = evaluate_gsm8k(
num_questions=NUM_QUESTIONS,
num_shots=NUM_FEWSHOT,
host=f"http://{server.host}",
port=server.port,
)
print(
f"GSM8K run {run_idx}/2 + {cfg.connector} ({cfg.id}): "
f"accuracy={results['accuracy']:.4f}, "
f"invalid_rate={results['invalid_rate']:.3f}, "
f"latency={results['latency']:.1f}s"
)
assert results["accuracy"] >= (cfg.accuracy_threshold - cfg.tolerance), (
f"GSM8K run {run_idx}/2 accuracy "
f"{results['accuracy']:.4f} below "
f"{cfg.accuracy_threshold - cfg.tolerance:.4f}"
)
if run_idx == 1:
_reset_gpu_prefix_cache(base_url)