[Mini-vLLM] SM86-only cut: models, backends, Rust, tests

- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries
- Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels
- Remove inc/quark quant loaders; keep compressed-tensors and Marlin
- Drop the Rust frontend, pooling entrypoints, and idle test suites
- Rebuild kernels for sm_80;sm_86 and strip the extensions
- Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-18 19:19:56 +07:00
co-authored by Hinata
parent c5ab42bc0f
commit e61dcb9143
2728 changed files with 159 additions and 950401 deletions
+6 -4
View File
@@ -15,10 +15,12 @@ from vllm import _custom_ops as ops
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.torch_utils import direct_register_custom_op
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
rocm_aiter_sparse_attn_indexer,
rocm_aiter_sparse_attn_indexer_fake,
)
if current_platform.is_rocm():
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
rocm_aiter_sparse_attn_indexer,
rocm_aiter_sparse_attn_indexer_fake,
)
if TYPE_CHECKING:
from aiter import ActivationType
View File
-88
View File
@@ -1,88 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.benchmarks.datasets.datasets import (
DEFAULT_NUM_PROMPTS,
AIMODataset,
ASRDataset,
BenchmarkDataset,
BFCLDataset,
BlazeditDataset,
BurstGPTDataset,
ConversationDataset,
CustomAudioDataset,
CustomDataset,
CustomImageDataset,
HuggingFaceDataset,
InstructCoderDataset,
MLPerfDataset,
MMStarDataset,
MMVUDataset,
MTBenchDataset,
MultiModalConversationDataset,
NextEditPredictionDataset,
PrefixRepetitionRandomDataset,
RandomDataset,
RandomDatasetForReranking,
RandomMultiModalDataset,
SampleRequest,
ShareGPTDataset,
SpecBench,
VisionArenaDataset,
add_dataset_parser,
add_random_dataset_base_args,
add_random_multimodal_dataset_args,
gen_prompt_decode_to_target_len,
get_samples,
is_valid_sequence,
lora_path_on_disk,
lora_tokenizer_cache,
process_audio,
process_image,
process_video,
zeta_prompt,
)
from vllm.benchmarks.datasets.utils import RangeRatio
__all__ = [
"DEFAULT_NUM_PROMPTS",
"AIMODataset",
"ASRDataset",
"BenchmarkDataset",
"BFCLDataset",
"BlazeditDataset",
"BurstGPTDataset",
"ConversationDataset",
"CustomDataset",
"CustomAudioDataset",
"CustomImageDataset",
"HuggingFaceDataset",
"InstructCoderDataset",
"MLPerfDataset",
"MMStarDataset",
"MMVUDataset",
"MTBenchDataset",
"MultiModalConversationDataset",
"NextEditPredictionDataset",
"PrefixRepetitionRandomDataset",
"RandomDataset",
"RandomDatasetForReranking",
"RandomMultiModalDataset",
"SampleRequest",
"ShareGPTDataset",
"SpecBench",
"VisionArenaDataset",
"add_dataset_parser",
"add_random_dataset_base_args",
"add_random_multimodal_dataset_args",
"gen_prompt_decode_to_target_len",
"get_samples",
"is_valid_sequence",
"lora_path_on_disk",
"lora_tokenizer_cache",
"process_audio",
"process_image",
"process_video",
"RangeRatio",
"zeta_prompt",
]
@@ -1,207 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Convert a plain-text file (local path or URL) into a JSONL dataset
compatible with ``CustomDataset`` (``--dataset-name custom``), by
randomly slicing the tokenized text into prompts.
Each line of the output JSONL contains a ``prompt`` (decoded from a random
slice of the tokenized source text) and an ``output_tokens`` count.
Usage
-----
::
python -m vllm.benchmarks.datasets.create_txt_slices_dataset \\
--input corpus.txt \\
--output dataset.jsonl \\
--tokenizer gpt2 \\
--num-prompts 1000 \\
--input-len 1024 \\
--output-len 128
The resulting JSONL file can then be used with the serving benchmark::
python -m vllm.benchmarks.serve \\
--dataset-name custom \\
--dataset-path dataset.jsonl \\
...
"""
from __future__ import annotations
import json
import logging
import random
import urllib.request
import numpy as np
from transformers import AutoTokenizer
from vllm.benchmarks.datasets.utils import RangeRatio, get_sampling_params
from vllm.utils.argparse_utils import FlexibleArgumentParser
logger = logging.getLogger(__name__)
def load_text(path: str) -> str:
"""Load text from a local file or URL."""
if path.startswith(("http://", "https://")):
with urllib.request.urlopen(path) as response:
return response.read().decode("utf-8")
with open(path, encoding="utf-8") as f:
return f.read()
def create_txt_slices_jsonl(
*,
input_path: str,
output_path: str,
tokenizer_name: str,
num_prompts: int,
input_len: int,
output_len: int,
range_ratio: RangeRatio = 0.0,
seed: int = 0,
trust_remote_code: bool = False,
) -> None:
"""Read *input_path*, slice it into prompts, and write JSONL to
*output_path*."""
tokenizer = AutoTokenizer.from_pretrained(
tokenizer_name, trust_remote_code=trust_remote_code
)
text = load_text(input_path)
if not text:
raise ValueError("The text file is empty and cannot be sampled from.")
token_ids = tokenizer(text, add_special_tokens=False).input_ids
if not token_ids:
raise ValueError("Tokenizing the text produced zero tokens; cannot sample.")
rng_np = np.random.default_rng(seed)
rng_py = random.Random(seed)
input_lens, output_lens, _ = get_sampling_params(
rng_np,
num_prompts,
range_ratio,
input_len,
output_len,
tokenizer,
)
num_available_tokens = len(token_ids)
records: list[dict[str, object]] = []
for i in range(num_prompts):
req_input_len = int(input_lens[i])
req_output_len = int(output_lens[i])
# Randomly select a start position and slice with cycling
start_pos = rng_py.randint(0, num_available_tokens - 1)
prompt_token_ids = [
token_ids[(start_pos + j) % num_available_tokens]
for j in range(req_input_len)
]
prompt = tokenizer.decode(prompt_token_ids, skip_special_tokens=False)
records.append({"prompt": prompt, "output_tokens": req_output_len})
with open(output_path, "w", encoding="utf-8") as f:
for record in records:
f.write(json.dumps(record, ensure_ascii=False) + "\n")
logger.info(
"Wrote %d prompts to %s",
len(records),
output_path,
)
def main(argv: list[str] | None = None) -> None:
parser = FlexibleArgumentParser(
description="Convert a plain-text file into a JSONL dataset "
"for CustomDataset (--dataset-name custom).",
)
parser.add_argument(
"--input",
required=True,
help="Path or URL to the source text file.",
)
parser.add_argument(
"--output",
required=True,
help="Path for the output JSONL file.",
)
parser.add_argument(
"--tokenizer",
required=True,
help="HuggingFace tokenizer name or path.",
)
parser.add_argument(
"--num-prompts",
type=int,
default=1000,
help="Number of prompt samples to generate (default: 1000).",
)
parser.add_argument(
"--input-len",
type=int,
default=1024,
help="Target number of input tokens per prompt (default: 1024).",
)
parser.add_argument(
"--output-len",
type=int,
default=128,
help="Target number of output tokens per prompt (default: 128).",
)
parser.add_argument(
"--range-ratio",
type=str,
default="0.0",
help="Range ratio for input/output length sampling (default: 0.0). "
"A single float applies to both ISL and OSL. "
'A JSON dict like \'{"input": 0.3, "output": 0.5}\' sets them '
"independently. Values must be in [0, 1).",
)
parser.add_argument(
"--seed",
type=int,
default=0,
help="Random seed for reproducibility (default: 0).",
)
parser.add_argument(
"--trust-remote-code",
action="store_true",
help="Trust remote code from HuggingFace.",
)
args = parser.parse_args(argv)
logging.basicConfig(level=logging.INFO)
# Parse --range-ratio: try float first, then JSON dict.
range_ratio: RangeRatio
try:
range_ratio = float(args.range_ratio)
except ValueError:
import json as _json
range_ratio = _json.loads(args.range_ratio)
create_txt_slices_jsonl(
input_path=args.input,
output_path=args.output,
tokenizer_name=args.tokenizer,
num_prompts=args.num_prompts,
input_len=args.input_len,
output_len=args.output_len,
range_ratio=range_ratio,
seed=args.seed,
trust_remote_code=args.trust_remote_code,
)
if __name__ == "__main__":
main()
File diff suppressed because it is too large Load Diff
-99
View File
@@ -1,99 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared utilities for benchmark dataset sampling."""
import logging
import math
import numpy as np
from vllm.tokenizers import TokenizerLike
logger = logging.getLogger(__name__)
# Type alias: a single float applies to both ISL and OSL; a dict allows
# specifying them independently via ``{"input": …, "output": …}``.
RangeRatio = float | dict[str, float]
def _resolve_range_ratios(
range_ratio: RangeRatio,
) -> tuple[float, float]:
"""Return ``(input_range_ratio, output_range_ratio)`` from *range_ratio*.
*range_ratio* is either a single float (used for both input and output)
or a dict with ``"input"`` and ``"output"`` keys.
"""
if isinstance(range_ratio, dict):
try:
return float(range_ratio["input"]), float(range_ratio["output"])
except KeyError as exc:
raise ValueError(
"When range_ratio is a dict it must contain 'input' and "
f"'output' keys, got: {sorted(range_ratio)}"
) from exc
ratio = float(range_ratio)
return ratio, ratio
def get_sampling_params(
rng: np.random.Generator,
num_requests: int,
range_ratio: RangeRatio,
input_len: int,
output_len: int,
tokenizer: TokenizerLike,
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
"""Sample per-request input/output token lengths and vocab offsets.
Lengths are drawn uniformly from integer ranges around the configured
means, controlled by *range_ratio*. It may be a single ``float``
(applied to both input and output) or a ``dict`` with ``"input"`` and
``"output"`` keys for independent control.
Tokenizer special tokens are subtracted from ``input_len`` before
computing the sampling interval.
Returns:
(input_lens, output_lens, offsets) – three 1-D ``np.ndarray`` of
shape ``(num_requests,)``.
"""
input_range_ratio, output_range_ratio = _resolve_range_ratios(range_ratio)
if not (0.0 <= input_range_ratio < 1.0):
raise ValueError("input_range_ratio must be in [0, 1).")
if not (0.0 <= output_range_ratio < 1.0):
raise ValueError("output_range_ratio must be in [0, 1).")
num_special_tokens = int(tokenizer.num_special_tokens_to_add())
real_input_len = max(0, int(input_len) - num_special_tokens)
input_low = math.floor(real_input_len * (1 - input_range_ratio))
input_high = math.ceil(real_input_len * (1 + input_range_ratio))
output_low = math.floor(output_len * (1 - output_range_ratio))
output_high = math.ceil(output_len * (1 + output_range_ratio))
# Ensure the lower bound for output length is at least 1 to
# prevent sampling 0 tokens.
output_low = max(output_low, 1)
output_high = max(output_high, 1)
if input_low > input_high:
raise ValueError(
f"Invalid input sampling interval: low={input_low} > high={input_high}"
)
if output_low > output_high:
raise ValueError(
f"Invalid output sampling interval: low={output_low} > high={output_high}"
)
logger.info(
"Sampling input_len from [%s, %s] and output_len from [%s, %s]",
input_low,
input_high,
output_low,
output_high,
)
input_lens = rng.integers(input_low, input_high + 1, size=num_requests)
output_lens = rng.integers(output_low, output_high + 1, size=num_requests)
offsets = rng.integers(0, tokenizer.vocab_size, size=num_requests)
return input_lens, output_lens, offsets
-2
View File
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-148
View File
@@ -1,148 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Build a `SchedulerOutput` for an exact batch, without the scheduler."""
from dataclasses import dataclass
import numpy as np
from vllm.benchmarks.forward.shapes import BenchCase
from vllm.sampling_params import SamplingParams
from vllm.v1.core.sched.output import NewRequestData, SchedulerOutput
from vllm.v1.kv_cache_interface import (
CircularBufferSpec,
KVCacheSpec,
MambaSpec,
SlidingWindowSpec,
UniformTypeKVCacheSpecs,
)
@dataclass
class BenchBatch:
scheduler_output: SchedulerOutput
# req_id -> token IDs of the query tokens, in order.
query_token_ids: dict[str, np.ndarray]
# Per KV cache group: allocated (begin, end) block ranges.
group_block_ranges: list[list[tuple[int, int]]]
num_blocks: int
# req_id -> the runner's expected (scheduled, computed, is_decode).
expected: dict[str, tuple[int, int, bool]]
@property
def num_tokens(self) -> int:
return self.scheduler_output.total_num_scheduled_tokens
@property
def req_ids(self) -> list[str]:
return list(self.scheduler_output.num_scheduled_tokens)
def finish(self) -> SchedulerOutput:
"""A step that only removes this batch's requests."""
output = SchedulerOutput.make_empty()
output.finished_req_ids = set(self.req_ids)
return output
def kv_block_size(spec: KVCacheSpec, dcp_size: int) -> int:
"""Tokens per block as the scheduler counts them: DCP spreads each
block's tokens over the DCP ranks."""
return spec.block_size * (dcp_size if spec.dcp_sharded else 1)
def _num_blocks(
spec: KVCacheSpec, computed: int, kv_len: int, dcp_size: int
) -> tuple[int, int]:
"""Return (leading null blocks, live blocks) for one request."""
block_size = kv_block_size(spec, dcp_size)
if isinstance(spec, UniformTypeKVCacheSpecs):
spec = spec.first_spec
if isinstance(spec, (CircularBufferSpec, MambaSpec)):
return 0, 1
total = -(-kv_len // block_size)
skipped = 0
if isinstance(spec, SlidingWindowSpec):
# Blocks wholly before the window are freed to the null block.
skipped = max(0, computed - spec.sliding_window + 1) // block_size
return skipped, total - skipped
def build_batch(
case: BenchCase,
kv_cache_groups,
num_kv_blocks: int,
vocab_size: int,
rng: np.random.Generator,
dcp_size: int = 1,
) -> BenchBatch:
"""Allocate disjoint blocks and build `NewRequestData` for every request.
Groups share one block pool, so block IDs are unique across groups. Block
0 is the null block.
With prefix P = K - Q (KV length K, query length Q):
| Kind | prompt tokens | num_computed_tokens | spec tokens |
| ------------------ | ------------- | ------------------- | ----------- |
| Prefill | K | P | - |
| Decode | P | P | - |
| Verify (D drafts) | P | P | D |
Unless chunked, a prefill's prefix is a prefix hit, which replays its last
R = min(window, P) tokens on models with SWA bounded replay (window 0
otherwise), like the scheduler: it schedules R + Q tokens from
replay_start = num_computed_tokens = P - R.
"""
replay_window = max(
(group.kv_cache_spec.prefix_replay_tokens for group in kv_cache_groups),
default=0,
)
expected: dict[str, tuple[int, int, bool]] = {}
cursor = 1
new_reqs = []
query_token_ids = {}
group_block_ranges: list[list[tuple[int, int]]] = [[] for _ in kv_cache_groups]
output = SchedulerOutput.make_empty()
for i, req in enumerate(case.requests):
req_id = f"bench-{i}"
replay = min(replay_window, req.prefix_len) if req.prefix_hit else 0
computed = req.prefix_len - replay
block_ids = []
for group, ranges in zip(kv_cache_groups, group_block_ranges):
skipped, live = _num_blocks(
group.kv_cache_spec, computed, req.kv_len, dcp_size
)
block_ids.append([0] * skipped + list(range(cursor, cursor + live)))
ranges.append((cursor, cursor + live))
cursor += live
tokens = rng.integers(0, vocab_size, req.kv_len, dtype=np.int64)
prompt = tokens[: req.prefix_len] if req.is_decode else tokens
query_token_ids[req_id] = tokens[computed:]
expected[req_id] = (req.query_len + replay, computed, req.is_decode)
new_reqs.append(
NewRequestData(
req_id=req_id,
prompt_token_ids=prompt.tolist(),
prefill_token_ids=prompt.tolist(),
mm_features=[],
sampling_params=SamplingParams(max_tokens=16, temperature=0.0),
pooling_params=None,
block_ids=tuple(block_ids),
num_computed_tokens=computed,
lora_request=None,
replay_start=computed if replay else 0,
)
)
output.num_scheduled_tokens[req_id] = req.query_len + replay
if req.num_draft_tokens:
output.scheduled_spec_decode_tokens[req_id] = tokens[
req.prefix_len + 1 :
].tolist()
if cursor > num_kv_blocks:
raise ValueError(
f"Case {case.name!r} needs {cursor} KV blocks, but only "
f"{num_kv_blocks} exist. Increase the KV cache size."
)
output.scheduled_new_reqs = new_reqs
output.total_num_scheduled_tokens = sum(output.num_scheduled_tokens.values())
return BenchBatch(output, query_token_ids, group_block_ranges, cursor, expected)
-266
View File
@@ -1,266 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Capture the forward of one exact batch as a one-shot CUDA graph and time it.
The graph is captured for a single batch and only replays that batch, so the
host-side decisions baked in at capture (attention plans, kernel choices by
query length, chunk splits) are correct by construction. This deliberately
bypasses vLLM's CUDA graph manager: no capture sizes, padding or dummy
metadata. Each replay first restores the KV blocks the step writes, so state the
forward mutates in place (e.g. compressor state) is the same on every replay.
"""
import time
from typing import Any
import numpy as np
import torch
from vllm.benchmarks.forward.batch import BenchBatch
from vllm.config.compilation import CUDAGraphMode
from vllm.distributed.parallel_state import get_world_group, graph_capture
from vllm.forward_context import get_forward_context
from vllm.v1.worker.gpu.input_batch import InputBatch
def _first_tensor(output: Any) -> torch.Tensor:
if isinstance(output, tuple):
output = output[0]
assert isinstance(output, torch.Tensor), type(output)
return output
def _compare(actual: torch.Tensor, expected: torch.Tensor) -> dict[str, float]:
delta = actual.float() - expected.float()
scale = expected.float().square().mean().sqrt().clamp(min=1e-12)
return {
"max_abs": delta.abs().max().item(),
"relative_rms": (delta.square().mean().sqrt() / scale).item(),
}
class ForwardBench:
"""Runs one batch through the runner's `execute_model`, timing its forward.
`execute_model` prepares the inputs, attention metadata and forward context
as in serving; the bench takes over the eager model call. Two steps: the
first ("warmup") runs the model eagerly, which triggers JIT, autotuning and
lazy init. The second ("capture") captures, validates and times the graph.
"""
def __init__(
self,
runner: Any,
batch: BenchBatch,
iters: int,
samples: int,
state_views: list[torch.Tensor],
name: str,
profile_prefix: str | None = None,
):
self.runner = runner
self.batch = batch
self.device = runner.device
self.iters = iters
self.samples = samples
self.state_views = state_views
self.name = name
self.profile_prefix = profile_prefix
self.phase = "warmup"
self.result: dict[str, Any] = {}
def run(self) -> dict[str, Any]:
runner, model = self.runner, self.runner.model
# The runner's eager branch calls `model(**inputs)`.
assert runner.compilation_config.cudagraph_mode == CUDAGraphMode.NONE
assert runner.ubatch_runner is None
prepare_inputs = runner.prepare_inputs
self._model_forward = model.forward
def prepare_and_check(*args, **kwargs) -> InputBatch:
self._input_batch = prepare_inputs(*args, **kwargs)
self._check_batch(self._input_batch)
return self._input_batch
# Instance attributes shadow the methods for these two steps only.
runner.prepare_inputs = prepare_and_check
model.forward = self._forward
try:
for self.phase in ("warmup", "capture"):
runner.execute_model(self.batch.scheduler_output)
finally:
del runner.prepare_inputs, model.forward
# No sample_tokens follows.
runner.execute_model_state = None
return self.result
def _forward(self, **inputs) -> Any:
self._write_input_ids(self._input_batch, inputs["input_ids"])
# Models may update the forward context in place (e.g. sequence
# parallelism shards is_padding); every call must start from the
# context the runner built.
context = get_forward_context()
saved = dict(vars(context))
def run_model() -> Any:
for name, value in saved.items():
setattr(context, name, value)
return self._model_forward(**inputs)
self._run_model = run_model
if self.phase == "warmup":
return run_model()
return self._capture_and_time()
def _check_batch(self, input_batch: InputBatch) -> None:
"""Fail unless the runner built exactly the requested batch."""
expected = self.batch.expected
assert sorted(input_batch.req_ids) == sorted(expected), input_batch.req_ids
for i, req_id in enumerate(input_batch.req_ids):
actual = (
int(input_batch.num_scheduled_tokens[i]),
int(input_batch.num_computed_tokens_np[i]),
not bool(input_batch.is_prefilling_np[i]),
)
assert actual == expected[req_id], (
f"{req_id}: got (scheduled, computed, decode) = {actual}, "
f"wanted {expected[req_id]}"
)
assert input_batch.num_tokens == self.batch.num_tokens
self.result["num_tokens_after_padding"] = input_batch.num_tokens_after_padding
self.result["batch_order"] = input_batch.req_ids
def _write_input_ids(
self, input_batch: InputBatch, input_ids: torch.Tensor | None
) -> None:
"""Decode/verify inputs otherwise come from sampler state."""
if input_ids is None:
return
ids = np.concatenate(
[self.batch.query_token_ids[req_id] for req_id in input_batch.req_ids]
)
input_ids[: len(ids)].copy_(torch.from_numpy(ids))
def _capture_and_time(self) -> Any:
world = get_world_group()
# The forward mutates state it reads (e.g. compressor state), so
# every run starts by restoring the KV blocks the step writes.
snapshots = [(view, view.clone()) for view in self.state_views]
def restore() -> None:
for view, saved in snapshots:
view.copy_(saved)
# A rendezvous and the timing events live inside the graph, after the
# restore: neither rank skew, the restore nor host launch gaps reach
# the timed interval.
flag = torch.zeros(1, device=self.device)
# TODO(woosuk): Support ROCm. The bench hangs there, likely on these
# in-graph events.
self._events = (
torch.cuda.Event(enable_timing=True, external=True),
torch.cuda.Event(enable_timing=True, external=True),
)
if world.world_size > 1:
# Initialize the communicator outside capture.
torch.distributed.all_reduce(flag, group=world.device_group)
with graph_capture(self.device) as capture_context:
stream = capture_context.stream
# Some kernels set up per-stream state on first use, outside capture.
eager = _first_tensor(self._run_model()).clone()
restore()
reference = _first_tensor(self._run_model()).clone()
stream.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=stream):
restore()
if world.world_size > 1:
torch.distributed.all_reduce(flag, group=world.device_group)
self._events[0].record()
output = self._run_model()
self._events[1].record()
hidden = _first_tensor(output)
graph.replay()
torch.accelerator.synchronize()
first = hidden.clone()
graph.replay()
torch.accelerator.synchronize()
self.result.update(
# Same step from the same restored state: should match exactly.
eager_repeat=_compare(reference, eager),
graph_vs_eager=_compare(first, reference),
graph_deterministic=bool(torch.equal(first, hidden)),
finite=bool(hidden.isfinite().all()),
state_reset_bytes=sum(
v.numel() * v.element_size() for v in self.state_views
),
)
self.result["latency_us"] = self._time(graph)
from vllm.benchmarks.forward.profile import profile_graph
# Each replay restores the state views, then (multi-rank) rendezvouses,
# before the timed forward.
skip = len(self.state_views) + (world.world_size > 1)
summary = profile_graph(graph, self.name, self.profile_prefix, skip)
if summary is not None:
self.result["profile"] = {
"summary": f"{self.profile_prefix}.kernels.json",
"span_us": summary["span_us"],
"busy_us": summary["busy_us"],
"activities": summary["activities"],
"top_kernels": summary["kernels"][:10],
}
del graph
return output
def _time(self, graph: torch.cuda.CUDAGraph) -> dict[str, Any]:
"""Per-replay forward time between in-graph events, MAX across ranks."""
world = get_world_group()
start, end = self._events
self._warmup(graph)
times = []
for _ in range(self.iters * self.samples):
graph.replay()
end.synchronize()
times.append(start.elapsed_time(end) * 1000)
arr = torch.tensor(times, device=self.device)
if world.world_size > 1:
torch.distributed.all_reduce(
arr, op=torch.distributed.ReduceOp.MAX, group=world.device_group
)
arr = arr.cpu().numpy()
sample_medians = np.median(arr.reshape(self.samples, -1), axis=1)
median = float(np.median(arr))
return {
"timer": "graph_events" + ("_rank_max" if world.world_size > 1 else ""),
"median": median,
# Stability: spread of per-sample medians relative to the median.
"sample_spread_pct": float(
100 * (sample_medians.max() - sample_medians.min()) / median
),
"mean": float(arr.mean()),
"std": float(arr.std()),
"min": float(arr.min()),
"p90": float(np.percentile(arr, 90)),
"count": len(arr),
"samples_us": [round(float(t), 2) for t in arr],
}
def _warmup(self, graph: torch.cuda.CUDAGraph, seconds: float = 1.0) -> None:
"""Replay until clocks and caches reach a steady state."""
world = get_world_group()
graph.replay()
torch.accelerator.synchronize()
begin = time.perf_counter()
graph.replay()
torch.accelerator.synchronize()
per_replay = max(time.perf_counter() - begin, 1e-6)
# Same count on every rank: the graph may contain collectives.
count = torch.tensor([max(self.iters, int(seconds / per_replay))])
torch.distributed.all_reduce(
count, op=torch.distributed.ReduceOp.MAX, group=world.cpu_group
)
for _ in range(int(count.item())):
graph.replay()
torch.accelerator.synchronize()
-70
View File
@@ -1,70 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""KV blocks of a benchmark case: initialization and the blocks a step writes.
Only the case's blocks are touched, group by group: groups can alias one
allocation with different layouts, and the batch builder keeps their block IDs
disjoint. Zero KV times within a few percent of random KV, even for sparse
attention (DeepSeek-V4.1, ~100K context), so zeros are the only init for now.
"""
from collections.abc import Iterator
import torch
from vllm.benchmarks.forward.batch import BenchBatch, kv_block_size
def _tensors(kv_cache) -> Iterator[torch.Tensor]:
if isinstance(kv_cache, torch.Tensor):
if kv_cache.numel():
yield kv_cache
elif isinstance(kv_cache, (list, tuple)):
for item in kv_cache:
yield from _tensors(item)
def _block_dim(tensor: torch.Tensor, num_blocks: int) -> tuple[int, int]:
"""Return (block dim, kernel blocks per allocator block)."""
for dim in range(min(2, tensor.dim())):
if tensor.shape[dim] >= num_blocks and tensor.shape[dim] % num_blocks == 0:
return dim, tensor.shape[dim] // num_blocks
raise ValueError(f"No block dim of size {num_blocks} in {tuple(tensor.shape)}")
@torch.inference_mode()
def zero_kv_blocks(
static_forward_context: dict, kv_cache_groups, batch: BenchBatch, num_blocks: int
) -> None:
for group, ranges in zip(kv_cache_groups, batch.group_block_ranges):
for name in group.layer_names:
layer = static_forward_context[name]
for tensor in _tensors(getattr(layer, "kv_cache", None)):
dim, factor = _block_dim(tensor, num_blocks)
for begin, end in ranges:
tensor.narrow(dim, begin * factor, (end - begin) * factor).zero_()
def touched_kv_views(
static_forward_context: dict,
kv_cache_groups,
batch: BenchBatch,
num_blocks: int,
dcp_size: int,
) -> list[torch.Tensor]:
"""Views of the KV blocks a step may write: each request's last
ceil(scheduled / block_size) + 2 blocks, in every group."""
scheduled_tokens = [batch.expected[req_id][0] for req_id in batch.req_ids]
views = []
for group, ranges in zip(kv_cache_groups, batch.group_block_ranges):
block_size = kv_block_size(group.kv_cache_spec, dcp_size)
for name in group.layer_names:
layer = static_forward_context[name]
for tensor in _tensors(getattr(layer, "kv_cache", None)):
dim, factor = _block_dim(tensor, num_blocks)
for (begin, end), tokens in zip(ranges, scheduled_tokens):
first = max(begin, end - (-(-tokens // block_size) + 2))
views.append(
tensor.narrow(dim, first * factor, (end - first) * factor)
)
return views
-256
View File
@@ -1,256 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Benchmark the model forward of exact batches as one-shot CUDA graphs.
Each case is a list of requests `(query_len, kv_len, num_draft_tokens)`
per DP rank. The bench builds the batch without the scheduler, runs it through
the real `execute_model` (input prep, attention metadata, DP sync), and times
the target-model forward captured as a CUDA graph. Sampling, the speculator
and CPU step overhead aren't measured.
Single GPU: `vllm bench forward --model M --case decode=32*1@8192`.
Multi-GPU: `torchrun --nproc-per-node=N -m vllm.entrypoints.cli.main bench
forward ...`, one process per GPU.
"""
import argparse
import json
import os
import socket
from dataclasses import asdict
from typing import Any
import numpy as np
import torch
from vllm.benchmarks.forward.shapes import (
BenchCase,
jitter_kv_lens,
load_cases,
parse_case,
)
from vllm.engine.arg_utils import EngineArgs
from vllm.logger import init_logger
from vllm.utils.argparse_utils import FlexibleArgumentParser
logger = init_logger(__name__)
def add_cli_args(parser: FlexibleArgumentParser) -> None:
parser.add_argument(
"--case",
action="append",
default=[],
help="`NAME=REQS`, where REQS is a comma list of `[N*]Q[@K][c][:D]` "
"(query length Q, KV length K including the query (default Q), D draft "
"tokens, N requests). A prefill's prefix is a prefix-cache hit unless c "
"marks a chunked prefill. E.g. `decode=32*1@8192`, "
"`verify=16*4@4096:3`, `prefill=8192`, `hit=2048@10240`, "
"`mix=8192,32*1@8192`. Repeatable.",
)
parser.add_argument("--shapes", help="JSON or YAML file with cases.")
parser.add_argument(
"--kv-len-jitter",
type=float,
default=0.5,
help="Draw each request's prefix (KV length minus query length) "
"uniformly within +-this fraction of its value, like the varied lengths "
"of a real batch, capped at the max model length. Seeded by --seed only, "
"so every DP rank gets the same lengths. 0 keeps the exact lengths.",
)
parser.add_argument(
"--kv-init",
choices=["zeros"],
default="zeros",
help="Initial KV cache contents of each case's blocks.",
)
parser.add_argument(
"--routing",
choices=["auto", "real", "balanced"],
default="auto",
help="MoE routing. balanced: fixed routing with equal load per EP rank. "
"auto: balanced with dummy weights, else real.",
)
parser.add_argument("--iters", type=int, default=20, help="Replays per sample.")
parser.add_argument("--samples", type=int, default=5)
parser.add_argument("--output-json", help="Write results to this file.")
parser.add_argument(
"--profile-dir",
help="After timing, trace a few replays of each case and write "
"`<case>.rank<r>.trace.json.gz` (Perfetto) and `<case>.rank<r>.kernels.json` "
"(GPU time per kernel per forward) here. Summarize or compare with "
"`python -m vllm.benchmarks.forward.profile A.kernels.json [B.kernels.json]`.",
)
EngineArgs.add_cli_args(parser)
parser.set_defaults(load_format="dummy", enable_prefix_caching=False)
def _setup_env() -> None:
os.environ["VLLM_USE_V2_MODEL_RUNNER"] = "1"
os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] = "0"
# Each case's eager pass compiles exactly the shapes it needs.
os.environ.setdefault("VLLM_DEEP_GEMM_WARMUP", "skip")
if "RANK" in os.environ:
return # Launched by torchrun.
with socket.socket() as s:
s.bind(("127.0.0.1", 0))
port = s.getsockname()[1]
os.environ.update(
RANK="0",
LOCAL_RANK="0",
WORLD_SIZE="1",
MASTER_ADDR="127.0.0.1",
MASTER_PORT=str(port),
)
def _make_engine_args(args: argparse.Namespace, cases: list[BenchCase]):
from vllm.config.compilation import CompilationMode, CUDAGraphMode
engine_args = EngineArgs.from_cli_args(args)
engine_args.distributed_executor_backend = "external_launcher"
engine_args.enable_prefix_caching = False
engine_args.compilation_config.mode = CompilationMode.NONE
engine_args.compilation_config.cudagraph_mode = CUDAGraphMode.NONE
if engine_args.max_num_seqs is None:
engine_args.max_num_seqs = max(len(c.requests) for c in cases)
if engine_args.max_num_batched_tokens is None:
# Prefix hits may replay up to a window more per request.
engine_args.max_num_batched_tokens = max(
8192,
*(
c.num_tokens + 1024 * sum(r.prefix_hit for r in c.requests)
for c in cases
),
)
return engine_args
def run_case(runner, case: BenchCase, args: argparse.Namespace) -> dict[str, Any]:
required_drafts = max((r.num_draft_tokens for r in case.requests), default=0)
if required_drafts > runner.num_speculative_steps:
raise ValueError(
f"Case {case.name!r} needs {required_drafts} draft tokens per request, "
f"but the runner supports {runner.num_speculative_steps}. "
"Set --speculative-config with a sufficient num_speculative_tokens."
)
from vllm.benchmarks.forward.batch import build_batch
from vllm.benchmarks.forward.capture import ForwardBench
from vllm.benchmarks.forward.kv_init import touched_kv_views, zero_kv_blocks
from vllm.benchmarks.forward.routing import install_routing
context = runner.vllm_config.compilation_config.static_forward_context
kv_config = runner.kv_cache_config
dcp_size = runner.parallel_config.decode_context_parallel_size
rng = np.random.default_rng([args.seed, runner.dp_rank])
batch = build_batch(
case,
kv_config.kv_cache_groups,
kv_config.num_blocks,
runner.model_config.get_vocab_size(),
rng,
dcp_size,
)
zero_kv_blocks(context, kv_config.kv_cache_groups, batch, kv_config.num_blocks)
profile_prefix = None
if args.profile_dir:
from vllm.distributed.parallel_state import get_world_group
os.makedirs(args.profile_dir, exist_ok=True)
rank = get_world_group().rank
profile_prefix = os.path.join(args.profile_dir, f"{case.name}.rank{rank}")
bench = ForwardBench(
runner,
batch,
args.iters,
args.samples,
touched_kv_views(
context, kv_config.kv_cache_groups, batch, kv_config.num_blocks, dcp_size
),
case.name,
profile_prefix,
)
routing, balanced = install_routing(
runner, args.routing, args.load_format == "dummy", batch.num_tokens, args.seed
)
try:
result = bench.run()
if balanced is not None:
balanced.check_applied()
finally:
runner.execute_model(batch.finish())
if balanced is not None:
balanced.remove()
return {
"num_tokens": batch.num_tokens,
"kv_blocks": batch.num_blocks,
"routing": routing,
**result,
}
def main(args: argparse.Namespace) -> None:
cases = [parse_case(c) for c in args.case]
if args.shapes:
cases += load_cases(args.shapes)
if not cases:
raise ValueError("Pass at least one --case or --shapes.")
if not 0 <= args.kv_len_jitter < 1:
raise ValueError("--kv-len-jitter must be in [0, 1).")
_setup_env()
from vllm import LLM
from vllm.config import set_current_vllm_config
from vllm.distributed.parallel_state import get_world_group
llm = LLM.from_engine_args(_make_engine_args(args, cases))
runner = llm.llm_engine.model_executor.driver_worker.worker.model_runner
# Same draw on every rank: shapes are balanced across DP ranks.
max_kv_len = runner.model_config.max_model_len
cases = [
jitter_kv_lens(c, args.kv_len_jitter, max_kv_len, args.seed) for c in cases
]
world = get_world_group()
results = []
with torch.inference_mode(), set_current_vllm_config(runner.vllm_config):
for case in cases:
result = run_case(runner, case, args)
gathered: list[Any] = [None] * world.world_size
torch.distributed.all_gather_object(gathered, result, group=world.cpu_group)
if world.rank != 0:
continue
rank0 = gathered[0]
latency = rank0["latency_us"] # Each replay is the max over ranks.
results.append(
{
"case": case.name,
"requests": [asdict(r) for r in case.requests],
"median_us": latency["median"],
"ranks": gathered,
}
)
print(
f"[forward] {case.name}: {len(case.requests)} reqs, "
f"{rank0['num_tokens']} tokens, median {latency['median']:.1f} us, "
f"sample spread {latency['sample_spread_pct']:.2f}%, graph vs "
f"eager rel rms {rank0['graph_vs_eager']['relative_rms']:.1e}",
flush=True,
)
if profile := rank0.get("profile"):
from vllm.benchmarks.forward.profile import format_summary
print(
f"[forward] {case.name} rank 0 kernels ({profile['summary']}):\n"
+ format_summary({**profile, "kernels": profile["top_kernels"]}),
flush=True,
)
if latency["sample_spread_pct"] > 0.5:
logger.warning(
"%s: unstable timing (sample medians spread %.2f%%).",
case.name,
latency["sample_spread_pct"],
)
if world.rank == 0 and args.output_json:
with open(args.output_json, "w") as f:
json.dump(results, f, indent=2)
-236
View File
@@ -1,236 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""GPU kernel profiles of `vllm bench forward` cases.
With `--profile-dir`, each case's graph is replayed a few more times under
`torch.profiler` after timing. Each rank writes a Perfetto trace and a kernel
summary (`<case>.rank<r>.kernels.json`): per-kernel time per forward, sorted.
Summarize one file or compare two (e.g. before/after a change):
`python -m vllm.benchmarks.forward.profile A.kernels.json [B.kernels.json]`.
The same replays run between `cudaProfilerStart`/`Stop` in an NVTX range per
case (with or without `--profile-dir`), so Nsight Systems can record just them:
`nsys profile --capture-range=cudaProfilerApi --capture-range-end=repeat
--cuda-graph-trace=node vllm bench forward ...`.
"""
import argparse
import gzip
import json
import statistics
from collections import defaultdict
from typing import Any
import regex as re
import torch
_GPU_CATEGORIES = ("kernel", "gpu_memcpy", "gpu_memset")
# Kernels named after a generic launcher are identified by their first
# template argument instead.
_GENERIC = re.compile(r"(^|::)(device_kernel|Kernel|kernel|KernelWrapper)$")
# CuTe DSL kernel names embed per-shape layouts after these markers.
_CUTE_DSL_SUFFIX = re.compile(r"_object_at_|_tensorptr")
def _match_back(text: str, end: int, open_ch: str, close_ch: str) -> int:
"""Index of the bracket opening the one closing at `end`, skipping
brackets of the other kind nested inside parentheses."""
depth, parens = 0, 0
for i in range(end, -1, -1):
ch = text[i]
if open_ch == "<" and ch in "()":
parens += 1 if ch == ")" else -1
continue
if parens:
continue
if ch == close_ch:
depth += 1
elif ch == open_ch:
depth -= 1
if depth == 0:
return i
return -1
def short_name(name: str) -> str:
"""`void ns::foo<T, 4>(args)` -> `ns::foo`: return type, template and
function arguments dropped. Generic launchers (e.g.
`cutlass::device_kernel<Op<...>>`) keep their first template argument."""
if name.startswith("kernel_cutlass_"):
return _CUTE_DSL_SUFFIX.split(name, maxsplit=1)[0]
head = name.strip()
if head.endswith(")"):
head = head[: max(_match_back(head, len(head) - 1, "(", ")"), 0)].rstrip()
template = ""
if head.endswith(">"):
start = _match_back(head, len(head) - 1, "<", ">")
if start > 0:
head, template = head[:start], head[start + 1 : -1]
base = head.split()[-1] if head.split() else name
if _GENERIC.search(base) and template:
arg = re.split(r"[<>(,]", template, maxsplit=1)[0].strip()
if arg:
return f"{base}<{arg.removeprefix('void ')}>"
return base
def summarize(events: list[dict], replays: int, skip: int) -> dict[str, Any]:
"""Per-forward kernel times from the GPU activities of `replays` replays.
Each replay first runs `skip` activities that aren't part of the timed
forward (the KV state restore and the rank rendezvous); they are reported
separately as `pre_forward_us`.
"""
acts = sorted(
(e for e in events if e.get("cat") in _GPU_CATEGORIES and "dur" in e),
key=lambda e: e["ts"],
)
if not acts:
raise ValueError(
"No GPU activity recorded. Another profiler (e.g. nsys) may hold "
"CUPTI; don't combine --profile-dir with it."
)
if len(acts) % replays:
raise ValueError(
f"{len(acts)} GPU activities don't split into {replays} replays"
)
per_replay = len(acts) // replays
if per_replay <= skip:
raise ValueError(f"{per_replay} activities per replay, {skip} to skip")
kernels: dict[str, dict[str, Any]] = defaultdict(
lambda: {"total_us": 0.0, "calls": 0, "example": ""}
)
spans, busy, pre = [], [], []
for r in range(replays):
chunk = acts[r * per_replay : (r + 1) * per_replay]
pre.append(sum(e["dur"] for e in chunk[:skip]))
forward = chunk[skip:]
spans.append(
max(e["ts"] + e["dur"] for e in forward) - min(e["ts"] for e in forward)
)
busy.append(sum(e["dur"] for e in forward))
for e in forward:
name = short_name(e["name"]) if e["cat"] == "kernel" else e["name"]
k = kernels[name]
k["total_us"] += e["dur"]
k["calls"] += 1
k["example"] = k["example"] or e["name"][:300]
total = sum(k["total_us"] for k in kernels.values())
rows = [
{
"name": name,
"us": round(k["total_us"] / replays, 2),
"pct": round(100 * k["total_us"] / total, 2),
"calls": k["calls"] // replays,
"example": k["example"],
}
for name, k in kernels.items()
]
rows.sort(key=lambda row: -row["us"])
return {
"replays": replays,
# First to last forward activity; busy can exceed it when streams overlap.
"span_us": round(statistics.median(spans), 2),
"busy_us": round(statistics.median(busy), 2),
"pre_forward_us": round(statistics.median(pre), 2),
"activities": per_replay - skip,
"kernels": rows,
}
def profile_graph(
graph: torch.cuda.CUDAGraph,
name: str,
prefix: str | None,
skip: int,
replays: int = 3,
) -> dict[str, Any] | None:
"""Replay `replays` times between `cudaProfilerStart`/`Stop`, in an NVTX
range named after the case, for `nsys --capture-range=cudaProfilerApi`.
With `prefix`, also trace them with `torch.profiler` and write
`prefix.trace.json.gz` and `prefix.kernels.json`. Every rank must call
this (replays may contain collectives).
"""
def run() -> None:
torch.cuda.profiler.start()
with torch.cuda.nvtx.range(f"forward/{name}"):
for _ in range(replays):
graph.replay()
torch.accelerator.synchronize()
torch.cuda.profiler.stop()
if prefix is None:
run()
return None
activities = [torch.profiler.ProfilerActivity.CUDA]
with torch.profiler.profile(activities=activities) as prof:
run()
trace = f"{prefix}.trace.json.gz"
prof.export_chrome_trace(trace)
with gzip.open(trace, "rt") as f:
events = json.load(f)["traceEvents"]
summary = {"trace": trace, **summarize(events, replays, skip)}
with open(f"{prefix}.kernels.json", "w") as f:
json.dump(summary, f, indent=1)
return summary
def format_summary(summary: dict[str, Any], top: int = 20) -> str:
lines = [
f"span {summary['span_us']:.1f} us, busy {summary['busy_us']:.1f} us, "
f"{summary['activities']} activities/forward",
f"{'us':>10} {'%':>6} {'calls':>6} kernel",
]
for row in summary["kernels"][:top]:
lines.append(
f"{row['us']:10.1f} {row['pct']:6.2f} {row['calls']:6d} {row['name']}"
)
return "\n".join(lines)
def format_diff(a: dict[str, Any], b: dict[str, Any], top: int = 20) -> str:
rows_a = {row["name"]: row for row in a["kernels"]}
rows_b = {row["name"]: row for row in b["kernels"]}
deltas = []
for name in rows_a.keys() | rows_b.keys():
ra, rb = rows_a.get(name), rows_b.get(name)
us_a, us_b = ra["us"] if ra else 0.0, rb["us"] if rb else 0.0
calls_a, calls_b = ra["calls"] if ra else 0, rb["calls"] if rb else 0
deltas.append((us_b - us_a, us_a, us_b, calls_a, calls_b, name))
deltas.sort(key=lambda d: -abs(d[0]))
lines = [
f"span {a['span_us']:.1f} -> {b['span_us']:.1f} us "
f"({b['span_us'] - a['span_us']:+.1f}), busy {a['busy_us']:.1f} -> "
f"{b['busy_us']:.1f} us ({b['busy_us'] - a['busy_us']:+.1f})",
f"{'delta us':>10} {'A us':>10} {'B us':>10} {'A n':>5} {'B n':>5} kernel",
]
for delta, us_a, us_b, calls_a, calls_b, name in deltas[:top]:
lines.append(
f"{delta:+10.1f} {us_a:10.1f} {us_b:10.1f} {calls_a:5d} {calls_b:5d} "
f"{name}"
)
return "\n".join(lines)
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("summaries", nargs="+", help="One or two *.kernels.json.")
parser.add_argument("--top", type=int, default=20)
args = parser.parse_args()
if len(args.summaries) > 2:
parser.error("Pass one summary, or two to compare.")
loaded = []
for path in args.summaries:
with open(path) as f:
loaded.append(json.load(f))
if len(loaded) == 1:
print(format_summary(loaded[0], args.top))
else:
print(format_diff(loaded[0], loaded[1], args.top))
if __name__ == "__main__":
main()
-227
View File
@@ -1,227 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Balanced MoE routing for benchmarks with dummy weights.
Dummy gate weights give degenerate routing, which skews MoE and all-to-all
cost. The routing still runs; its result is replaced by the fixture at
whichever boundary the layer exposes:
- Gate output: fixed logits whose top-k is the fixture. Covers every backend
that routes from logits, including monolithic kernels that run top-k
internally (e.g. FlashInfer TRTLLM).
- Experts input: models that compute top-k themselves and pass
`topk_ids` to their experts (e.g. MegaMoE with a fused gate) get the
fixture's IDs. The routing weights are kept; they don't affect cost.
Hash-routed layers ignore logits and keep their tables unless their experts
take `topk_ids`.
"""
import inspect
import numpy as np
import torch
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
logger = init_logger(__name__)
# Fixture logits: chosen experts score far above the rest under any monotonic
# scoring function, so top-k picks exactly them.
_CHOSEN_LOGIT = 4.0
_OTHER_LOGIT = -4.0
def balanced_topk_ids(
num_tokens: int, num_experts: int, top_k: int, ep_size: int, rng
) -> np.ndarray:
"""Expert IDs [num_tokens, top_k] giving every EP rank equal load.
Each token's slots go round-robin over EP ranks, to distinct experts.
"""
assert num_experts % ep_size == 0
local = num_experts // ep_size
slots = np.arange(num_tokens)[:, None] * top_k + np.arange(top_k)
ranks = slots % ep_size
# Slots of one token that land on the same rank take consecutive experts.
repeat = np.zeros_like(ranks)
for j in range(1, top_k):
repeat[:, j] = (ranks[:, :j] == ranks[:, j : j + 1]).sum(axis=1)
assert repeat.max() < local
base = rng.integers(local, size=(num_tokens, ep_size))
offset = np.take_along_axis(base, ranks, axis=1)
return ranks * local + (offset + repeat) % local
def _moe_layers(model: torch.nn.Module) -> list[tuple[torch.nn.Module, GateLinear]]:
"""(MoE module, gate) per layer, in model order."""
layers: dict[GateLinear, torch.nn.Module] = {}
for module in model.modules():
gate = getattr(module, "gate", None)
if not isinstance(gate, GateLinear) or gate in layers:
continue
if _top_k(module) is not None:
layers[gate] = module
return [(module, gate) for gate, module in layers.items()]
def _top_k(module: torch.nn.Module) -> int | None:
"""The MoE layer's top-k, from the module or its experts submodules."""
for sub in module.modules():
for name in ("top_k", "n_activated_experts"):
top_k = getattr(sub, name, None)
if isinstance(top_k, int):
return top_k
return None
class BalancedRouting:
"""Installs balanced routing for one case; `remove()` restores the model."""
def __init__(
self,
model: torch.nn.Module,
num_tokens: int,
dp_rank: int,
dp_size: int,
ep_size: int,
seed: int,
device: torch.device,
):
layers = _moe_layers(model)
if not layers:
raise ValueError("No MoE gates found; use --routing real.")
self._hooks = []
self._biases: list[tuple[torch.Tensor, torch.Tensor]] = []
self._fired = [0] * len(layers)
self._layers = layers
# Per-rank rows, with headroom for padding; every rank generates the
# same global fixture and takes its slice.
rows = -(-num_tokens // 256) * 256 + 256
for layer, (module, gate) in enumerate(layers):
if (getattr(module, "num_expert_group", None) or 1) > 1:
raise NotImplementedError("Balanced routing with grouped top-k.")
num_experts = gate.weight.shape[0]
top_k = _top_k(module)
assert top_k is not None
rng = np.random.default_rng([seed, layer])
ids = balanced_topk_ids(rows * dp_size, num_experts, top_k, ep_size, rng)[
dp_rank * rows : (dp_rank + 1) * rows
]
logits = torch.full((rows, num_experts), _OTHER_LOGIT, dtype=torch.float32)
logits.scatter_(1, torch.from_numpy(ids), _CHOSEN_LOGIT)
# Distinct values: no ties for any top-k implementation.
logits += torch.from_numpy(rng.uniform(0, 0.5, logits.shape)).float()
self._hooks.append(
gate.register_forward_hook(
self._substitute(logits.to(device), self._fired, layer)
)
)
bias = getattr(gate, "e_score_correction_bias", None)
if bias is None:
bias = getattr(module, "e_score_correction_bias", None)
if isinstance(bias, torch.Tensor):
self._biases.append((bias, bias.detach().clone()))
bias.data.zero_()
experts = getattr(module, "experts", None)
if isinstance(experts, torch.nn.Module):
params = list(inspect.signature(experts.forward).parameters)
if "topk_ids" in params:
self._hooks.append(
experts.register_forward_pre_hook(
self._substitute_ids(
torch.from_numpy(ids).to(device),
params.index("topk_ids"),
self._fired,
layer,
),
with_kwargs=True,
)
)
@staticmethod
def _substitute(fixture: torch.Tensor, fired: list[int], layer: int):
cast = [fixture]
def hook(module, inputs, output):
fired[layer] += 1
logits = output[0] if isinstance(output, tuple) else output
if cast[0].dtype != logits.dtype:
cast[0] = fixture.to(logits.dtype) # Once, at warmup.
replaced = cast[0][: logits.shape[0]]
return (replaced, *output[1:]) if isinstance(output, tuple) else replaced
return hook
@staticmethod
def _substitute_ids(fixture: torch.Tensor, index: int, fired, layer: int):
cast = [fixture]
def hook(module, args, kwargs):
fired[layer] += 1
ids = kwargs["topk_ids"] if "topk_ids" in kwargs else args[index]
if cast[0].dtype != ids.dtype:
cast[0] = fixture.to(ids.dtype) # Once, at warmup.
replaced = cast[0][: ids.shape[0]]
if "topk_ids" in kwargs:
return args, {**kwargs, "topk_ids": replaced}
return (*args[:index], replaced, *args[index + 1 :]), kwargs
return hook
def check_applied(self) -> None:
"""Fail if a layer bypassed both the gate and the experts hooks."""
missed = [
type(module).__name__
for (module, _), fired in zip(self._layers, self._fired)
if not fired
]
if missed:
raise RuntimeError(
f"Balanced routing didn't reach {len(missed)} MoE layers "
f"({missed[0]}): their gate is fused away. Use --routing real."
)
def remove(self) -> None:
for hook in self._hooks:
hook.remove()
for bias, saved in self._biases:
bias.data.copy_(saved)
def install_routing(
runner, requested: str, dummy_weights: bool, num_tokens: int, seed: int
) -> tuple[str, BalancedRouting | None]:
"""Resolve `--routing` and install balanced routing if selected.
`auto` is balanced with dummy weights and real otherwise; it falls back to
real routing when the model doesn't support the fixture.
"""
mode = requested
if mode == "auto":
mode = "balanced" if dummy_weights else "real"
if mode == "real":
return mode, None
parallel = runner.parallel_config
ep_size = (
parallel.data_parallel_size * parallel.tensor_parallel_size
if parallel.enable_expert_parallel
else 1
)
try:
routing = BalancedRouting(
runner.model,
num_tokens,
runner.dp_rank,
runner.dp_size,
ep_size,
seed,
runner.device,
)
except (ValueError, NotImplementedError) as e:
if requested == "balanced":
raise
logger.warning("Using real routing: %s", e)
return "real", None
return mode, routing
-137
View File
@@ -1,137 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Exact batch shapes for `vllm bench forward`."""
import json
import zlib
from dataclasses import dataclass, replace
from pathlib import Path
import numpy as np
import regex as re
@dataclass(frozen=True)
class BenchRequest:
query_len: int
# KV length after the step: the prefix already in the KV cache plus the
# query tokens.
kv_len: int
# Spec-decode verify: query_len == 1 + num_draft_tokens.
num_draft_tokens: int = 0
# The prefix was computed by this request's earlier chunks. Otherwise a
# prefill's prefix is a prefix-cache hit, of which models with SWA bounded
# replay (DeepSeek-V4.1) recompute the last window.
chunked: bool = False
def __post_init__(self) -> None:
if (
self.query_len < 1
or self.kv_len < self.query_len
or self.num_draft_tokens < 0
):
raise ValueError(f"Invalid request: {self}")
if self.num_draft_tokens and not self.is_decode:
raise ValueError(
f"Verify needs query_len == 1 + num_draft_tokens and a prefix: {self}"
)
if self.chunked and (self.is_decode or not self.prefix_len):
raise ValueError(f"A chunk needs a prefill with a prefix: {self}")
@property
def prefix_len(self) -> int:
return self.kv_len - self.query_len
@property
def is_decode(self) -> bool:
"""Decode or verify; otherwise a prefill."""
return self.prefix_len > 0 and self.query_len == 1 + self.num_draft_tokens
@property
def prefix_hit(self) -> bool:
return self.prefix_len > 0 and not self.is_decode and not self.chunked
@dataclass(frozen=True)
class BenchCase:
name: str
requests: tuple[BenchRequest, ...]
@property
def num_tokens(self) -> int:
return sum(r.query_len for r in self.requests)
def jitter_kv_lens(
case: BenchCase, jitter: float, max_kv_len: int, seed: int
) -> BenchCase:
"""Draw each request's prefix uniformly within +-jitter of its value.
Query lengths are kept, so a fresh prefill stays one and a non-empty
prefix stays non-empty. KV lengths are capped at `max_kv_len`. The draw
depends only on the seed and the case's name, not on the other cases.
"""
rng = np.random.default_rng([seed, zlib.crc32(case.name.encode())])
requests = []
for req in case.requests:
if req.kv_len > max_kv_len:
raise ValueError(f"{case.name}: KV length {req.kv_len} > {max_kv_len}")
prefix = req.prefix_len
if prefix and jitter:
prefix = max(1, round(prefix * rng.uniform(1 - jitter, 1 + jitter)))
kv_len = min(req.query_len + prefix, max_kv_len)
requests.append(replace(req, kv_len=kv_len))
return replace(case, requests=tuple(requests))
_REQ_RE = re.compile(r"^(?:(\d+)\*)?(\d+)(?:@(\d+))?(c)?(?::(\d+))?$")
def parse_case(text: str) -> BenchCase:
"""Parse `NAME=REQS`, where REQS is a comma list of `[N*]Q[@K][c][:D]`.
Q is the query length, K the KV length including the query (default Q, a
fresh prefill), D the number of draft tokens (verify) and N the number of
such requests. A prefill's prefix is a prefix-cache hit unless `c` marks
it as a chunked prefill. For example, `decode=32*1@8192`,
`verify=16*4@4096:3`, `prefill=8192`, `hit=8192@40960`,
`chunk=8192@40960c` or `mix=8192,32*1@8192`.
"""
name, sep, body = text.partition("=")
if not sep:
name, body = text, text
requests: list[BenchRequest] = []
for item in body.split(","):
match = _REQ_RE.match(item.strip())
if match is None:
raise ValueError(f"Invalid request {item!r} in case {text!r}")
count, query, kv, chunked, drafts = match.groups()
request = BenchRequest(
int(query), int(kv or query), int(drafts or 0), chunked=bool(chunked)
)
requests.extend([request] * int(count or 1))
return BenchCase(name, tuple(requests))
def load_cases(path: str) -> list[BenchCase]:
"""Load cases from a JSON or YAML file.
Format: `{"cases": [{"name": ..., "requests": [{"query_len": ...,
"kv_len": ..., "num_draft_tokens": ..., "chunked": ..., "count": ...}]}]}`.
"""
text = Path(path).read_text()
if path.endswith((".yaml", ".yml")):
import yaml
data = yaml.safe_load(text)
else:
data = json.loads(text)
cases = []
for case in data["cases"]:
requests: list[BenchRequest] = []
for req in case["requests"]:
req = dict(req)
count = req.pop("count", 1)
requests.extend([BenchRequest(**req)] * count)
cases.append(BenchCase(case["name"], tuple(requests)))
return cases
-180
View File
@@ -1,180 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Benchmark the latency of processing a single batch of requests."""
import argparse
import json
import os
import time
from typing import Any
import numpy as np
from tqdm import tqdm
from vllm.benchmarks.lib.utils import convert_to_pytorch_benchmark_format, write_to_json
from vllm.engine.arg_utils import EngineArgs
from vllm.inputs import TextPrompt, TokensPrompt
from vllm.sampling_params import BeamSearchParams
from vllm.utils.argparse_utils import FlexibleArgumentParser
def save_to_pytorch_benchmark_format(
args: argparse.Namespace, results: dict[str, Any]
) -> None:
pt_records = convert_to_pytorch_benchmark_format(
args=args,
metrics={"latency": results["latencies"]},
extra_info={k: results[k] for k in ["avg_latency", "percentiles"]},
)
if pt_records:
pt_file = f"{os.path.splitext(args.output_json)[0]}.pytorch.json"
write_to_json(pt_file, pt_records)
def add_cli_args(parser: FlexibleArgumentParser):
parser.add_argument("--input-len", type=int, default=32)
parser.add_argument("--output-len", type=int, default=128)
parser.add_argument("--batch-size", type=int, default=8)
parser.add_argument(
"--n",
type=int,
default=1,
help="Number of generated sequences per prompt.",
)
parser.add_argument("--use-beam-search", action="store_true")
parser.add_argument(
"--num-iters-warmup",
type=int,
default=10,
help="Number of iterations to run for warmup.",
)
parser.add_argument(
"--num-iters", type=int, default=30, help="Number of iterations to run."
)
parser.add_argument(
"--profile",
action="store_true",
help="profile the generation process of a single batch",
)
parser.add_argument(
"--output-json",
type=str,
default=None,
help="Path to save the latency results in JSON format.",
)
parser.add_argument(
"--disable-detokenize",
action="store_true",
help=(
"Do not detokenize responses (i.e. do not include "
"detokenization time in the latency measurement)"
),
)
parser = EngineArgs.add_cli_args(parser)
# V1 enables prefix caching by default which skews the latency
# numbers. We need to disable prefix caching by default.
parser.set_defaults(enable_prefix_caching=False)
def main(args: argparse.Namespace):
engine_args = EngineArgs.from_cli_args(args)
# Lazy import to avoid importing LLM when the bench command is not selected.
from vllm import LLM, SamplingParams
# NOTE(woosuk): If the request cannot be processed in a single batch,
# the engine will automatically process the request in multiple batches.
llm = LLM.from_engine_args(engine_args)
assert llm.llm_engine.model_config.max_model_len >= (
args.input_len + args.output_len
), (
"Please ensure that max_model_len is greater than"
" the sum of input_len and output_len."
)
sampling_params = SamplingParams(
n=args.n,
temperature=1.0,
top_p=1.0,
ignore_eos=True,
max_tokens=args.output_len,
detokenize=not args.disable_detokenize,
)
dummy_prompt_token_ids = np.random.randint(
10000, size=(args.batch_size, args.input_len)
)
dummy_prompts: list[TokensPrompt | TextPrompt] = [
TokensPrompt(prompt_token_ids=batch)
for batch in dummy_prompt_token_ids.tolist()
]
def llm_generate():
if not args.use_beam_search:
llm.generate(dummy_prompts, sampling_params=sampling_params, use_tqdm=False)
else:
llm.beam_search(
dummy_prompts,
BeamSearchParams(
beam_width=args.n,
max_tokens=args.output_len,
ignore_eos=True,
watermarking=False,
),
)
def run_to_completion(do_profile: bool = False):
if do_profile:
llm.start_profile()
llm_generate()
llm.stop_profile()
else:
start_time = time.perf_counter()
llm_generate()
end_time = time.perf_counter()
latency = end_time - start_time
return latency
print("Warming up...")
for _ in tqdm(range(args.num_iters_warmup), desc="Warmup iterations"):
run_to_completion(do_profile=False)
if args.profile:
profiler_config = engine_args.profiler_config
if profiler_config.profiler == "torch":
print(
"Profiling with torch profiler (results will be saved to"
f" {profiler_config.torch_profiler_dir})..."
)
elif profiler_config.profiler == "cuda":
print("Profiling with cuda profiler ...")
elif profiler_config.profiler == "proton":
print(
"Profiling with Proton (results will be saved under"
f" {profiler_config.proton_profiler_dir})..."
)
run_to_completion(do_profile=True)
return
# Benchmark.
latencies = []
for _ in tqdm(range(args.num_iters), desc="Bench iterations"):
latencies.append(run_to_completion(do_profile=False))
latencies = np.array(latencies)
percentages = [10, 25, 50, 75, 90, 99]
percentiles = np.percentile(latencies, percentages)
print(f"Avg latency: {np.mean(latencies)} seconds")
for percentage, percentile in zip(percentages, percentiles):
print(f"{percentage}% percentile latency: {percentile} seconds")
# Output JSON results if specified
if args.output_json:
results = {
"model_id": args.model,
"avg_latency": np.mean(latencies),
"latencies": latencies.tolist(),
"percentiles": dict(zip(percentages, percentiles.tolist())),
}
with open(args.output_json, "w") as f:
json.dump(results, f, indent=4)
save_to_pytorch_benchmark_format(args, results)
-3
View File
@@ -1,3 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Benchmark library utilities."""
File diff suppressed because it is too large Load Diff
-80
View File
@@ -1,80 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Utilities for checking endpoint readiness."""
import asyncio
import time
import aiohttp
from tqdm.asyncio import tqdm
from vllm.logger import init_logger
from .endpoint_request_func import RequestFunc, RequestFuncInput, RequestFuncOutput
logger = init_logger(__name__)
async def wait_for_endpoint(
request_func: RequestFunc,
test_input: RequestFuncInput,
session: aiohttp.ClientSession,
timeout_seconds: int = 600,
retry_interval: int = 5,
) -> RequestFuncOutput:
"""Wait for an endpoint to become available before starting benchmarks.
Args:
request_func: The async request function to call
test_input: The RequestFuncInput to test with
session: The aiohttp session used to issue the probe requests
timeout_seconds: Maximum time to wait in seconds (default: 10 minutes)
retry_interval: Time between retries in seconds (default: 5 seconds)
Returns:
RequestFuncOutput: The successful response
Raises:
ValueError: If the endpoint doesn't become available within the timeout
"""
deadline = time.perf_counter() + timeout_seconds
output = RequestFuncOutput(success=False)
print(f"Waiting for endpoint to become up in {timeout_seconds} seconds")
with tqdm(
total=timeout_seconds,
bar_format="{desc} |{bar}| {elapsed} elapsed, {remaining} remaining",
unit="s",
) as pbar:
while True:
# update progress bar
remaining = deadline - time.perf_counter()
elapsed = timeout_seconds - remaining
update_amount = min(elapsed - pbar.n, timeout_seconds - pbar.n)
pbar.update(update_amount)
pbar.refresh()
if remaining <= 0:
pbar.close()
break
# ping the endpoint using request_func
try:
output = await request_func(
request_func_input=test_input, session=session
)
if output.success:
pbar.close()
return output
else:
err_last_line = str(output.error).rstrip().rsplit("\n", 1)[-1]
logger.warning("Endpoint is not ready. Error='%s'", err_last_line)
except aiohttp.ClientConnectorError:
pass
# retry after a delay
sleep_duration = min(retry_interval, remaining)
if sleep_duration > 0:
await asyncio.sleep(sleep_duration)
return output
-139
View File
@@ -1,139 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
import json
import math
import os
from contextlib import contextmanager
from typing import Any
def redact_sensitive_namespace(
args: argparse.Namespace, fields: tuple[str, ...]
) -> argparse.Namespace:
"""Return a copy of CLI arguments with selected values redacted."""
redacted_args = argparse.Namespace(**vars(args))
for field in fields:
if getattr(redacted_args, field, None) is not None:
setattr(redacted_args, field, "***")
return redacted_args
def extract_field(
args: argparse.Namespace, extra_info: dict[str, Any], field_name: str
) -> str:
if field_name in extra_info:
return extra_info[field_name]
v: Any = args
# For example, args.compilation_config.mode
for nested_field in field_name.split("."):
if not hasattr(v, nested_field):
return ""
v = getattr(v, nested_field)
return v
def use_compile(args: argparse.Namespace, extra_info: dict[str, Any]) -> bool:
"""Check if the benchmark is run with torch.compile."""
return not (
extract_field(args, extra_info, "compilation_config.mode") == "0"
or "eager" in getattr(args, "output_json", "")
or "eager" in getattr(args, "result_filename", "")
)
def convert_to_pytorch_benchmark_format(
args: argparse.Namespace, metrics: dict[str, list], extra_info: dict[str, Any]
) -> list:
"""Save the benchmark results in the format used by PyTorch OSS benchmark with
on metric per record
https://github.com/pytorch/pytorch/wiki/How-to-integrate-with-PyTorch-OSS-benchmark-database
"""
records: list[Any] = []
if not os.environ.get("SAVE_TO_PYTORCH_BENCHMARK_FORMAT", False):
return records
for name, benchmark_values in metrics.items():
if not isinstance(benchmark_values, list):
raise TypeError(
f"benchmark_values for metric '{name}' must be a list, "
f"but got {type(benchmark_values).__name__}"
)
record = {
"benchmark": {
"name": "vLLM benchmark",
"extra_info": {
"args": vars(args),
"compilation_config.mode": extract_field(
args, extra_info, "compilation_config.mode"
),
"optimization_level": extract_field(
args, extra_info, "optimization_level"
),
# A boolean field used by vLLM benchmark HUD dashboard
"use_compile": use_compile(args, extra_info),
},
},
"model": {
"name": args.model,
},
"metric": {
"name": name,
"benchmark_values": benchmark_values,
"extra_info": extra_info,
},
}
tp = record["benchmark"]["extra_info"]["args"].get("tensor_parallel_size")
# Save tensor_parallel_size parameter if it's part of the metadata
if not tp and "tensor_parallel_size" in extra_info:
record["benchmark"]["extra_info"]["args"]["tensor_parallel_size"] = (
extra_info["tensor_parallel_size"]
)
records.append(record)
return records
class InfEncoder(json.JSONEncoder):
def clear_inf(self, o: Any):
if isinstance(o, dict):
return {
str(k)
if not isinstance(k, (str, int, float, bool, type(None)))
else k: self.clear_inf(v)
for k, v in o.items()
}
elif isinstance(o, list):
return [self.clear_inf(v) for v in o]
elif isinstance(o, float) and math.isinf(o):
return "inf"
return o
def iterencode(self, o: Any, *args, **kwargs) -> Any:
return super().iterencode(self.clear_inf(o), *args, **kwargs)
def write_to_json(filename: str, records: list) -> None:
with open(filename, "w") as f:
json.dump(
records,
f,
cls=InfEncoder,
default=lambda o: f"<{type(o).__name__} is not JSON serializable>",
)
@contextmanager
def default_vllm_config():
"""Set a default VllmConfig for cases that directly test CustomOps or pathways
that use get_current_vllm_config() outside of a full engine context.
"""
from vllm.config import VllmConfig, set_current_vllm_config
with set_current_vllm_config(VllmConfig()):
yield
-533
View File
@@ -1,533 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
r"""Benchmark multimodal processor latency.
This benchmark measures the latency of the mm processor module
using multimodal prompts from datasets.
MM processor stats are automatically enabled.
Run:
vllm bench mm-processor \
--model <your_model> \
--dataset-name random-mm \
--num-prompts 10 \
"""
import argparse
import json
import time
from collections import defaultdict
from datetime import datetime
from typing import TYPE_CHECKING, Any, Literal
import numpy as np
from vllm.benchmarks.datasets import (
MultiModalConversationDataset,
VisionArenaDataset,
)
from vllm.benchmarks.throughput import get_requests
from vllm.engine.arg_utils import EngineArgs
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.utils.gc_utils import freeze_gc_heap
from vllm.utils.import_utils import PlaceholderModule
try:
import pandas as pd
except ImportError:
pd = PlaceholderModule("pandas")
if TYPE_CHECKING: # Avoid having to mock during docs build
from vllm.v1.engine.llm_engine import LLMEngine
else:
LLMEngine = object
def get_timing_stats_from_engine(llm_engine: LLMEngine) -> dict[str, dict[str, float]]:
"""Get all multimodal timing stats from the LLM engine.
Collects both preprocessing stats (HF processor, hashing, cache lookup,
prompt update) and encoder forward pass timing, merged by request_id.
Args:
llm_engine: The LLM engine (has input_processor and workers).
Returns:
Dictionary mapping request_id to merged stats dict containing
both preprocessing and encoder timing metrics.
Example:
{
'request-123': {
'get_mm_hashes_secs': 0.02,
'get_cache_missing_items_secs': 0.01,
'apply_hf_processor_secs': 0.45,
'merge_mm_kwargs_secs': 0.01,
'apply_prompt_updates_secs': 0.03,
'preprocessor_total_secs': 0.51,
'encoder_forward_secs': 0.23,
'num_encoder_calls': 1
}
}
"""
observability_config = llm_engine.vllm_config.observability_config
if not observability_config or not observability_config.enable_mm_processor_stats:
return {}
renderer = llm_engine.renderer
mm_processor_stats = renderer._mm_timing_registry.stat()
encoder_stats = dict[str, dict[str, float]]()
for worker_stats in llm_engine.collective_rpc("get_encoder_timing_stats"):
if not worker_stats:
continue
for request_id, stats_dict in worker_stats.items():
if request_id not in encoder_stats:
encoder_stats[request_id] = dict(stats_dict)
else:
# Aggregate timing metrics across workers
current_time = encoder_stats[request_id].get(
"encoder_forward_secs", 0.0
)
new_time = stats_dict.get("encoder_forward_secs", 0.0)
encoder_stats[request_id]["encoder_forward_secs"] = max(
current_time, new_time
)
current_calls = encoder_stats[request_id].get("num_encoder_calls", 0)
new_calls = stats_dict.get("num_encoder_calls", 0)
encoder_stats[request_id]["num_encoder_calls"] = max(
current_calls, new_calls
)
merged_stats = dict[str, dict[str, float]]()
for request_id, prep_dict in mm_processor_stats.items():
merged_stats[request_id] = dict(prep_dict)
for request_id, enc_dict in encoder_stats.items():
if request_id in merged_stats:
merged_stats[request_id].update(enc_dict)
continue
# In V1 engine, the request_id in encoder_stats has a suffix
# appended to the original request_id (which is used in
# preprocessing_stats).
# We try to strip the suffix to find the matching request.
possible_original_id = request_id.rpartition("-")[0]
if possible_original_id and possible_original_id in merged_stats:
merged_stats[possible_original_id].update(enc_dict)
else:
merged_stats[request_id] = dict(enc_dict)
return merged_stats
def collect_mm_processor_stats(llm_engine: LLMEngine) -> dict[str, list[float]]:
"""Collect multimodal processor timing stats.
Returns a dictionary mapping stage names to lists of timing values (in seconds).
"""
all_stats = get_timing_stats_from_engine(llm_engine)
stats_by_stage = defaultdict[str, list[float]](list)
for stats_dict in all_stats.values():
for stat_key, stat_val in stats_dict.items():
stats_by_stage[stat_key].append(stat_val)
return stats_by_stage
def calculate_mm_processor_metrics(
stats_by_stage: dict[str, list[float]],
selected_percentiles: list[float],
*,
unit: Literal["us", "ms", "s"] = "ms",
) -> dict[str, dict[str, float]]:
"""Calculate aggregate metrics from stats by stage."""
unit2mult = {"us": 1000000, "ms": 1000, "s": 1}
unit_mult = unit2mult[unit]
metrics = {}
for stage, times in stats_by_stage.items():
stage_name = stage.replace("_secs", "_" + unit)
if not times:
metrics[stage_name] = {
"mean": 0.0,
"median": 0.0,
"std": 0.0,
**{f"p{p}": 0.0 for p in selected_percentiles},
}
continue
is_count_metric = stage == "num_encoder_calls"
values = times if is_count_metric else [t * unit_mult for t in times]
metrics[stage_name] = {
"mean": float(np.mean(values)),
"median": float(np.median(values)),
"std": float(np.std(values)),
**{f"p{p}": float(np.percentile(values, p)) for p in selected_percentiles},
}
return metrics
def validate_args(args):
"""Validate command-line arguments for mm_processor benchmark."""
if not getattr(args, "tokenizer", None):
args.tokenizer = args.model
if not hasattr(args, "dataset_path"):
args.dataset_path = None
if not hasattr(args, "lora_path"):
args.lora_path = None
if not hasattr(args, "max_loras"):
args.max_loras = None
if args.dataset_name == "hf" and not args.dataset_path:
raise ValueError(
"--dataset-path is required when using --dataset-name hf. "
"For multimodal benchmarking, specify a dataset like "
"'lmarena-ai/VisionArena-Chat'."
)
if args.dataset_name == "hf":
supported_mm_datasets = (
VisionArenaDataset.SUPPORTED_DATASET_PATHS.keys()
| MultiModalConversationDataset.SUPPORTED_DATASET_PATHS
)
if args.dataset_path not in supported_mm_datasets:
raise ValueError(
f"{args.dataset_path} is not a supported multimodal dataset. "
f"Supported multimodal datasets are: {sorted(supported_mm_datasets)}"
)
def benchmark_multimodal_processor(
args: argparse.Namespace,
) -> dict[str, Any]:
"""Run the multimodal processor benchmark."""
from vllm import LLM, SamplingParams
validate_args(args)
if args.seed is None:
args.seed = 0
engine_args = EngineArgs.from_cli_args(args)
llm = LLM.from_engine_args(engine_args)
tokenizer = llm.get_tokenizer()
requests = get_requests(args, tokenizer)
assert all(
llm.llm_engine.model_config.max_model_len
>= (request.prompt_len + request.expected_output_len)
for request in requests
), (
"Please ensure that max_model_len is greater than the sum of "
"prompt_len and expected_output_len for all requests."
)
prompts = [request.prompt for request in requests]
expected_output_lens = [request.expected_output_len for request in requests]
sampling_params = [
SamplingParams(
n=1,
temperature=0.0,
max_tokens=output_len,
detokenize=True,
watermarking=False,
)
for output_len in expected_output_lens
]
selected_percentiles = [
float(p) for p in getattr(args, "metric_percentiles", "99").split(",")
]
freeze_gc_heap()
num_warmups = getattr(args, "num_warmups", 0)
if num_warmups > 0:
print(f"Processing {num_warmups} warmup requests...")
# Create a temporary args object for warmup requests
warmup_args = argparse.Namespace(**vars(args))
warmup_args.num_prompts = num_warmups
warmup_args.seed += 1
warmup_requests = get_requests(warmup_args, tokenizer)
warmup_prompts = [req.prompt for req in warmup_requests]
warmup_output_lens = [req.expected_output_len for req in warmup_requests]
warmup_sampling_params = [
SamplingParams(max_tokens=output_len, watermarking=False)
for output_len in warmup_output_lens
]
llm.chat(
warmup_prompts,
warmup_sampling_params,
use_tqdm=not getattr(args, "disable_tqdm", False),
)
# Clear stats from warmup requests
collect_mm_processor_stats(llm.llm_engine)
print(f"Processing {len(prompts)} requests...")
start_time = time.perf_counter()
outputs = llm.chat(
prompts, sampling_params, use_tqdm=not getattr(args, "disable_tqdm", False)
)
end_time = time.perf_counter()
total_time = end_time - start_time
mm_stats_by_stage = collect_mm_processor_stats(llm.llm_engine)
if not any(mm_stats_by_stage.values()):
print(
"\n⚠️ Warning: No MM processor stats found in registry.\n"
" This may indicate that:\n"
" - No multimodal requests were processed\n"
" - Stats were already retrieved (registry is cleared after retrieval)\n"
)
mm_processor_metrics = calculate_mm_processor_metrics(
mm_stats_by_stage, selected_percentiles
)
completed = len([o for o in outputs if o.finished])
failed = len(outputs) - completed
e2el_times = []
for output in outputs:
if not output.finished or output.metrics is None:
continue
metrics = output.metrics
# Calculate E2E latency as: TTFT + (last_token_ts - first_token_ts)
if (
getattr(metrics, "first_token_latency", None) is not None
and getattr(metrics, "last_token_ts", None) is not None
and getattr(metrics, "first_token_ts", None) is not None
):
ttft = metrics.first_token_latency
# Decode time is the duration between the first and last token generation
decode_time = max(0.0, metrics.last_token_ts - metrics.first_token_ts)
e2el_times.append((ttft + decode_time) * 1000)
if not e2el_times and completed > 0:
print(
"\n⚠️ Warning: Detailed end-to-end latency metrics not available.\n"
" Falling back to average request latency "
"(total_time / num_completed_requests).\n"
)
avg_time_per_request = total_time / completed
e2el_times = [avg_time_per_request * 1000] * completed
if e2el_times:
mean_e2el_ms = float(np.mean(e2el_times))
median_e2el_ms = float(np.median(e2el_times))
std_e2el_ms = float(np.std(e2el_times))
percentiles_e2el_ms = [
(p, float(np.percentile(e2el_times, p))) for p in selected_percentiles
]
else:
mean_e2el_ms = 0.0
median_e2el_ms = 0.0
std_e2el_ms = 0.0
percentiles_e2el_ms = [(p, 0.0) for p in selected_percentiles]
encoder_summary = {}
if (
"num_encoder_calls" in mm_stats_by_stage
and mm_stats_by_stage["num_encoder_calls"]
):
encoder_calls = mm_stats_by_stage["num_encoder_calls"]
encoder_summary = {
"total_encoder_calls": int(sum(encoder_calls)),
"num_requests_with_encoder_calls": len(encoder_calls),
}
benchmark_result = {
"completed": completed,
"failed": failed,
"mean_e2el_ms": mean_e2el_ms,
"median_e2el_ms": median_e2el_ms,
"std_e2el_ms": std_e2el_ms,
"percentiles_e2el_ms": percentiles_e2el_ms,
"mm_processor_stats": mm_processor_metrics,
"encoder_summary": encoder_summary,
}
return benchmark_result
def add_cli_args(parser: FlexibleArgumentParser) -> None:
"""Add CLI arguments for the multimodal processor benchmark."""
from vllm.engine.arg_utils import EngineArgs
EngineArgs.add_cli_args(parser)
parser.set_defaults(enable_mm_processor_stats=True)
parser.add_argument(
"--dataset-name",
type=str,
default="random-mm",
choices=["random-mm", "hf"],
help="Name of the dataset to benchmark on. Defaults to 'random-mm'.",
)
parser.add_argument(
"--num-prompts",
type=int,
default=10,
help="Number of prompts to process.",
)
parser.add_argument(
"--num-warmups",
type=int,
default=1,
help="Number of warmup prompts to process.",
)
from vllm.benchmarks.datasets import (
add_random_dataset_base_args,
add_random_multimodal_dataset_args,
)
add_random_dataset_base_args(parser)
add_random_multimodal_dataset_args(parser)
# HuggingFace dataset arguments
parser.add_argument(
"--dataset-path",
type=str,
default=None,
help="Path to the dataset file or HuggingFace dataset name "
"(e.g., 'yale-nlp/MMVU', 'lmarena-ai/VisionArena-Chat').",
)
parser.add_argument(
"--hf-subset",
type=str,
default=None,
help="Subset of the HuggingFace dataset (optional).",
)
parser.add_argument(
"--hf-split",
type=str,
default=None,
help="Split of the HuggingFace dataset (e.g., 'train', 'test', 'validation').",
)
parser.add_argument(
"--output-len",
type=int,
default=None,
help="Output length for each request. "
"Overrides the default output lengths from the dataset.",
)
parser.add_argument(
"--output-json",
type=str,
default=None,
help="Path to save the benchmark results in JSON format.",
)
parser.add_argument(
"--metric-percentiles",
type=str,
default="99",
help="Comma-separated list of percentiles to calculate (e.g., '50,90,99').",
)
parser.add_argument(
"--disable-tqdm",
action="store_true",
help="Disable tqdm progress bar.",
)
def main(args: argparse.Namespace) -> None:
"""Main entry point for the multimodal processor benchmark."""
print("Starting multimodal processor benchmark...")
result = benchmark_multimodal_processor(args)
print("\n" + "=" * 80)
print("Multimodal Processor Benchmark Results")
print("=" * 80)
if "mm_processor_stats" in result:
print("\nMM Processor Metrics:")
selected_percentiles = [
float(p) for p in getattr(args, "metric_percentiles", "99").split(",")
]
mm_data = []
for stage, metrics in result["mm_processor_stats"].items():
row = {
"Stage": stage,
"Mean": f"{metrics['mean']:.2f}",
"Median": f"{metrics['median']:.2f}",
"Std": f"{metrics['std']:.2f}",
}
for p in selected_percentiles:
row[f"P{p}"] = f"{metrics.get(f'p{p}', 0.0):.2f}"
mm_data.append(row)
mm_df = pd.DataFrame(mm_data)
print(mm_df.to_string(index=False))
if "encoder_summary" in result and result["encoder_summary"]:
total_calls = result["encoder_summary"]["total_encoder_calls"]
num_requests = result["encoder_summary"]["num_requests_with_encoder_calls"]
print(
f"\nSummary: {total_calls} total encoder calls "
f"across {num_requests} requests."
)
if "mean_e2el_ms" in result:
print("\nEnd-to-End Latency (ms):")
selected_percentiles = [
float(p) for p in getattr(args, "metric_percentiles", "99").split(",")
]
e2el_data = [
{"Metric": "Mean", "Value (ms)": f"{result['mean_e2el_ms']:.2f}"},
{"Metric": "Median", "Value (ms)": f"{result['median_e2el_ms']:.2f}"},
{"Metric": "Std", "Value (ms)": f"{result['std_e2el_ms']:.2f}"},
]
for p in selected_percentiles:
percentile_value = next(
(val for pct, val in result["percentiles_e2el_ms"] if pct == p),
0.0,
)
e2el_data.append(
{
"Metric": f"P{p}",
"Value (ms)": f"{percentile_value:.2f}",
}
)
e2el_df = pd.DataFrame(e2el_data)
print(e2el_df.to_string(index=False))
if args.output_json:
result["config"] = {
"model": args.model,
"num_prompts": args.num_prompts,
"input_len": getattr(args, "random_input_len", None),
"output_len": getattr(args, "random_output_len", None),
}
result["timestamp"] = datetime.now().isoformat()
with open(args.output_json, "w") as f:
json.dump(result, f, indent=2)
print(f"\nResults saved to {args.output_json}")
if __name__ == "__main__":
parser = FlexibleArgumentParser(description="Benchmark mm processor latency")
add_cli_args(parser)
args = parser.parse_args()
main(args)
-315
View File
@@ -1,315 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Generate plots for benchmark results."""
from pathlib import Path
from typing import Any
from vllm.utils.import_utils import PlaceholderModule
try:
import plotly.express as px
import plotly.io as pio
except ImportError:
_plotly = PlaceholderModule("plotly")
px = _plotly.placeholder_attr("express")
pio = _plotly.placeholder_attr("io")
try:
import matplotlib.pyplot as plt
except ImportError:
_matplotlib = PlaceholderModule("matplotlib")
plt = _matplotlib.placeholder_attr("pyplot")
def generate_timeline_plot(
results: list[dict[str, Any]],
output_path: Path,
colors: list[str] | None = None,
itl_thresholds: list[float] | None = None,
labels: list[str] | None = None,
) -> None:
"""Generate an HTML timeline plot from benchmark results.
Args:
results: List of per-request result dictionaries containing:
- start_time: Request start time (seconds)
- ttft: Time to first token (seconds)
- itl: List of inter-token latencies (seconds)
- latency: Total request latency (seconds)
- prompt_len: Number of prompt tokens
- output_tokens: Number of output tokens
output_path: Path where the HTML file will be saved
colors: List of colors for ITL categories (default: green, orange, red, black)
itl_thresholds: ITL thresholds in seconds (default: [1.0, 4.0, 6.0])
labels: Labels for ITL categories (default based on thresholds)
"""
# Set defaults
if colors is None:
colors = ["#109618", "#FF7F0E", "#D62728"]
if itl_thresholds is None:
itl_thresholds = [0.025, 0.050]
if labels is None:
labels = [
f"ITL < {itl_thresholds[0] * 1000:.0f}ms",
f"{itl_thresholds[0] * 1000:.0f}ms ≤ ITL < {itl_thresholds[1] * 1000:.0f}ms", # noqa
f"ITL ≥ {itl_thresholds[1] * 1000:.0f}ms",
]
labels_colors = {"TTFT": "#636EFA", **dict(zip(labels, colors))}
labels_order = ["TTFT"] + labels
timeline_data = construct_timeline_data(results, itl_thresholds, labels)
if not timeline_data:
print("No timeline data to plot")
return
# Create the plot
fig = px.timeline(
timeline_data,
x_start="start",
x_end="end",
y="request_id",
color="type",
color_discrete_map=labels_colors,
category_orders={"type": labels_order},
hover_data=[
"prompt_tokens",
"output_tokens",
"req_start_time",
"req_finish_time",
"segment_start",
"segment_end",
"duration",
],
)
# Customize hover template to show only time without date
fig.update_traces(
hovertemplate="<b>%{y}</b><br>"
"Type: %{fullData.name}<br>"
"Start: %{customdata[4]}<br>"
"End: %{customdata[5]}<br>"
"Duration: %{customdata[6]}<br>"
"Prompt Tokens: %{customdata[0]}<br>"
"Output Tokens: %{customdata[1]}<br>"
"Request Start Time: %{customdata[2]}<br>"
"Request End Time: %{customdata[3]}<br>"
"<extra></extra>"
)
fig.update_yaxes(autorange="reversed")
fig.update_layout(
xaxis_title="Time",
yaxis_title="Request ID",
showlegend=True,
)
# Save to HTML
pio.write_html(fig, str(output_path))
print(f"Timeline plot saved to: {output_path}")
def construct_timeline_data(
requests_data: list[dict[str, Any]],
itl_thresholds: list[float],
labels: list[str],
) -> list[dict[str, Any]]:
"""Construct timeline data from request results.
Args:
requests_data: List of per-request result dictionaries
itl_thresholds: ITL thresholds in seconds
labels: Labels for ITL categories
Returns:
List of timeline segments for plotting
"""
def tostr(sec_time: float) -> str:
"""Convert seconds to HH:MM:SS.mmm format."""
h = int(sec_time // 3600)
assert h < 100, "time seems to last more than 100 hours"
m = int((sec_time % 3600) // 60)
s = sec_time % 60
return f"{h:02d}:{m:02d}:{s:06.3f}"
def itl_type(itl: float) -> str:
"""Categorize ITL based on thresholds."""
if itl < itl_thresholds[0]:
return labels[0]
elif itl < itl_thresholds[1]:
return labels[1]
else:
return labels[2]
# Find the earliest start time to use as t0
t0 = None
for request in requests_data:
start_time = request.get("start_time")
if start_time is not None and (t0 is None or start_time < t0):
t0 = start_time
if t0 is None:
return []
timeline_data = []
for i, request in enumerate(requests_data):
start_time = request.get("start_time")
ttft = request.get("ttft")
itl = request.get("itl", [])
latency = request.get("latency")
prompt_len = request.get("prompt_len", 0)
output_tokens = request.get("output_tokens", 0)
# Skip requests without required data
if start_time is None or ttft is None or latency is None:
continue
# Normalize start time
start_time = start_time - t0
start_time_str = tostr(start_time)
# TTFT segment
ttft_end = start_time + ttft
ttft_end_str = tostr(ttft_end)
timeline_data.append(
{
"request_id": f"Req {i}",
"start": start_time_str,
"end": ttft_end_str,
"type": "TTFT",
"prompt_tokens": prompt_len,
"output_tokens": output_tokens,
"req_start_time": tostr(start_time),
"req_finish_time": tostr(start_time + latency),
"segment_start": start_time_str,
"segment_end": ttft_end_str,
"duration": f"{ttft:.3f}s",
}
)
# ITL segments
prev_time = ttft_end
prev_time_str = ttft_end_str
for itl_value in itl:
itl_end = prev_time + itl_value
itl_end_str = tostr(itl_end)
timeline_data.append(
{
"request_id": f"Req {i}",
"start": prev_time_str,
"end": itl_end_str,
"type": itl_type(itl_value),
"prompt_tokens": prompt_len,
"output_tokens": output_tokens,
"req_start_time": tostr(start_time),
"req_finish_time": tostr(start_time + latency),
"segment_start": prev_time_str,
"segment_end": itl_end_str,
"duration": f"{itl_value:.3f}s",
}
)
prev_time = itl_end
prev_time_str = itl_end_str
return timeline_data
def generate_dataset_stats_plot(
results: list[dict[str, Any]],
output_path: Path,
) -> None:
"""Generate a matplotlib figure with dataset statistics.
Creates a figure with 4 subplots:
- Top-left: Prompt tokens distribution (histogram)
- Top-right: Output tokens distribution (histogram)
- Bottom-left: Prompt+output tokens distribution (histogram)
- Bottom-right: Stacked bar chart (request_id vs tokens)
Args:
results: List of per-request result dictionaries containing:
- prompt_len: Number of prompt tokens
- output_tokens: Number of output tokens
output_path: Path where the figure will be saved
"""
# Extract data
prompt_tokens = []
output_tokens = []
total_tokens = []
for request in results:
prompt_len = request.get("prompt_len", 0)
output_len = request.get("output_tokens", 0)
prompt_tokens.append(prompt_len)
output_tokens.append(output_len)
total_tokens.append(prompt_len + output_len)
if not prompt_tokens:
print("No data available for dataset statistics plot")
return
# Create figure with 4 subplots
fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(14, 10))
# Top-left: Prompt tokens distribution
ax1.hist(prompt_tokens, bins=30, color="steelblue", edgecolor="black", alpha=0.7)
ax1.set_xlabel("Prompt Tokens")
ax1.set_ylabel("Frequency")
ax1.set_title("Prompt Tokens Distribution")
ax1.grid(True, alpha=0.3)
# Top-right: Output tokens distribution
ax2.hist(output_tokens, bins=30, color="coral", edgecolor="black", alpha=0.7)
ax2.set_xlabel("Output Tokens")
ax2.set_ylabel("Frequency")
ax2.set_title("Output Tokens Distribution")
ax2.grid(True, alpha=0.3)
# Bottom-left: Prompt+output tokens distribution
ax3.hist(
total_tokens, bins=30, color="mediumseagreen", edgecolor="black", alpha=0.7
)
ax3.set_xlabel("Total Tokens (Prompt + Output)")
ax3.set_ylabel("Frequency")
ax3.set_title("Total Tokens Distribution")
ax3.grid(True, alpha=0.3)
# Bottom-right: Stacked bar chart
request_ids = list(range(len(prompt_tokens)))
ax4.bar(
request_ids, prompt_tokens, label="Prompt Tokens", color="steelblue", alpha=0.7
)
ax4.bar(
request_ids,
output_tokens,
bottom=prompt_tokens,
label="Output Tokens",
color="coral",
alpha=0.7,
)
ax4.set_xlabel("Request ID")
ax4.set_ylabel("Tokens")
ax4.set_title("Tokens per Request (Stacked)")
ax4.legend()
ax4.grid(True, alpha=0.3, axis="y")
# Adjust layout to prevent overlap
plt.tight_layout()
# Save figure
plt.savefig(str(output_path), dpi=150, bbox_inches="tight")
plt.close(fig)
print(f"Dataset statistics plot saved to: {output_path}")
File diff suppressed because it is too large Load Diff
-302
View File
@@ -1,302 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Benchmark the cold and warm startup time of vLLM models.
This script measures total startup time (including model loading, compilation,
and cache operations) for both cold and warm scenarios:
- Cold startup: Fresh start with no caches (temporary cache directories)
- Warm startup: Using cached compilation and model info
"""
import argparse
import json
import multiprocessing
import os
import shutil
import tempfile
import time
from contextlib import contextmanager
from typing import Any, NamedTuple
import numpy as np
from tqdm import tqdm
from vllm.benchmarks.lib.utils import (
convert_to_pytorch_benchmark_format,
write_to_json,
)
from vllm.engine.arg_utils import EngineArgs
from vllm.utils.argparse_utils import FlexibleArgumentParser
PERCENTAGES = [10, 25, 50, 75, 90, 99]
class MetricDesc(NamedTuple):
"""Descriptor for a metric to collect from each iteration."""
iter_key: str # key in the iteration result dict
suffix: str # result key suffix, e.g. "startup", "compilation"
display_name: str
class MetricStats(NamedTuple):
"""Aggregated statistics for a single benchmark metric."""
key: str # e.g. "cold_startup", "warm_encoder_compilation"
display_name: str
values: list[float]
avg: float
percentiles: dict[int, float]
_BASE_METRICS = [
MetricDesc("total_startup_time", "startup", "Startup time"),
MetricDesc("compilation_time", "compilation", "Compilation time"),
]
_ENCODER_METRIC = MetricDesc(
"encoder_compilation_time",
"encoder_compilation",
"Encoder compilation time",
)
def _compute_metric(
phase: str,
desc: MetricDesc,
iterations: list[dict[str, float]],
) -> MetricStats:
values = [m[desc.iter_key] for m in iterations]
arr = np.array(values)
return MetricStats(
key=f"{phase}_{desc.suffix}",
display_name=desc.display_name,
values=values,
avg=float(np.mean(arr)),
percentiles=dict(zip(PERCENTAGES, np.percentile(arr, PERCENTAGES).tolist())),
)
def _collect_phase_metrics(
phase: str,
iterations: list[dict[str, float]],
has_encoder: bool,
) -> list[MetricStats]:
metrics = [_compute_metric(phase, desc, iterations) for desc in _BASE_METRICS]
if has_encoder:
metrics.append(_compute_metric(phase, _ENCODER_METRIC, iterations))
return metrics
def _print_phase(phase_name: str, metrics: list[MetricStats]) -> None:
print(f"\n{phase_name}:")
for m in metrics:
print(f"Avg {m.display_name.lower()}: {m.avg:.2f} seconds")
for m in metrics:
print(f"{m.display_name} percentiles:")
for pct, val in m.percentiles.items():
print(f" {pct}%: {val:.2f} seconds")
def _metric_to_json(m: MetricStats) -> dict[str, Any]:
return {
f"avg_{m.key}_time": m.avg,
f"{m.key}_times": m.values,
f"{m.key}_percentiles": m.percentiles,
}
@contextmanager
def cold_startup():
"""Context manager to measure cold startup time:
1. Uses a temporary directory for vLLM cache to avoid any pollution
between cold startup iterations.
2. Uses inductor's fresh_cache to clear torch.compile caches.
"""
from torch._inductor.utils import fresh_cache
# Use temporary directory for caching to avoid any pollution between cold startups
original_cache_root = os.environ.get("VLLM_CACHE_ROOT")
temp_cache_dir = tempfile.mkdtemp(prefix="vllm_startup_bench_cold_")
try:
os.environ["VLLM_CACHE_ROOT"] = temp_cache_dir
with fresh_cache():
yield
finally:
# Clean up temporary cache directory
shutil.rmtree(temp_cache_dir, ignore_errors=True)
if original_cache_root:
os.environ["VLLM_CACHE_ROOT"] = original_cache_root
else:
os.environ.pop("VLLM_CACHE_ROOT", None)
def run_startup_in_subprocess(engine_args, result_queue):
"""Run LLM startup in a subprocess and return timing metrics via a queue.
This ensures complete isolation between iterations.
"""
try:
# Import inside the subprocess to avoid issues with forking
from vllm import LLM
# Measure total startup time
start_time = time.perf_counter()
llm = LLM.from_engine_args(engine_args)
total_startup_time = time.perf_counter() - start_time
# Extract compilation time if available
compilation_time = 0.0
encoder_compilation_time = 0.0
if hasattr(llm.llm_engine, "vllm_config"):
vllm_config = llm.llm_engine.vllm_config
if (
hasattr(vllm_config, "compilation_config")
and vllm_config.compilation_config is not None
):
compilation_time = vllm_config.compilation_config.compilation_time
encoder_compilation_time = (
vllm_config.compilation_config.encoder_compilation_time
)
result_queue.put(
{
"total_startup_time": total_startup_time,
"compilation_time": compilation_time,
"encoder_compilation_time": encoder_compilation_time,
}
)
except Exception as e:
result_queue.put(None)
result_queue.put(str(e))
def save_to_pytorch_benchmark_format(
args: argparse.Namespace, metrics: list[MetricStats]
) -> None:
base_name = os.path.splitext(args.output_json)[0]
for m in metrics:
records = convert_to_pytorch_benchmark_format(
args=args,
metrics={f"avg_{m.key}_time": [m.avg]},
extra_info={
f"{m.key}_times": m.values,
f"{m.key}_percentiles": m.percentiles,
},
)
if records:
write_to_json(f"{base_name}.{m.key}.pytorch.json", records)
def add_cli_args(parser: FlexibleArgumentParser):
parser.add_argument(
"--num-iters-cold",
type=int,
default=3,
help="Number of cold startup iterations.",
)
parser.add_argument(
"--num-iters-warmup",
type=int,
default=1,
help="Number of warmup iterations before benchmarking warm startups.",
)
parser.add_argument(
"--num-iters-warm",
type=int,
default=3,
help="Number of warm startup iterations.",
)
parser.add_argument(
"--output-json",
type=str,
default=None,
help="Path to save the startup time results in JSON format.",
)
parser = EngineArgs.add_cli_args(parser)
return parser
def main(args: argparse.Namespace):
# Set multiprocessing start method to 'spawn' for clean process isolation
# This ensures each subprocess starts fresh without inheriting state
multiprocessing.set_start_method("spawn", force=True)
engine_args = EngineArgs.from_cli_args(args)
def create_llm_and_measure_startup():
"""Create LLM instance in a subprocess and measure startup time.
Returns timing metrics, using subprocess for complete isolation.
"""
# Create a queue for inter-process communication
result_queue: multiprocessing.Queue[Any] = multiprocessing.Queue()
process = multiprocessing.Process(
target=run_startup_in_subprocess,
args=(
engine_args,
result_queue,
),
)
process.start()
process.join()
if not result_queue.empty():
result = result_queue.get()
if result is None:
if not result_queue.empty():
error_msg = result_queue.get()
raise RuntimeError(f"Subprocess failed: {error_msg}")
else:
raise RuntimeError("Subprocess failed with unknown error")
return result
else:
raise RuntimeError("Subprocess did not return a result")
os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] = "0"
print("Setting VLLM_ENABLE_V1_MULTIPROCESSING=0 to collect startup metrics.\n")
# Collect cold startup iterations
print("Measuring cold startup time...\n")
cold_iterations = []
for i in tqdm(range(args.num_iters_cold), desc="Cold startup iterations"):
with cold_startup():
cold_iterations.append(create_llm_and_measure_startup())
# Warmup for warm startup
print("\nWarming up for warm startup measurement...\n")
for _ in tqdm(range(args.num_iters_warmup), desc="Warmup iterations"):
create_llm_and_measure_startup()
# Collect warm startup iterations
print("\nMeasuring warm startup time...\n")
warm_iterations = []
for i in tqdm(range(args.num_iters_warm), desc="Warm startup iterations"):
warm_iterations.append(create_llm_and_measure_startup())
# Determine if encoder compilation occurred in any iteration
has_encoder = any(
m["encoder_compilation_time"] > 0 for m in cold_iterations + warm_iterations
)
cold_metrics = _collect_phase_metrics("cold", cold_iterations, has_encoder)
warm_metrics = _collect_phase_metrics("warm", warm_iterations, has_encoder)
all_metrics = cold_metrics + warm_metrics
# Print results
print("\n" + "=" * 60)
print("STARTUP TIME BENCHMARK RESULTS")
print("=" * 60)
_print_phase("COLD STARTUP", cold_metrics)
_print_phase("WARM STARTUP", warm_metrics)
print("=" * 60)
# Output JSON results if specified
if args.output_json:
results: dict[str, Any] = {}
for m in all_metrics:
results.update(_metric_to_json(m))
with open(args.output_json, "w") as f:
json.dump(results, f, indent=4)
save_to_pytorch_benchmark_format(args, all_metrics)
View File
-45
View File
@@ -1,45 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
from vllm.utils.argparse_utils import FlexibleArgumentParser
from .plot import SweepPlotArgs
from .plot import main as plot_main
from .plot_pareto import SweepPlotParetoArgs
from .plot_pareto import main as plot_pareto_main
from .serve import SweepServeArgs
from .serve import main as serve_main
from .serve_workload import SweepServeWorkloadArgs
from .serve_workload import main as serve_workload_main
from .startup import SweepStartupArgs
from .startup import main as startup_main
SUBCOMMANDS = (
(SweepServeArgs, serve_main),
(SweepServeWorkloadArgs, serve_workload_main),
(SweepStartupArgs, startup_main),
(SweepPlotArgs, plot_main),
(SweepPlotParetoArgs, plot_pareto_main),
)
def add_cli_args(parser: FlexibleArgumentParser):
subparsers = parser.add_subparsers(required=True, dest="sweep_type")
for cmd, entrypoint in SUBCOMMANDS:
cmd_subparser = subparsers.add_parser(
cmd.parser_name,
description=cmd.parser_help,
usage=f"vllm bench sweep {cmd.parser_name} [options]",
)
cmd_subparser.set_defaults(dispatch_function=entrypoint)
cmd.add_cli_args(cmd_subparser)
cmd_subparser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(
subcmd=f"sweep {cmd.parser_name}"
)
def main(args: argparse.Namespace):
args.dispatch_function(args)
-156
View File
@@ -1,156 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import os
class ParameterSweep(list["ParameterSweepItem"]):
@classmethod
def read_json(cls, filepath: os.PathLike):
with open(filepath, "rb") as f:
data = json.load(f)
# Support both list and dict formats
if isinstance(data, dict):
return cls.read_from_dict(data)
return cls.from_records(data)
@classmethod
def read_from_dict(cls, data: dict[str, dict[str, object]]):
"""Read parameter sweep from a dict format where keys are names.
Example:
{
"experiment1": {"max_tokens": 100, "temperature": 0.7},
"experiment2": {"max_tokens": 200, "temperature": 0.9}
}
"""
records = [{"_benchmark_name": name, **params} for name, params in data.items()]
return cls.from_records(records)
@classmethod
def from_records(cls, records: list[dict[str, object]]):
if not isinstance(records, list):
raise TypeError(
f"The parameter sweep should be a list of dictionaries, "
f"but found type: {type(records)}"
)
# Validate that all _benchmark_name values are unique if provided
names = [r["_benchmark_name"] for r in records if "_benchmark_name" in r]
if names and len(names) != len(set(names)):
duplicates = [name for name in names if names.count(name) > 1]
raise ValueError(
f"Duplicate _benchmark_name values found: {set(duplicates)}. "
f"All _benchmark_name values must be unique."
)
return cls(ParameterSweepItem.from_record(record) for record in records)
class ParameterSweepItem(dict[str, object]):
@classmethod
def from_record(cls, record: dict[str, object]):
if not isinstance(record, dict):
raise TypeError(
f"Each item in the parameter sweep should be a dictionary, "
f"but found type: {type(record)}"
)
return cls(record)
def __or__(self, other: dict[str, object], /) -> "ParameterSweepItem": # type: ignore[override]
return ParameterSweepItem(super().__or__(other))
@property
def name(self) -> str:
"""Get the name for this parameter sweep item.
Returns the '_benchmark_name' field if present, otherwise returns a text
representation of all parameters.
"""
if "_benchmark_name" in self:
return str(self["_benchmark_name"])
return self.as_text(sep="-")
# In JSON, we prefer "_"
def _iter_param_key_candidates(self, param_key: str):
# Inner config arguments are not converted by the CLI
if "." in param_key:
prefix, rest = param_key.split(".", 1)
for prefix_candidate in self._iter_param_key_candidates(prefix):
yield prefix_candidate + "." + rest
return
yield param_key
yield param_key.replace("-", "_")
yield param_key.replace("_", "-")
# In CLI, we prefer "-"
def _iter_cmd_key_candidates(self, param_key: str):
for k in reversed(tuple(self._iter_param_key_candidates(param_key))):
yield "--" + k
def _normalize_cmd_key(self, param_key: str):
return next(self._iter_cmd_key_candidates(param_key))
def has_param(self, param_key: str) -> bool:
return any(k in self for k in self._iter_param_key_candidates(param_key))
def _normalize_cmd_kv_pair(self, k: str, v: object) -> list[str]:
"""Normalize a key-value pair into command-line arguments.
Returns a list containing either:
- A single element for boolean flags (e.g., ['--flag'] or ['--flag=true'])
- Two elements for key-value pairs (e.g., ['--key', 'value'])
"""
if isinstance(v, bool):
# For nested params (containing "."), use =true/false syntax
if "." in k:
return [f"{self._normalize_cmd_key(k)}={'true' if v else 'false'}"]
else:
return [self._normalize_cmd_key(k if v else "no-" + k)]
else:
return [self._normalize_cmd_key(k), str(v)]
def apply_to_cmd(self, cmd: list[str]) -> list[str]:
cmd = list(cmd)
for k, v in self.items():
# Skip the '_benchmark_name' field, not a parameter
if k == "_benchmark_name":
continue
# Serialize dict values as JSON
if isinstance(v, dict):
v = json.dumps(v)
for k_candidate in self._iter_cmd_key_candidates(k):
try:
k_idx = cmd.index(k_candidate)
# Replace existing parameter
normalized = self._normalize_cmd_kv_pair(k, v)
if len(normalized) == 1:
# Boolean flag
cmd[k_idx] = normalized[0]
else:
# Key-value pair
cmd[k_idx] = normalized[0]
cmd[k_idx + 1] = normalized[1]
break
except ValueError:
continue
else:
# Add new parameter
cmd.extend(self._normalize_cmd_kv_pair(k, v))
return cmd
def as_text(self, sep: str = ", ") -> str:
return sep.join(f"{k}={v}" for k, v in self.items() if k != "_benchmark_name")
-690
View File
@@ -1,690 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
import json
from abc import ABC, abstractmethod
from concurrent.futures import ProcessPoolExecutor
from dataclasses import dataclass
from functools import partial
from pathlib import Path
from types import TracebackType
from typing import TYPE_CHECKING, ClassVar, Self
from typing_extensions import override
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.utils.collection_utils import full_groupby
from vllm.utils.import_utils import PlaceholderModule
from .utils import sanitize_filename
if TYPE_CHECKING:
import pandas as pd
@dataclass
class PlotFilterBase(ABC):
var: str
target: str
@classmethod
def parse_str(cls, s: str):
for op_key in PLOT_FILTERS:
if op_key in s:
key, value = s.split(op_key)
return PLOT_FILTERS[op_key](
key,
value.removeprefix(op_key).strip("'").strip('"'),
)
else:
raise ValueError(
f"Invalid operator for plot filter '{s}'. "
f"Valid operators are: {sorted(PLOT_FILTERS)}",
)
@abstractmethod
def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
"""Applies this filter to a DataFrame."""
raise NotImplementedError
@dataclass
class PlotEqualTo(PlotFilterBase):
@override
def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
target: float | str
try:
target = float(self.target)
except ValueError:
target = self.target
return df[df[self.var] == target]
@dataclass
class PlotNotEqualTo(PlotFilterBase):
@override
def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
target: float | str
try:
target = float(self.target)
except ValueError:
target = self.target
return df[df[self.var] != target]
@dataclass
class PlotLessThan(PlotFilterBase):
@override
def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
return df[df[self.var] < float(self.target)]
@dataclass
class PlotLessThanOrEqualTo(PlotFilterBase):
@override
def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
return df[df[self.var] <= float(self.target)]
@dataclass
class PlotGreaterThan(PlotFilterBase):
@override
def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
return df[df[self.var] > float(self.target)]
@dataclass
class PlotGreaterThanOrEqualTo(PlotFilterBase):
@override
def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
return df[df[self.var] >= float(self.target)]
# NOTE: The ordering is important! Match longer op_keys first
PLOT_FILTERS: dict[str, type[PlotFilterBase]] = {
"==": PlotEqualTo,
"!=": PlotNotEqualTo,
"<=": PlotLessThanOrEqualTo,
">=": PlotGreaterThanOrEqualTo,
"<": PlotLessThan,
">": PlotGreaterThan,
}
class PlotFilters(list[PlotFilterBase]):
@classmethod
def parse_str(cls, s: str):
if not s:
return cls()
return cls(PlotFilterBase.parse_str(e) for e in s.split(","))
def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
for item in self:
df = item.apply(df)
return df
@dataclass
class PlotBinner:
var: str
bin_size: float
@classmethod
def parse_str(cls, s: str):
for op_key in PLOT_BINNERS:
if op_key in s:
key, value = s.split(op_key)
return PLOT_BINNERS[op_key](key, float(value.removeprefix(op_key)))
else:
raise ValueError(
f"Invalid operator for plot binner '{s}'. "
f"Valid operators are: {sorted(PLOT_BINNERS)}",
)
def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
"""Applies this binner to a DataFrame."""
df = df.copy()
df[self.var] = df[self.var] // self.bin_size * self.bin_size
return df
PLOT_BINNERS: dict[str, type[PlotBinner]] = {
"%": PlotBinner,
}
class PlotBinners(list[PlotBinner]):
@classmethod
def parse_str(cls, s: str):
if not s:
return cls()
return cls(PlotBinner.parse_str(e) for e in s.split(","))
def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
for item in self:
df = item.apply(df)
return df
def _json_load_bytes(path: Path) -> list[dict[str, object]]:
with path.open("rb") as f:
return json.load(f)
def _convert_inf_nan_strings(data: list[dict[str, object]]) -> list[dict[str, object]]:
"""Convert string values "inf", "-inf", and "nan" to their float equivalents.
This handles the case where JSON serialization represents inf/nan as strings.
"""
converted_data = []
for record in data:
converted_record: dict[str, object] = {}
for key, value in record.items():
if isinstance(value, str):
if value in ["inf", "-inf", "nan"]:
converted_record[key] = float(value)
else:
converted_record[key] = value
else:
converted_record[key] = value
converted_data.append(converted_record)
return converted_data
def _get_metric(run_data: dict[str, object], metric_key: str):
try:
return run_data[metric_key]
except KeyError as exc:
raise ValueError(f"Cannot find metric {metric_key!r} in {run_data=}") from exc
def _get_group(run_data: dict[str, object], group_keys: list[str]):
return tuple((k, str(_get_metric(run_data, k))) for k in group_keys)
def _get_fig_path(fig_dir: Path, group: tuple[tuple[str, str], ...], fig_name: str):
parts = list[str]()
# Start with figure name (always provided, defaults to "FIGURE")
parts.append(fig_name)
# Always append group data if present
if group:
parts.extend(f"{k}={v}" for k, v in group)
return fig_dir / sanitize_filename("-".join(parts) + ".png")
class DummyExecutor:
map = map
def __enter__(self) -> Self:
return self
def __exit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
exc_traceback: TracebackType | None,
) -> None:
return None
def _plot_fig(
fig_dir: Path,
fig_group_data: tuple[tuple[tuple[str, str], ...], list[dict[str, object]]],
row_by: list[str],
col_by: list[str],
curve_by: list[str],
*,
var_x: str,
var_y: str,
filter_by: PlotFilters,
bin_by: PlotBinners,
scale_x: str | None,
scale_y: str | None,
dry_run: bool,
fig_name: str,
error_bars: bool,
fig_height: float,
fig_dpi: int,
):
# Lazy-import matplotlib/pandas/seaborn
try:
import matplotlib.pyplot as plt
except ImportError:
plt = PlaceholderModule("matplotlib").placeholder_attr("pyplot")
try:
import pandas as pd
except ImportError:
pd = PlaceholderModule("pandas")
try:
import seaborn as sns
except ImportError:
sns = PlaceholderModule("seaborn")
fig_group, fig_data = fig_group_data
row_groups = full_groupby(
fig_data,
key=lambda item: _get_group(item, row_by),
)
num_rows = len(row_groups)
num_cols = max(
len(full_groupby(row_data, key=lambda item: _get_group(item, col_by)))
for _, row_data in row_groups
)
fig_path = _get_fig_path(fig_dir, fig_group, fig_name)
print("[BEGIN FIGURE]")
print(f"Group: {dict(fig_group)}")
print(f"Grid: {num_rows} rows x {num_cols} cols")
print(f"Output file: {fig_path}")
if dry_run:
print("[END FIGURE]")
return
# Convert string "inf", "-inf", and "nan" to their float equivalents
fig_data = _convert_inf_nan_strings(fig_data)
df = pd.DataFrame.from_records(fig_data)
if var_x not in df.columns:
raise ValueError(
f"Cannot find {var_x=!r} in parameter sweep results. "
f"Available variables: {df.columns.tolist()}"
)
if var_y not in df.columns:
raise ValueError(
f"Cannot find {var_y=!r} in parameter sweep results. "
f"Available variables: {df.columns.tolist()}"
)
for k in row_by:
if k not in df.columns:
raise ValueError(
f"Cannot find row_by={k!r} in parameter sweep results. "
f"Available variables: {df.columns.tolist()}"
)
for k in col_by:
if k not in df.columns:
raise ValueError(
f"Cannot find col_by={k!r} in parameter sweep results. "
f"Available variables: {df.columns.tolist()}"
)
for k in curve_by:
if k not in df.columns:
raise ValueError(
f"Cannot find curve_by={k!r} in parameter sweep results. "
f"Available variables: {df.columns.tolist()}"
)
df = filter_by.apply(df)
df = bin_by.apply(df)
if len(df) == 0:
print(f"No data to plot. Filters: {filter_by}")
print("[END FIGURE]")
return
# Sort by curve_by columns alphabetically for consistent legend ordering
if curve_by:
df = df.sort_values(by=curve_by)
df["row_group"] = (
pd.concat(
[k + "=" + df[k].astype(str) for k in row_by],
axis=1,
).agg("\n".join, axis=1)
if row_by
else "(All)"
)
df["col_group"] = (
pd.concat(
[k + "=" + df[k].astype(str) for k in col_by],
axis=1,
).agg("\n".join, axis=1)
if col_by
else "(All)"
)
if len(curve_by) <= 3:
hue, style, size, *_ = (*curve_by, None, None, None)
g = sns.relplot(
df,
x=var_x,
y=var_y,
hue=hue,
style=style,
size=size,
markers=True,
errorbar="sd" if error_bars else None,
kind="line",
row="row_group",
col="col_group",
height=fig_height,
)
else:
df["curve_group"] = (
pd.concat(
[k + "=" + df[k].astype(str) for k in curve_by],
axis=1,
).agg("\n".join, axis=1)
if curve_by
else "(All)"
)
g = sns.relplot(
df,
x=var_x,
y=var_y,
hue="curve_group",
markers=True,
errorbar="sd" if error_bars else None,
kind="line",
row="row_group",
col="col_group",
height=fig_height,
)
if row_by and col_by:
g.set_titles("{row_name}\n{col_name}")
elif row_by:
g.set_titles("{row_name}")
elif col_by:
g.set_titles("{col_name}")
else:
g.set_titles("")
if scale_x:
g.set(xscale=scale_x)
if scale_y:
g.set(yscale=scale_y)
g.savefig(fig_path, dpi=fig_dpi)
plt.close(g.figure)
print("[END FIGURE]")
def plot(
output_dir: Path,
fig_dir: Path,
fig_by: list[str],
row_by: list[str],
col_by: list[str],
curve_by: list[str],
*,
var_x: str,
var_y: str,
filter_by: PlotFilters,
bin_by: PlotBinners,
scale_x: str | None,
scale_y: str | None,
dry_run: bool,
fig_name: str = "FIGURE",
error_bars: bool = True,
fig_height: float = 6.4,
fig_dpi: int = 300,
):
all_data = [
run_data
for path in output_dir.rglob("**/summary.json")
for run_data in _json_load_bytes(path)
]
if not all_data:
raise ValueError(f"Did not find any parameter sweep results under {output_dir}")
fig_dir.mkdir(parents=True, exist_ok=True)
fig_groups = full_groupby(
all_data,
key=lambda item: _get_group(item, fig_by),
)
with DummyExecutor() if len(fig_groups) <= 1 else ProcessPoolExecutor() as executor:
# Resolve the iterable to ensure that the workers are run
all(
executor.map(
partial(
_plot_fig,
fig_dir,
row_by=row_by,
col_by=col_by,
curve_by=curve_by,
var_x=var_x,
var_y=var_y,
filter_by=filter_by,
bin_by=bin_by,
scale_x=scale_x,
scale_y=scale_y,
dry_run=dry_run,
fig_name=fig_name,
error_bars=error_bars,
fig_height=fig_height,
fig_dpi=fig_dpi,
),
fig_groups,
)
)
@dataclass
class SweepPlotArgs:
output_dir: Path
fig_dir: Path
fig_by: list[str]
row_by: list[str]
col_by: list[str]
curve_by: list[str]
var_x: str
var_y: str
filter_by: PlotFilters
bin_by: PlotBinners
scale_x: str | None
scale_y: str | None
dry_run: bool
fig_name: str = "FIGURE"
error_bars: bool = True
fig_height: float = 6.4
fig_dpi: int = 300
parser_name: ClassVar[str] = "plot"
parser_help: ClassVar[str] = "Plot performance curves from parameter sweep results."
@classmethod
def from_cli_args(cls, args: argparse.Namespace):
output_dir = Path(args.EXPERIMENT_DIR)
if not output_dir.exists():
raise ValueError(f"No parameter sweep results under {output_dir}")
curve_by = [] if not args.curve_by else args.curve_by.split(",")
row_by = [] if not args.row_by else args.row_by.split(",")
col_by = [] if not args.col_by else args.col_by.split(",")
fig_by = [] if not args.fig_by else args.fig_by.split(",")
return cls(
output_dir=output_dir,
fig_dir=output_dir / args.fig_dir,
fig_by=fig_by,
row_by=row_by,
col_by=col_by,
curve_by=curve_by,
var_x=args.var_x,
var_y=args.var_y,
filter_by=PlotFilters.parse_str(args.filter_by),
bin_by=PlotBinners.parse_str(args.bin_by),
scale_x=args.scale_x,
scale_y=args.scale_y,
dry_run=args.dry_run,
fig_name=args.fig_name,
error_bars=not args.no_error_bars,
fig_height=args.fig_height,
fig_dpi=args.fig_dpi,
)
@classmethod
def add_cli_args(cls, parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
parser.add_argument(
"EXPERIMENT_DIR",
type=str,
help="The directory containing the sweep results to plot.",
)
parser.add_argument(
"--fig-dir",
type=str,
default="",
help="The directory to save the figures, relative to `OUTPUT_DIR`. "
"By default, the same directory is used.",
)
parser.add_argument(
"--fig-by",
type=str,
default="",
help="A comma-separated list of variables, such that a separate figure "
"is created for each combination of these variables.",
)
parser.add_argument(
"--row-by",
type=str,
default="",
help="A comma-separated list of variables, such that a separate row "
"is created for each combination of these variables.",
)
parser.add_argument(
"--col-by",
type=str,
default="",
help="A comma-separated list of variables, such that a separate column "
"is created for each combination of these variables.",
)
parser.add_argument(
"--curve-by",
type=str,
default=None,
help="A comma-separated list of variables, such that a separate curve "
"is created for each combination of these variables.",
)
parser.add_argument(
"--var-x",
type=str,
default="total_token_throughput",
help="The variable for the x-axis.",
)
parser.add_argument(
"--var-y",
type=str,
default="median_ttft_ms",
help="The variable for the y-axis",
)
parser.add_argument(
"--filter-by",
type=str,
default="",
help="A comma-separated list of statements indicating values to filter by. "
"This is useful to remove outliers. "
"Example: `max_concurrency<1000,max_num_batched_tokens<=4096` means "
"plot only the points where `max_concurrency` is less than 1000 and "
"`max_num_batched_tokens` is no greater than 4096.",
)
parser.add_argument(
"--bin-by",
type=str,
default="",
help="A comma-separated list of statements indicating values to bin by. "
"This is useful to avoid plotting points that are too close together. "
"Example: `request_throughput%%1` means "
"use a bin size of 1 for the `request_throughput` variable.",
)
parser.add_argument(
"--scale-x",
type=str,
default=None,
help="The scale to use for the x-axis. "
"Currently only accepts string values such as 'log' and 'sqrt'. "
"See also: https://seaborn.pydata.org/generated/seaborn.objects.Plot.scale.html",
)
parser.add_argument(
"--scale-y",
type=str,
default=None,
help="The scale to use for the y-axis. "
"Currently only accepts string values such as 'log' and 'sqrt'. "
"See also: https://seaborn.pydata.org/generated/seaborn.objects.Plot.scale.html",
)
parser.add_argument(
"--fig-name",
type=str,
default="FIGURE",
help="Name prefix for the output figure file. "
"Group data is always appended when present. "
"Default: 'FIGURE'. Example: --fig-name my_performance_plot",
)
parser.add_argument(
"--no-error-bars",
action="store_true",
help="If set, disables error bars on the plot. "
"By default, error bars are shown.",
)
parser.add_argument(
"--fig-height",
type=float,
default=6.4,
help="Height of each subplot in inches. Default: 6.4",
)
parser.add_argument(
"--fig-dpi",
type=int,
default=300,
help="Resolution of the output figure in dots per inch. Default: 300",
)
parser.add_argument(
"--dry-run",
action="store_true",
help="If set, prints the information about each figure to plot, "
"then exits without drawing them.",
)
return parser
def run_main(args: SweepPlotArgs):
return plot(
output_dir=args.output_dir,
fig_dir=args.fig_dir,
fig_by=args.fig_by,
row_by=args.row_by,
col_by=args.col_by,
curve_by=args.curve_by,
var_x=args.var_x,
var_y=args.var_y,
filter_by=args.filter_by,
bin_by=args.bin_by,
scale_x=args.scale_x,
scale_y=args.scale_y,
dry_run=args.dry_run,
fig_name=args.fig_name,
error_bars=args.error_bars,
fig_height=args.fig_height,
fig_dpi=args.fig_dpi,
)
def main(args: argparse.Namespace):
run_main(SweepPlotArgs.from_cli_args(args))
if __name__ == "__main__":
parser = FlexibleArgumentParser(description=SweepPlotArgs.parser_help)
SweepPlotArgs.add_cli_args(parser)
main(parser.parse_args())
-401
View File
@@ -1,401 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
import math
from concurrent.futures import ProcessPoolExecutor
from dataclasses import dataclass
from functools import partial
from pathlib import Path
from typing import TYPE_CHECKING, ClassVar
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.utils.collection_utils import full_groupby
from vllm.utils.import_utils import PlaceholderModule
from .plot import DummyExecutor, _json_load_bytes
from .utils import sanitize_filename
if TYPE_CHECKING:
import pandas as pd
def _first_present(run_data: dict[str, object], keys: list[str]):
for key in keys:
for candidate in {key, key.replace("_", "-"), key.replace("-", "_")}:
if candidate in run_data:
return run_data[candidate]
return None
def _get_numeric(
run_data: dict[str, object],
keys: list[str],
*,
allow_zero: bool = True,
) -> float | None:
value = _first_present(run_data, keys)
if value is None:
return None
try:
numeric = float(value)
except (TypeError, ValueError) as exc:
raise ValueError(
f"Expected numeric value for one of {keys}, "
f"but found {value!r} in {run_data=}"
) from exc
if not allow_zero and numeric == 0:
return None
return numeric
def _infer_user_count(
run_data: dict[str, object],
user_count_var: str | None,
) -> float | None:
candidates = [user_count_var] if user_count_var else []
candidates.extend(["request_rate"])
user_count = _get_numeric(run_data, candidates, allow_zero=False)
if user_count is not None:
return user_count
# Fallback to the observed peak if configured value is missing.
return _get_numeric(run_data, ["max_concurrent_requests"], allow_zero=False)
def _infer_gpu_count(
run_data: dict[str, object],
gpu_count_var: str | None,
) -> float:
direct_candidates = [gpu_count_var] if gpu_count_var else []
direct_gpu_count = _get_numeric(run_data, direct_candidates, allow_zero=False)
if direct_gpu_count:
return direct_gpu_count
tp_size = _get_numeric(run_data, ["tensor_parallel_size", "tp"])
pp_size = _get_numeric(run_data, ["pipeline_parallel_size", "pp"])
dp_size = _get_numeric(run_data, ["data_parallel_size", "dp"])
world_size = 1.0
if tp_size:
world_size *= tp_size
if pp_size:
world_size *= pp_size
if dp_size:
world_size *= dp_size
return world_size
def _get_throughput(
run_data: dict[str, object],
throughput_var: str,
) -> float:
throughput = _get_numeric(run_data, [throughput_var])
if throughput is None:
raise ValueError(
f"Cannot find throughput metric {throughput_var!r} in run data. "
f"Available keys: {sorted(run_data)}"
)
return throughput
def _prepare_records(
all_data: list[dict[str, object]],
*,
user_count_var: str | None,
gpu_count_var: str | None,
) -> tuple[list[dict[str, object]], int]:
prepared = []
skipped_missing_users = 0
for record in all_data:
throughput = _get_throughput(record, "output_throughput")
user_count = _infer_user_count(record, user_count_var)
if user_count is None:
skipped_missing_users += 1
continue
gpu_count = _infer_gpu_count(record, gpu_count_var)
tokens_per_user = throughput / user_count
tokens_per_gpu = throughput / gpu_count
prepared.append(
{
**record,
"tokens_per_user": tokens_per_user,
"tokens_per_gpu": tokens_per_gpu,
"user_count_estimate": user_count,
"gpu_count": gpu_count,
}
)
return prepared, skipped_missing_users
def _pareto_frontier(
df: "pd.DataFrame",
x_col: str,
y_col: str,
*,
epsilon: float = 1e-9,
) -> "pd.DataFrame":
sorted_df = df.sort_values([x_col, y_col], ascending=[False, False])
frontier_indices = []
best_y = -math.inf
for idx, row in sorted_df.iterrows():
y_val = row[y_col]
if y_val >= best_y - epsilon:
frontier_indices.append(idx)
best_y = max(best_y, y_val)
return df.loc[frontier_indices]
def _get_fig_path(
fig_dir: Path,
fig_group: tuple[tuple[str, str], ...],
) -> Path:
parts = ["PARETO"]
if fig_group:
parts.extend(f"{k}={v}" for k, v in fig_group)
filename = sanitize_filename("-".join(parts) + ".png")
return fig_dir / filename
def _plot_fig(
fig_dir: Path,
fig_group_data: tuple[tuple[tuple[str, str], ...], list[dict[str, object]]],
label_by: list[str],
*,
dry_run: bool,
):
fig_group, fig_data = fig_group_data
fig_path = _get_fig_path(fig_dir, fig_group)
print("[BEGIN FIGURE]")
print(f"Group: {dict(fig_group)}")
print(f"Output file: {fig_path}")
if dry_run:
print("[END FIGURE]")
return
# Lazy-import matplotlib/pandas/seaborn
try:
import matplotlib.pyplot as plt
except ImportError:
plt = PlaceholderModule("matplotlib").placeholder_attr("pyplot")
try:
import pandas as pd
except ImportError:
pd = PlaceholderModule("pandas")
try:
import seaborn as sns
except ImportError:
sns = PlaceholderModule("seaborn")
df = pd.DataFrame.from_records(fig_data)
df = df.dropna(subset=["tokens_per_user", "tokens_per_gpu"])
if df.empty:
print("No data points available after filtering; skipping.")
print("[END FIGURE]")
return
frontier = _pareto_frontier(df, "tokens_per_user", "tokens_per_gpu")
frontier = frontier.sort_values("tokens_per_user")
fig, ax = plt.subplots()
sns.scatterplot(
data=df,
x="tokens_per_user",
y="tokens_per_gpu",
color="0.5",
alpha=0.6,
ax=ax,
label="All runs",
)
sns.lineplot(
data=frontier,
x="tokens_per_user",
y="tokens_per_gpu",
marker="o",
ax=ax,
label="Pareto frontier",
)
if label_by:
for _, row in frontier.iterrows():
label_parts = []
for key in label_by:
if key in row:
label_parts.append(f"{key}={row[key]}")
if label_parts:
ax.text(
row["tokens_per_user"],
row["tokens_per_gpu"],
"\n".join(label_parts),
fontsize=8,
)
ax.set_xlabel("Tokens/s/user")
ax.set_ylabel("Tokens/s/GPU")
ax.grid(True, linestyle="--", linewidth=0.5, alpha=0.6)
fig.tight_layout()
fig.savefig(fig_path)
plt.close(fig)
print(
f"Plotted {len(df)} points; Pareto frontier size: {len(frontier)}.",
)
print("[END FIGURE]")
def plot_pareto(
output_dir: Path,
user_count_var: str | None,
gpu_count_var: str | None,
label_by: list[str],
*,
dry_run: bool,
):
fig_dir = output_dir / "pareto"
raw_data = [
run_data
for path in output_dir.rglob("**/summary.json")
for run_data in _json_load_bytes(path)
]
if not raw_data:
raise ValueError(f"Did not find any parameter sweep results under {output_dir}")
fig_dir.mkdir(parents=True, exist_ok=True)
prepared_data, skipped_missing_users = _prepare_records(
raw_data,
user_count_var=user_count_var,
gpu_count_var=gpu_count_var,
)
if skipped_missing_users:
print(
f"Skipped {skipped_missing_users} runs without a user count "
"(`max_concurrency` or `max_concurrent_requests`).",
)
if not prepared_data:
raise ValueError(
"No data points with both throughput and user count available "
"to plot Pareto frontier.",
)
fig_groups = full_groupby(
prepared_data,
key=lambda item: tuple(),
)
with DummyExecutor() if len(fig_groups) <= 1 else ProcessPoolExecutor() as executor:
all(
executor.map(
partial(
_plot_fig,
fig_dir,
label_by=label_by,
dry_run=dry_run,
),
fig_groups,
)
)
@dataclass
class SweepPlotParetoArgs:
output_dir: Path
user_count_var: str | None
gpu_count_var: str | None
label_by: list[str]
dry_run: bool
parser_name: ClassVar[str] = "plot_pareto"
parser_help: ClassVar[str] = (
"Plot Pareto frontier between tokens/s/user and tokens/s/GPU "
"from parameter sweep results."
)
@classmethod
def from_cli_args(cls, args: argparse.Namespace):
output_dir = Path(args.EXPERIMENT_DIR)
if not output_dir.exists():
raise ValueError(f"No parameter sweep results under {output_dir}")
label_by = [] if not args.label_by else args.label_by.split(",")
return cls(
output_dir=output_dir,
user_count_var=args.user_count_var,
gpu_count_var=args.gpu_count_var,
label_by=label_by,
dry_run=args.dry_run,
)
@classmethod
def add_cli_args(cls, parser: FlexibleArgumentParser):
parser.add_argument(
"EXPERIMENT_DIR",
type=str,
help="The directory containing the sweep results to plot.",
)
parser.add_argument(
"--user-count-var",
type=str,
default="max_concurrency",
help="Result key that stores concurrent user count. "
"Falls back to max_concurrent_requests if missing.",
)
parser.add_argument(
"--gpu-count-var",
type=str,
default=None,
help="Result key that stores GPU count. "
"If not provided, falls back to num_gpus/gpu_count "
"or tensor_parallel_size * pipeline_parallel_size.",
)
parser.add_argument(
"--label-by",
type=str,
default="max_concurrency,gpu_count",
help="Comma-separated list of fields to annotate on Pareto frontier "
"points.",
)
parser.add_argument(
"--dry-run",
action="store_true",
help="If set, prints the figures to plot without drawing them.",
)
return parser
def run_main(args: SweepPlotParetoArgs):
return plot_pareto(
output_dir=args.output_dir,
user_count_var=args.user_count_var,
gpu_count_var=args.gpu_count_var,
label_by=args.label_by,
dry_run=args.dry_run,
)
def main(args: argparse.Namespace):
run_main(SweepPlotParetoArgs.from_cli_args(args))
if __name__ == "__main__":
parser = FlexibleArgumentParser(description=SweepPlotParetoArgs.parser_help)
SweepPlotParetoArgs.add_cli_args(parser)
main(parser.parse_args())
-649
View File
@@ -1,649 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
import contextlib
import json
import shlex
from contextlib import contextmanager
from dataclasses import dataclass
from datetime import datetime
from pathlib import Path
from typing import ClassVar
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.utils.import_utils import PlaceholderModule
from .param_sweep import ParameterSweep, ParameterSweepItem
from .server import ServerProcess
from .utils import sanitize_filename
try:
import pandas as pd
except ImportError:
pd = PlaceholderModule("pandas")
@contextlib.contextmanager
def run_server(
serve_cmd: list[str],
after_bench_cmd: list[str],
*,
show_stdout: bool,
serve_overrides: ParameterSweepItem,
dry_run: bool,
server_ready_timeout: int = 300,
):
server_cmd = serve_overrides.apply_to_cmd(serve_cmd)
print("[BEGIN SERVER]")
print(f"Server overrides: {serve_overrides}")
print(f"Server command: {server_cmd}")
if dry_run:
yield None
print("[END SERVER]")
return
with ServerProcess(server_cmd, after_bench_cmd, show_stdout=show_stdout) as server:
server.wait_until_ready(timeout=server_ready_timeout)
yield server
print("[END SERVER]")
def _update_run_data(
run_data: dict[str, object],
serve_overrides: ParameterSweepItem,
bench_overrides: ParameterSweepItem,
run_number: int,
):
run_data["run_number"] = run_number
run_data.update(serve_overrides)
run_data.update(bench_overrides)
return run_data
def run_benchmark(
server: ServerProcess | None,
bench_cmd: list[str],
*,
serve_overrides: ParameterSweepItem,
bench_overrides: ParameterSweepItem,
run_number: int,
output_path: Path,
dry_run: bool,
):
benchmark_cmd = [
*bench_overrides.apply_to_cmd(bench_cmd),
"--percentile-metrics",
"ttft,tpot,itl,e2el",
"--save-result",
"--result-dir",
str(output_path.parent),
"--result-filename",
output_path.name,
]
print("[BEGIN BENCHMARK]")
print(f"Benchmark overrides: {bench_overrides}")
print(f"Run Number: {run_number}")
print(f"Benchmark command: {benchmark_cmd}")
print(f"Output file: {output_path}")
run_data: dict[str, object]
if output_path.exists():
print("Found existing results.")
print("[SKIPPED BENCHMARK]")
with output_path.open("rb") as f:
run_data = json.load(f)
return _update_run_data(
run_data,
serve_overrides,
bench_overrides,
run_number,
)
if server is None:
if not dry_run:
raise ValueError(f"Cannot find results at {output_path}")
print("[END BENCHMARK]")
return None
output_path.parent.mkdir(parents=True, exist_ok=True)
server.run_subcommand(benchmark_cmd)
server.after_bench()
with output_path.open("rb") as f:
run_data = json.load(f)
run_data = _update_run_data(
run_data,
serve_overrides,
bench_overrides,
run_number,
)
with output_path.open("w") as f:
json.dump(run_data, f, indent=4)
print("[END BENCHMARK]")
return run_data
def _get_comb_base_path(
experiment_dir: Path,
serve_comb: ParameterSweepItem,
bench_comb: ParameterSweepItem,
*,
extra_parts: tuple[str, ...] = (),
):
parts = list[str]()
if serve_comb:
parts.extend(("SERVE-", serve_comb.name))
if bench_comb:
parts.extend(("BENCH-", bench_comb.name))
if extra_parts:
parts.extend(extra_parts)
return experiment_dir / sanitize_filename("-".join(parts))
def _get_comb_run_path(base_path: Path, run_number: int | None):
if run_number is None:
return base_path / "summary.json"
return base_path / f"run={run_number}.json"
def _comb_needs_server(
serve_comb: ParameterSweepItem,
bench_combs: ParameterSweep,
experiment_dir: Path,
):
for bench_comb in bench_combs:
base_path = _get_comb_base_path(experiment_dir, serve_comb, bench_comb)
if not _get_comb_run_path(base_path, run_number=None).exists():
return True
return False
def server_ctx(
serve_cmd: list[str],
after_bench_cmd: list[str],
*,
show_stdout: bool,
serve_comb: ParameterSweepItem,
bench_params: ParameterSweep,
experiment_dir: Path,
dry_run: bool,
server_ready_timeout: int = 300,
):
if not _comb_needs_server(serve_comb, bench_params, experiment_dir):
return contextlib.nullcontext()
return run_server(
serve_cmd,
after_bench_cmd,
show_stdout=show_stdout,
serve_overrides=serve_comb,
dry_run=dry_run,
server_ready_timeout=server_ready_timeout,
)
def _comb_is_valid(
serve_comb: ParameterSweepItem,
bench_comb: ParameterSweepItem,
link_vars: list[tuple[str, str]],
) -> bool:
return all(
serve_key in serve_comb
and bench_key in bench_comb
and serve_comb[serve_key] == bench_comb[bench_key]
for serve_key, bench_key in link_vars
)
def run_comb(
server: ServerProcess | None,
bench_cmd: list[str],
*,
serve_comb: ParameterSweepItem,
bench_comb: ParameterSweepItem,
link_vars: list[tuple[str, str]],
base_path: Path,
num_runs: int,
dry_run: bool,
warmup_num_prompts: int = 0,
continue_on_error: bool = False,
):
if not _comb_is_valid(serve_comb, bench_comb, link_vars):
return None
comb_data = list[dict[str, object]]()
def run_benchmark_with_error_handling(
*,
bench_overrides: ParameterSweepItem,
run_number: int,
output_path: Path,
):
try:
return run_benchmark(
server,
bench_cmd,
serve_overrides=serve_comb,
bench_overrides=bench_overrides,
run_number=run_number,
output_path=output_path,
dry_run=dry_run,
)
except Exception as exc:
if not continue_on_error:
raise
failure_path = output_path.with_name(output_path.stem + ".failure.json")
failure_path.parent.mkdir(parents=True, exist_ok=True)
failure_path.write_text(
json.dumps(
{
"run_number": run_number,
"error_type": type(exc).__name__,
"error": str(exc),
"serve_overrides": dict(serve_comb),
"bench_overrides": dict(bench_overrides),
},
indent=2,
)
+ "\n",
encoding="utf-8",
)
print(f"[FAILED BENCHMARK] run={run_number}: {type(exc).__name__}: {exc}")
print(f"Failure details: {failure_path}")
if server is not None:
with contextlib.suppress(Exception):
server.after_bench()
return None
if warmup_num_prompts > 0:
warmup_overrides = bench_comb | {"num_prompts": warmup_num_prompts}
run_benchmark_with_error_handling(
bench_overrides=warmup_overrides,
run_number=-1,
output_path=base_path / "warmup.json",
)
for run_number in range(num_runs):
run_data = run_benchmark_with_error_handling(
bench_overrides=bench_comb,
run_number=run_number,
output_path=_get_comb_run_path(base_path, run_number),
)
if run_data is not None:
comb_data.append(run_data)
if dry_run:
return None
if not comb_data:
print("[NO SUCCESSFUL MEASURED RUNS] leaving combination retryable")
return []
with _get_comb_run_path(base_path, run_number=None).open("w") as f:
json.dump(comb_data, f, indent=4)
return comb_data
def run_combs(
serve_cmd: list[str],
bench_cmd: list[str],
after_bench_cmd: list[str],
*,
show_stdout: bool,
server_ready_timeout: int,
serve_params: ParameterSweep,
bench_params: ParameterSweep,
link_vars: list[tuple[str, str]],
experiment_dir: Path,
num_runs: int,
warmup_num_prompts: int,
dry_run: bool,
continue_on_error: bool = False,
):
all_data = list[dict[str, object]]()
for serve_comb in serve_params:
try:
with server_ctx(
serve_cmd,
after_bench_cmd,
show_stdout=show_stdout,
serve_comb=serve_comb,
bench_params=bench_params,
experiment_dir=experiment_dir,
dry_run=dry_run,
server_ready_timeout=server_ready_timeout,
) as server:
for bench_comb in bench_params:
base_path = _get_comb_base_path(
experiment_dir, serve_comb, bench_comb
)
comb_data = run_comb(
server,
bench_cmd,
serve_comb=serve_comb,
bench_comb=bench_comb,
link_vars=link_vars,
base_path=base_path,
num_runs=num_runs,
warmup_num_prompts=warmup_num_prompts,
dry_run=dry_run,
continue_on_error=continue_on_error,
)
if comb_data is not None:
all_data.extend(comb_data)
except Exception as exc:
if not continue_on_error:
raise
failure_name = sanitize_filename(
f"SERVE-{serve_comb.name or 'default'}-failure.json"
)
failure_path = experiment_dir / failure_name
failure_path.parent.mkdir(parents=True, exist_ok=True)
failure_path.write_text(
json.dumps(
{
"error_type": type(exc).__name__,
"error": str(exc),
"serve_overrides": dict(serve_comb),
},
indent=2,
)
+ "\n",
encoding="utf-8",
)
print(
"[FAILED SERVER COMBINATION] "
f"{serve_comb.name or '(default)'}: "
f"{type(exc).__name__}: {exc}"
)
print(f"Failure details: {failure_path}")
if dry_run:
return None
combined_df = pd.DataFrame.from_records(all_data)
combined_df.to_csv(experiment_dir / "summary.csv")
return combined_df
@dataclass
class SweepServeArgs:
serve_cmd: list[str]
bench_cmd: list[str]
after_bench_cmd: list[str]
show_stdout: bool
server_ready_timeout: int
serve_params: ParameterSweep
bench_params: ParameterSweep
link_vars: list[tuple[str, str]]
output_dir: Path
experiment_name: str
num_runs: int
warmup_num_prompts: int
continue_on_error: bool
dry_run: bool
resume: bool
parser_name: ClassVar[str] = "serve"
parser_help: ClassVar[str] = "Run vLLM server benchmark under multiple settings."
@classmethod
def from_cli_args(cls, args: argparse.Namespace):
serve_cmd = shlex.split(args.serve_cmd)
bench_cmd = shlex.split(args.bench_cmd)
after_bench_cmd = (
[] if args.after_bench_cmd is None else shlex.split(args.after_bench_cmd)
)
if args.serve_params:
serve_params = ParameterSweep.read_json(args.serve_params)
else:
# i.e.: run serve_cmd without any modification
serve_params = ParameterSweep.from_records([{}])
if args.bench_params:
bench_params = ParameterSweep.read_json(args.bench_params)
else:
# i.e.: run bench_cmd without any modification
bench_params = ParameterSweep.from_records([{}])
link_vars = cls.parse_link_vars(args.link_vars)
if args.experiment_name:
experiment_name = args.experiment_name
else:
experiment_name = datetime.now().strftime("%Y%m%d_%H%M%S")
num_runs = args.num_runs
if num_runs < 1:
raise ValueError("`num_runs` should be at least 1.")
warmup_num_prompts = args.warmup_num_prompts
if warmup_num_prompts < 0:
raise ValueError("`warmup_num_prompts` should be at least 0.")
return cls(
serve_cmd=serve_cmd,
bench_cmd=bench_cmd,
after_bench_cmd=after_bench_cmd,
show_stdout=args.show_stdout,
serve_params=serve_params,
bench_params=bench_params,
link_vars=link_vars,
output_dir=Path(args.output_dir),
experiment_name=experiment_name,
num_runs=num_runs,
warmup_num_prompts=warmup_num_prompts,
continue_on_error=args.continue_on_error,
dry_run=args.dry_run,
resume=args.resume,
server_ready_timeout=args.server_ready_timeout,
)
@classmethod
def add_cli_args(cls, parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
parser.add_argument(
"--serve-cmd",
type=str,
required=True,
help="The command used to run the server: `vllm serve ...`",
)
parser.add_argument(
"--bench-cmd",
type=str,
required=True,
help="The command used to run the benchmark: `vllm bench serve ...`",
)
parser.add_argument(
"--after-bench-cmd",
type=str,
default=None,
help="After a benchmark run is complete, invoke this command instead of "
"the default `ServerWrapper.clear_cache()`.",
)
parser.add_argument(
"--show-stdout",
action="store_true",
help="If set, logs the standard output of subcommands. "
"Useful for debugging but can be quite spammy.",
)
parser.add_argument(
"--server-ready-timeout",
type=int,
default=300,
help="Timeout in seconds to wait for the server to become ready.",
)
parser.add_argument(
"--serve-params",
type=str,
default=None,
help="Path to JSON file containing parameter combinations "
"for the `vllm serve` command. Can be either a list of dicts or a dict "
"where keys are benchmark names. "
"If both `serve_params` and `bench_params` are given, "
"this script will iterate over their Cartesian product.",
)
parser.add_argument(
"--link-vars",
type=str,
default="",
help=(
"Comma-separated list of linked variables between serve and bench, "
"e.g. max_num_seqs=max_concurrency,max_model_len=random_input_len"
),
)
parser.add_argument(
"--bench-params",
type=str,
default=None,
help="Path to JSON file containing parameter combinations "
"for the `vllm bench serve` command. Can be either a list of dicts or "
"a dict where keys are benchmark names. "
"If both `serve_params` and `bench_params` are given, "
"this script will iterate over their Cartesian product.",
)
parser.add_argument(
"-o",
"--output-dir",
type=str,
default="results",
help="The main directory to which results are written.",
)
parser.add_argument(
"-e",
"--experiment-name",
type=str,
default=None,
help="The name of this experiment (defaults to current timestamp). "
"Results will be stored under `output_dir/experiment_name`.",
)
parser.add_argument(
"--num-runs",
type=int,
default=3,
help="Number of runs per parameter combination.",
)
parser.add_argument(
"--warmup-num-prompts",
type=int,
default=0,
help=(
"Number of prompts in one unmeasured warmup run before each "
"parameter combination. The warmup is saved as warmup.json but "
"excluded from summary statistics."
),
)
parser.add_argument(
"--continue-on-error",
action="store_true",
help=(
"Record and skip a failed warmup or benchmark run instead of "
"terminating the entire sweep."
),
)
parser.add_argument(
"--dry-run",
action="store_true",
help="If set, prints the commands to run, "
"then exits without executing them.",
)
parser.add_argument(
"--resume",
action="store_true",
help="Resume a previous execution of this script, i.e., only run "
"parameter combinations for which there are still no output files "
"under `output_dir/experiment_name`.",
)
return parser
@staticmethod
def parse_link_vars(s: str) -> list[tuple[str, str]]:
if not s:
return []
pairs = []
for item in s.split(","):
a, b = item.split("=")
pairs.append((a.strip(), b.strip()))
return pairs
def resolve_experiment_dir(self) -> Path:
experiment_dir = self.output_dir / self.experiment_name
if self.resume:
if not experiment_dir.exists():
raise ValueError(f"Cannot resume from non-existent {experiment_dir=}")
else:
if experiment_dir.exists():
raise ValueError(f"Cannot overwrite existing {experiment_dir=}")
return experiment_dir
@contextmanager
def run_ctx(self, experiment_dir: Path):
if self.dry_run:
yield
print(f"Experiment will be saved at: {experiment_dir}")
return
try:
yield
print(f"Experiment has been saved at: {experiment_dir}")
except BaseException as exc:
raise RuntimeError(
"The script was terminated early. Use `--resume` "
"to continue the script from its last checkpoint."
) from exc
def run_main(args: SweepServeArgs):
experiment_dir = args.resolve_experiment_dir()
with args.run_ctx(experiment_dir):
return run_combs(
serve_cmd=args.serve_cmd,
bench_cmd=args.bench_cmd,
link_vars=args.link_vars,
after_bench_cmd=args.after_bench_cmd,
show_stdout=args.show_stdout,
server_ready_timeout=args.server_ready_timeout,
serve_params=args.serve_params,
bench_params=args.bench_params,
experiment_dir=experiment_dir,
num_runs=args.num_runs,
warmup_num_prompts=args.warmup_num_prompts,
dry_run=args.dry_run,
continue_on_error=args.continue_on_error,
)
def main(args: argparse.Namespace):
run_main(SweepServeArgs.from_cli_args(args))
if __name__ == "__main__":
parser = FlexibleArgumentParser(description=SweepServeArgs.parser_help)
SweepServeArgs.add_cli_args(parser)
main(parser.parse_args())
-350
View File
@@ -1,350 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
import math
from dataclasses import dataclass, fields
from pathlib import Path
from typing import ClassVar, Literal, assert_never, get_args
import numpy as np
from vllm.benchmarks.datasets import DEFAULT_NUM_PROMPTS
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.utils.import_utils import PlaceholderModule
from .param_sweep import ParameterSweep, ParameterSweepItem
from .serve import (
SweepServeArgs,
_get_comb_base_path,
run_comb,
server_ctx,
)
from .server import ServerProcess
try:
import pandas as pd
except ImportError:
pd = PlaceholderModule("pandas")
WorkloadVariable = Literal["request_rate", "max_concurrency"]
def _estimate_workload_value(
run_data: dict[str, object],
workload_var: WorkloadVariable,
):
request_throughput = float(run_data["request_throughput"]) # type: ignore
if workload_var == "request_rate":
return request_throughput
if workload_var == "max_concurrency":
mean_latency_ms = float(run_data["mean_e2el_ms"]) # type: ignore
return request_throughput * mean_latency_ms / 1000
assert_never(workload_var)
def _estimate_workload_avg(
runs: list[dict[str, object]],
workload_var: WorkloadVariable,
):
total = sum(_estimate_workload_value(run, workload_var) for run in runs)
return total / len(runs)
def run_comb_workload(
server: ServerProcess | None,
bench_cmd: list[str],
*,
serve_comb: ParameterSweepItem,
bench_comb: ParameterSweepItem,
link_vars: list[tuple[str, str]],
experiment_dir: Path,
num_runs: int,
dry_run: bool,
warmup_num_prompts: int,
continue_on_error: bool,
workload_var: WorkloadVariable,
workload_value: int,
) -> list[dict[str, object]] | None:
bench_comb_workload = bench_comb | {workload_var: workload_value}
return run_comb(
server,
bench_cmd,
serve_comb=serve_comb,
bench_comb=bench_comb_workload,
link_vars=link_vars,
base_path=_get_comb_base_path(
experiment_dir,
serve_comb,
bench_comb,
extra_parts=("WL-", f"{workload_var}={workload_value}"),
),
num_runs=num_runs,
dry_run=dry_run,
warmup_num_prompts=warmup_num_prompts,
continue_on_error=continue_on_error,
)
def explore_comb_workloads(
server: ServerProcess | None,
bench_cmd: list[str],
*,
serve_comb: ParameterSweepItem,
bench_comb: ParameterSweepItem,
link_vars: list[tuple[str, str]],
workload_var: WorkloadVariable,
workload_iters: int,
experiment_dir: Path,
num_runs: int,
dry_run: bool,
warmup_num_prompts: int,
continue_on_error: bool,
):
print("[WL START]")
print(f"Serve parameters: {serve_comb.as_text() or '(None)'}")
print(f"Bench parameters: {bench_comb.as_text() or '(None)'}")
print(f"Number of workload iterations: {workload_iters}")
if workload_iters < 2:
raise ValueError("`workload_iters` should be at least 2")
dataset_size = DEFAULT_NUM_PROMPTS
if "num_prompts" in bench_comb:
dataset_size = int(bench_comb["num_prompts"]) # type: ignore
else:
for i, arg in enumerate(bench_cmd):
if arg == "--num-prompts" and i + 1 < len(bench_cmd):
dataset_size = int(bench_cmd[i + 1])
break
elif arg.startswith("--num-prompts="):
dataset_size = int(arg.split("=", 1)[1])
break
print(f"Dataset size: {dataset_size}")
serial_workload_data = run_comb_workload(
server,
bench_cmd,
serve_comb=serve_comb,
bench_comb=bench_comb | {"max_concurrency": 1},
link_vars=link_vars,
experiment_dir=experiment_dir,
num_runs=num_runs,
dry_run=dry_run,
warmup_num_prompts=warmup_num_prompts,
continue_on_error=continue_on_error,
workload_var=workload_var,
workload_value=1,
)
batch_workload_data = run_comb_workload(
server,
bench_cmd,
serve_comb=serve_comb,
bench_comb=bench_comb | {"max_concurrency": dataset_size},
link_vars=link_vars,
experiment_dir=experiment_dir,
num_runs=num_runs,
dry_run=dry_run,
warmup_num_prompts=warmup_num_prompts,
continue_on_error=continue_on_error,
workload_var=workload_var,
workload_value=dataset_size,
)
if not serial_workload_data or not batch_workload_data:
if dry_run:
print("Omitting intermediate Workload iterations.")
else:
print(
"Unable to establish both Workload Explorer endpoints; "
"skipping intermediate workload levels."
)
print("[WL END]")
return []
serial_workload_value = math.ceil(
_estimate_workload_avg(serial_workload_data, workload_var)
)
print(f"Serial inference: {workload_var}={serial_workload_value}")
batch_workload_value = math.floor(
_estimate_workload_avg(batch_workload_data, workload_var)
)
print(f"Batch inference: {workload_var}={batch_workload_value}")
# Avoid duplicated runs for intermediate values if the range between
# `serial_workload_value` and `batch_workload_value` is small
inter_workload_values = np.linspace(
serial_workload_value, batch_workload_value, workload_iters
)[1:-1]
inter_workload_values = sorted(set(map(round, inter_workload_values)))
inter_workloads_data: list[dict[str, object]] = []
for inter_workload_value in inter_workload_values:
print(f"Exploring: {workload_var}={inter_workload_value}")
inter_workload_data = run_comb_workload(
server,
bench_cmd,
serve_comb=serve_comb,
bench_comb=bench_comb,
link_vars=link_vars,
experiment_dir=experiment_dir,
num_runs=num_runs,
dry_run=dry_run,
warmup_num_prompts=warmup_num_prompts,
continue_on_error=continue_on_error,
workload_var=workload_var,
workload_value=inter_workload_value,
)
if inter_workload_data is not None:
inter_workloads_data.extend(inter_workload_data)
print("[WL END]")
return serial_workload_data + inter_workloads_data + batch_workload_data
def explore_combs_workloads(
serve_cmd: list[str],
bench_cmd: list[str],
after_bench_cmd: list[str],
*,
show_stdout: bool,
server_ready_timeout: int,
serve_params: ParameterSweep,
bench_params: ParameterSweep,
link_vars: list[tuple[str, str]],
workload_var: WorkloadVariable,
workload_iters: int,
experiment_dir: Path,
num_runs: int,
dry_run: bool,
warmup_num_prompts: int,
continue_on_error: bool,
):
if any(bench_comb.has_param(workload_var) for bench_comb in bench_params):
raise ValueError(
f"You should not override `{workload_var}` in `bench_params` "
"since it is supposed to be explored automatically."
)
all_data = list[dict[str, object]]()
for serve_comb in serve_params:
with server_ctx(
serve_cmd,
after_bench_cmd,
show_stdout=show_stdout,
server_ready_timeout=server_ready_timeout,
serve_comb=serve_comb,
bench_params=bench_params,
experiment_dir=experiment_dir,
dry_run=dry_run,
) as server:
for bench_comb in bench_params:
comb_data = explore_comb_workloads(
server,
bench_cmd,
serve_comb=serve_comb,
bench_comb=bench_comb,
link_vars=link_vars,
workload_var=workload_var,
workload_iters=workload_iters,
experiment_dir=experiment_dir,
num_runs=num_runs,
dry_run=dry_run,
warmup_num_prompts=warmup_num_prompts,
continue_on_error=continue_on_error,
)
if comb_data is not None:
all_data.extend(comb_data)
if dry_run:
return None
combined_df = pd.DataFrame.from_records(all_data)
combined_df.to_csv(experiment_dir / "summary.csv")
return combined_df
@dataclass
class SweepServeWorkloadArgs(SweepServeArgs):
workload_var: WorkloadVariable
workload_iters: int
parser_name: ClassVar[str] = "serve_workload"
parser_help: ClassVar[str] = (
"Explore the latency-throughput tradeoff for different workload levels."
)
@classmethod
def from_cli_args(cls, args: argparse.Namespace):
# NOTE: Don't use super() as `from_cli_args` calls `cls()`
base_args = SweepServeArgs.from_cli_args(args)
return cls(
**{f.name: getattr(base_args, f.name) for f in fields(base_args)},
workload_var=args.workload_var,
workload_iters=args.workload_iters,
)
@classmethod
def add_cli_args(cls, parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
parser = super().add_cli_args(parser)
workload_group = parser.add_argument_group("workload options")
workload_group.add_argument(
"--workload-var",
type=str,
choices=get_args(WorkloadVariable),
default="request_rate",
help="The variable to adjust in each iteration.",
)
workload_group.add_argument(
"--workload-iters",
type=int,
default=10,
help="Number of workload levels to explore. "
"This includes the first two iterations used to interpolate the value of "
"`workload_var` for remaining iterations.",
)
return parser
def run_main(args: SweepServeWorkloadArgs):
experiment_dir = args.resolve_experiment_dir()
with args.run_ctx(experiment_dir):
return explore_combs_workloads(
serve_cmd=args.serve_cmd,
bench_cmd=args.bench_cmd,
after_bench_cmd=args.after_bench_cmd,
show_stdout=args.show_stdout,
server_ready_timeout=args.server_ready_timeout,
serve_params=args.serve_params,
bench_params=args.bench_params,
link_vars=args.link_vars,
workload_var=args.workload_var,
workload_iters=args.workload_iters,
experiment_dir=experiment_dir,
num_runs=args.num_runs,
dry_run=args.dry_run,
warmup_num_prompts=args.warmup_num_prompts,
continue_on_error=args.continue_on_error,
)
def main(args: argparse.Namespace):
run_main(SweepServeWorkloadArgs.from_cli_args(args))
if __name__ == "__main__":
parser = FlexibleArgumentParser(description=SweepServeWorkloadArgs.parser_help)
SweepServeWorkloadArgs.add_cli_args(parser)
main(parser.parse_args())
-142
View File
@@ -1,142 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import contextlib
import os
import signal
import subprocess
import time
from types import TracebackType
from typing import Self
import requests
class ServerProcess:
VLLM_RESET_CACHE_ENDPOINTS = [
"/reset_prefix_cache",
"/reset_mm_cache",
"/reset_encoder_cache",
]
def __init__(
self,
server_cmd: list[str],
after_bench_cmd: list[str],
*,
show_stdout: bool,
) -> None:
super().__init__()
self.server_cmd = server_cmd
self.after_bench_cmd = after_bench_cmd
self.show_stdout = show_stdout
def __enter__(self) -> Self:
self.start()
return self
def __exit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
exc_traceback: TracebackType | None,
) -> None:
self.stop()
def start(self):
# Create new process for clean termination
self._server_process = subprocess.Popen(
self.server_cmd,
start_new_session=True,
stdout=None if self.show_stdout else subprocess.DEVNULL,
# Need `VLLM_SERVER_DEV_MODE=1` for `_reset_caches`
env=os.environ | {"VLLM_SERVER_DEV_MODE": "1"},
)
def stop(self):
server_process = self._server_process
if server_process.poll() is None:
# In case only some processes have been terminated
with contextlib.suppress(ProcessLookupError):
# We need to kill both API Server and Engine processes
os.killpg(os.getpgid(server_process.pid), signal.SIGKILL)
def run_subcommand(self, cmd: list[str]):
return subprocess.run(
cmd,
stdout=None if self.show_stdout else subprocess.DEVNULL,
check=True,
)
def after_bench(self) -> None:
if not self.after_bench_cmd:
self.reset_caches()
return
self.run_subcommand(self.after_bench_cmd)
def _get_vllm_server_address(self) -> str:
server_cmd = self.server_cmd
for host_key in ("--host",):
if host_key in server_cmd:
host = server_cmd[server_cmd.index(host_key) + 1]
break
else:
host = "localhost"
for port_key in ("-p", "--port"):
if port_key in server_cmd:
port = int(server_cmd[server_cmd.index(port_key) + 1])
break
else:
port = 8000 # The default value in vllm serve
return f"http://{host}:{port}"
def is_server_ready(self) -> bool:
server_address = self._get_vllm_server_address()
try:
response = requests.get(f"{server_address}/health")
return response.status_code == 200
except requests.RequestException:
return False
def wait_until_ready(self, timeout: int) -> None:
start_time = time.monotonic()
while not self.is_server_ready():
# Check if server process has crashed
if self._server_process.poll() is not None:
returncode = self._server_process.returncode
raise RuntimeError(
f"Server process crashed with return code {returncode}"
)
if time.monotonic() - start_time > timeout:
raise TimeoutError(
f"Server failed to become ready within {timeout} seconds."
)
time.sleep(1)
def reset_caches(self) -> None:
server_cmd = self.server_cmd
# Use `.endswith()` to match `/bin/...`
if server_cmd[0].endswith("vllm"):
server_address = self._get_vllm_server_address()
print(f"Resetting caches at {server_address}")
for endpoint in self.VLLM_RESET_CACHE_ENDPOINTS:
res = requests.post(server_address + endpoint)
res.raise_for_status()
elif server_cmd[0].endswith("infinity_emb"):
if "--vector-disk-cache" in server_cmd:
raise NotImplementedError(
"Infinity server uses caching but does not expose a method "
"to reset the cache"
)
else:
raise NotImplementedError(
f"No implementation of `reset_caches` for `{server_cmd[0]}` server. "
"Please specify a custom command via `--after-bench-cmd`."
)
-443
View File
@@ -1,443 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
import json
import shlex
import subprocess
from contextlib import contextmanager
from dataclasses import dataclass
from datetime import datetime
from functools import lru_cache
from pathlib import Path
from typing import ClassVar
from vllm.benchmarks.startup import add_cli_args as add_startup_cli_args
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.utils.import_utils import PlaceholderModule
from .param_sweep import ParameterSweep, ParameterSweepItem
from .utils import sanitize_filename
try:
import pandas as pd
except ImportError:
pd = PlaceholderModule("pandas")
@lru_cache(maxsize=1)
def _get_supported_startup_keys() -> set[str]:
parser = FlexibleArgumentParser(add_help=False)
add_startup_cli_args(parser)
supported: set[str] = {"config"}
for action in parser._actions:
if action.dest and action.dest is not argparse.SUPPRESS:
supported.add(action.dest)
for option in action.option_strings:
if option.startswith("--"):
supported.add(option.lstrip("-").replace("-", "_"))
return supported
def _is_supported_param(param_key: str, supported: set[str]) -> bool:
if param_key == "_benchmark_name":
return True
prefix = param_key.split(".", 1)[0]
normalized = prefix.replace("-", "_")
return normalized in supported
def _filter_params(
params: ParameterSweep, *, supported: set[str], strict: bool
) -> ParameterSweep:
filtered = []
for item in params:
kept: dict[str, object] = {}
dropped: list[str] = []
for key, value in item.items():
if _is_supported_param(key, supported):
kept[key] = value
else:
dropped.append(key)
if dropped:
label = item.get("_benchmark_name") or item.as_text()
message = (
"Ignoring unsupported startup params"
f"{' for ' + str(label) if label else ''}: "
f"{', '.join(sorted(dropped))}"
)
if strict:
raise ValueError(message)
print(message)
filtered.append(ParameterSweepItem.from_record(kept))
return ParameterSweep(filtered)
def _update_run_data(
run_data: dict[str, object],
serve_overrides: ParameterSweepItem,
startup_overrides: ParameterSweepItem,
run_number: int,
) -> dict[str, object]:
run_data["run_number"] = run_number
run_data.update(serve_overrides)
run_data.update(startup_overrides)
return run_data
def _strip_arg(cmd: list[str], keys: tuple[str, ...]) -> list[str]:
stripped: list[str] = []
skip_next = False
for arg in cmd:
if skip_next:
skip_next = False
continue
if arg in keys:
skip_next = True
continue
if any(arg.startswith(f"{key}=") for key in keys):
continue
stripped.append(arg)
return stripped
def _apply_output_json(cmd: list[str], output_path: Path) -> list[str]:
keys = ("--output-json", "--output_json")
cmd = _strip_arg(cmd, keys)
return [*cmd, keys[0], str(output_path)]
def _get_comb_base_path(
experiment_dir: Path,
serve_comb: ParameterSweepItem,
startup_comb: ParameterSweepItem,
) -> Path:
parts = list[str]()
if serve_comb:
parts.extend(("SERVE-", serve_comb.name))
if startup_comb:
parts.extend(("STARTUP-", startup_comb.name))
return experiment_dir / sanitize_filename("-".join(parts))
def _get_comb_run_path(base_path: Path, run_number: int | None) -> Path:
if run_number is None:
return base_path / "summary.json"
return base_path / f"run={run_number}.json"
def run_benchmark(
startup_cmd: list[str],
*,
serve_overrides: ParameterSweepItem,
startup_overrides: ParameterSweepItem,
run_number: int,
output_path: Path,
show_stdout: bool,
dry_run: bool,
) -> dict[str, object] | None:
cmd = serve_overrides.apply_to_cmd(startup_cmd)
cmd = startup_overrides.apply_to_cmd(cmd)
cmd = _apply_output_json(cmd, output_path)
print("[BEGIN BENCHMARK]")
print(f"Serve overrides: {serve_overrides}")
print(f"Startup overrides: {startup_overrides}")
print(f"Run Number: {run_number}")
print(f"Benchmark command: {cmd}")
print(f"Output file: {output_path}")
if output_path.exists():
print("Found existing results.")
print("[SKIPPED BENCHMARK]")
with output_path.open("r", encoding="utf-8") as f:
run_data = json.load(f)
return _update_run_data(
run_data, serve_overrides, startup_overrides, run_number
)
if dry_run:
print("[END BENCHMARK]")
return None
output_path.parent.mkdir(parents=True, exist_ok=True)
subprocess.run(
cmd,
stdout=None if show_stdout else subprocess.DEVNULL,
check=True,
)
with output_path.open("r", encoding="utf-8") as f:
run_data = json.load(f)
run_data = _update_run_data(
run_data, serve_overrides, startup_overrides, run_number
)
with output_path.open("w", encoding="utf-8") as f:
json.dump(run_data, f, indent=4)
print("[END BENCHMARK]")
return run_data
def run_comb(
startup_cmd: list[str],
*,
serve_comb: ParameterSweepItem,
startup_comb: ParameterSweepItem,
base_path: Path,
num_runs: int,
show_stdout: bool,
dry_run: bool,
) -> list[dict[str, object]] | None:
comb_data = list[dict[str, object]]()
for run_number in range(num_runs):
run_data = run_benchmark(
startup_cmd,
serve_overrides=serve_comb,
startup_overrides=startup_comb,
run_number=run_number,
output_path=_get_comb_run_path(base_path, run_number),
show_stdout=show_stdout,
dry_run=dry_run,
)
if run_data is not None:
comb_data.append(run_data)
if dry_run:
return None
with _get_comb_run_path(base_path, run_number=None).open(
"w", encoding="utf-8"
) as f:
json.dump(comb_data, f, indent=4)
return comb_data
def run_combs(
startup_cmd: list[str],
*,
serve_params: ParameterSweep,
startup_params: ParameterSweep,
experiment_dir: Path,
num_runs: int,
show_stdout: bool,
dry_run: bool,
) -> "pd.DataFrame | None":
all_data = list[dict[str, object]]()
for serve_comb in serve_params:
for startup_comb in startup_params:
base_path = _get_comb_base_path(experiment_dir, serve_comb, startup_comb)
comb_data = run_comb(
startup_cmd,
serve_comb=serve_comb,
startup_comb=startup_comb,
base_path=base_path,
num_runs=num_runs,
show_stdout=show_stdout,
dry_run=dry_run,
)
if comb_data is not None:
all_data.extend(comb_data)
if dry_run:
return None
combined_df = pd.DataFrame.from_records(all_data)
combined_df.to_csv(experiment_dir / "summary.csv")
return combined_df
@dataclass
class SweepStartupArgs:
startup_cmd: list[str]
serve_params: ParameterSweep
startup_params: ParameterSweep
output_dir: Path
experiment_name: str
num_runs: int
show_stdout: bool
dry_run: bool
resume: bool
parser_name: ClassVar[str] = "startup"
parser_help: ClassVar[str] = (
"Benchmark vLLM startup time over parameter combinations."
)
@classmethod
def from_cli_args(cls, args: argparse.Namespace):
startup_cmd = shlex.split(args.startup_cmd)
if args.serve_params:
serve_params = ParameterSweep.read_json(args.serve_params)
else:
serve_params = ParameterSweep.from_records([{}])
if args.startup_params:
startup_params = ParameterSweep.read_json(args.startup_params)
else:
startup_params = ParameterSweep.from_records([{}])
supported = _get_supported_startup_keys()
strict_params = args.strict_params
serve_params = _filter_params(
serve_params, supported=supported, strict=strict_params
)
startup_params = _filter_params(
startup_params, supported=supported, strict=strict_params
)
if args.experiment_name:
experiment_name = args.experiment_name
else:
experiment_name = datetime.now().strftime("%Y%m%d_%H%M%S")
if args.num_runs < 1:
raise ValueError("`num_runs` should be at least 1.")
return cls(
startup_cmd=startup_cmd,
serve_params=serve_params,
startup_params=startup_params,
output_dir=Path(args.output_dir),
experiment_name=experiment_name,
num_runs=args.num_runs,
show_stdout=args.show_stdout,
dry_run=args.dry_run,
resume=args.resume,
)
@classmethod
def add_cli_args(cls, parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
parser.add_argument(
"--startup-cmd",
type=str,
default="vllm bench startup",
help="The command used to run the startup benchmark.",
)
parser.add_argument(
"--serve-params",
type=str,
default=None,
help="Path to JSON file containing parameter combinations "
"for the `vllm serve` command. Only parameters supported by "
"`vllm bench startup` will be applied.",
)
parser.add_argument(
"--startup-params",
type=str,
default=None,
help="Path to JSON file containing parameter combinations "
"for the `vllm bench startup` command.",
)
parser.add_argument(
"--strict-params",
action="store_true",
help="If set, unknown parameters in sweep files raise an error "
"instead of being ignored.",
)
parser.add_argument(
"-o",
"--output-dir",
type=str,
default="results",
help="The main directory to which results are written.",
)
parser.add_argument(
"-e",
"--experiment-name",
type=str,
default=None,
help="The name of this experiment (defaults to current timestamp). "
"Results will be stored under `output_dir/experiment_name`.",
)
parser.add_argument(
"--num-runs",
type=int,
default=1,
help="Number of runs per parameter combination.",
)
parser.add_argument(
"--show-stdout",
action="store_true",
help="If set, logs the standard output of subcommands.",
)
parser.add_argument(
"--dry-run",
action="store_true",
help="If set, prints the commands to run, "
"then exits without executing them.",
)
parser.add_argument(
"--resume",
action="store_true",
help="Resume a previous execution of this script, i.e., only run "
"parameter combinations for which there are still no output files "
"under `output_dir/experiment_name`.",
)
return parser
def resolve_experiment_dir(self) -> Path:
experiment_dir = self.output_dir / self.experiment_name
if self.resume:
if not experiment_dir.exists():
raise ValueError(f"Cannot resume from non-existent {experiment_dir=}")
else:
if experiment_dir.exists():
raise ValueError(f"Cannot overwrite existing {experiment_dir=}")
return experiment_dir
@contextmanager
def run_ctx(self, experiment_dir: Path):
if self.dry_run:
yield
print(f"Experiment will be saved at: {experiment_dir}")
return
try:
yield
print(f"Experiment has been saved at: {experiment_dir}")
except BaseException as exc:
raise RuntimeError(
"The script was terminated early. Use `--resume` "
"to continue the script from its last checkpoint."
) from exc
def run_main(args: SweepStartupArgs):
experiment_dir = args.resolve_experiment_dir()
with args.run_ctx(experiment_dir):
return run_combs(
startup_cmd=args.startup_cmd,
serve_params=args.serve_params,
startup_params=args.startup_params,
experiment_dir=experiment_dir,
num_runs=args.num_runs,
show_stdout=args.show_stdout,
dry_run=args.dry_run,
)
def main(args: argparse.Namespace):
run_main(SweepStartupArgs.from_cli_args(args))
if __name__ == "__main__":
parser = FlexibleArgumentParser(description=SweepStartupArgs.parser_help)
SweepStartupArgs.add_cli_args(parser)
main(parser.parse_args())
-4
View File
@@ -1,4 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
def sanitize_filename(filename: str) -> str:
return filename.replace("/", "_").replace("..", "__").strip("'").strip('"')
File diff suppressed because it is too large Load Diff
@@ -1,629 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
import torch
from torch._higher_order_ops.auto_functionalize import auto_functionalized
from vllm.config import VllmConfig, get_layers_from_vllm_config
from vllm.logger import init_logger
from vllm.model_executor.layers.attention.mla_attention import MLAAttention
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kFp8Dynamic64Sym,
kFp8Dynamic128Sym,
kFp8StaticTensorSym,
kNvfp4Dynamic,
)
from vllm.platforms import current_platform
from vllm.utils.math_utils import round_up
from vllm.utils.torch_utils import _USE_LAYERNAME, _encode_layer_name
from ..vllm_inductor_pass import VllmFusionPatternMatcherPass, VllmPatternReplacement
from .matcher_utils import MatcherQuantFP8
from .rms_quant_fusion import QUANT_OPS
logger = init_logger(__name__)
FP8_DTYPE = current_platform.fp8_dtype()
FP4_DTYPE = torch.uint8
MLA_ATTN_OP = torch.ops.vllm.unified_mla_attention_with_output.default
class MLAAttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
"""Fusion for MLA Attention+Fp8StaticQuant.
Matches the pattern: MLA attention -> static FP8 quant, and replaces
it with MLA attention(output_scale=scale, output=fp8_buffer).
"""
def __init__(self, layer: MLAAttention, dtype: torch.dtype) -> None:
self._layer_name = layer.layer_name
self._num_heads = layer.num_heads
self._v_head_dim = layer.v_head_dim
self._kv_lora_rank = layer.kv_lora_rank
self._qk_rope_head_dim = layer.qk_rope_head_dim
self._qk_head_dim = layer.qk_nope_head_dim + layer.qk_rope_head_dim
self._output_dim = layer.num_heads * layer.v_head_dim
self._dtype = dtype
self._quant_matcher = MatcherQuantFP8(kFp8StaticTensorSym)
@property
def pattern(self) -> Callable[..., torch.Tensor]:
_ln = _encode_layer_name(self._layer_name)
if _USE_LAYERNAME:
def _pattern_with_ln( # type: ignore[misc]
q,
kv_c_normed,
k_pe,
output_attn,
scale,
kv_cache_dummy_dep,
layer_name,
):
at1 = auto_functionalized(
MLA_ATTN_OP,
q=q,
kv_c_normed=kv_c_normed,
k_pe=k_pe,
output=output_attn,
layer_name=layer_name,
output_scale=None,
output_block_scale=None,
kv_cache_dummy_dep=kv_cache_dummy_dep,
)
# MLA output is already 2D (T, N*V), no reshape needed
return self._quant_matcher(at1[1], scale)[0]
return _pattern_with_ln
def _pattern(q, kv_c_normed, k_pe, output_attn, scale, kv_cache_dummy_dep):
at1 = auto_functionalized(
MLA_ATTN_OP,
q=q,
kv_c_normed=kv_c_normed,
k_pe=k_pe,
output=output_attn,
layer_name=_ln,
output_scale=None,
output_block_scale=None,
kv_cache_dummy_dep=kv_cache_dummy_dep,
)
# MLA output is already 2D (T, N*V), no reshape needed
return self._quant_matcher(at1[1], scale)[0]
return _pattern
@property
def replacement(self) -> Callable[..., torch.Tensor]:
_ln = _encode_layer_name(self._layer_name)
if _USE_LAYERNAME:
def _replacement_with_ln( # type: ignore[misc]
q,
kv_c_normed,
k_pe,
output_attn,
scale,
kv_cache_dummy_dep,
layer_name,
):
# MLA output in quant_dtype
output_attn = torch.empty(
[q.shape[0], self._output_dim],
dtype=FP8_DTYPE,
device=q.device,
)
at1 = auto_functionalized(
MLA_ATTN_OP,
q=q,
kv_c_normed=kv_c_normed,
k_pe=k_pe,
output=output_attn,
layer_name=layer_name,
output_scale=scale,
output_block_scale=None,
kv_cache_dummy_dep=kv_cache_dummy_dep,
)
return at1[1]
return _replacement_with_ln
def _replacement(q, kv_c_normed, k_pe, output_attn, scale, kv_cache_dummy_dep):
# MLA output in quant_dtype
output_attn = torch.empty(
[q.shape[0], self._output_dim],
dtype=FP8_DTYPE,
device=q.device,
)
at1 = auto_functionalized(
MLA_ATTN_OP,
q=q,
kv_c_normed=kv_c_normed,
k_pe=k_pe,
output=output_attn,
layer_name=_ln,
output_scale=scale,
output_block_scale=None,
kv_cache_dummy_dep=kv_cache_dummy_dep,
)
return at1[1]
return _replacement
def get_inputs(self) -> list[torch.Tensor]:
inputs: list = [
self.empty(5, self._num_heads, self._qk_head_dim, dtype=self._dtype),
self.empty(5, self._kv_lora_rank, dtype=self._dtype),
self.empty(5, 1, self._qk_rope_head_dim, dtype=self._dtype),
self.empty(5, self._output_dim, dtype=self._dtype),
self.empty_fp32(1, 1),
self.empty(0, dtype=self._dtype),
]
if _USE_LAYERNAME:
inputs.append(_encode_layer_name(self._layer_name))
return inputs
class MLAAttnNvfp4QuantPattern(
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor]]
):
"""Fusion for MLA Attention+Nvfp4Quant.
Matches the pattern: MLA attention -> NVFP4 quant, and replaces
it with MLA attention(output_scale=scale, output_block_scale=block_scale,
output=fp4_buffer).
"""
def __init__(self, layer: MLAAttention, dtype: torch.dtype) -> None:
self._layer_name = layer.layer_name
self._num_heads = layer.num_heads
self._v_head_dim = layer.v_head_dim
self._kv_lora_rank = layer.kv_lora_rank
self._qk_rope_head_dim = layer.qk_rope_head_dim
self._qk_head_dim = layer.qk_nope_head_dim + layer.qk_rope_head_dim
self._output_dim = layer.num_heads * layer.v_head_dim
self._dtype = dtype
self._QUANT_OP = QUANT_OPS[kNvfp4Dynamic]
@property
def pattern(
self,
) -> Callable[..., tuple[torch.Tensor, torch.Tensor]]:
_ln = _encode_layer_name(self._layer_name)
if _USE_LAYERNAME:
def _pattern_with_ln( # type: ignore[misc]
q,
kv_c_normed,
k_pe,
output_attn,
output_quant,
output_scale,
input_scale,
kv_cache_dummy_dep,
layer_name,
):
at1 = auto_functionalized(
MLA_ATTN_OP,
q=q,
kv_c_normed=kv_c_normed,
k_pe=k_pe,
output=output_attn,
layer_name=layer_name,
output_scale=None,
output_block_scale=None,
kv_cache_dummy_dep=kv_cache_dummy_dep,
)
at2 = auto_functionalized(
self._QUANT_OP,
input=at1[1],
input_scale=input_scale,
is_sf_swizzled_layout=True,
output=output_quant,
output_scale=output_scale,
)
output_scale_view = torch.ops.aten.view.dtype(at2[2], FP8_DTYPE)
return at2[1], output_scale_view
return _pattern_with_ln
def _pattern(
q,
kv_c_normed,
k_pe,
output_attn,
output_quant,
output_scale,
input_scale,
kv_cache_dummy_dep,
):
at1 = auto_functionalized(
MLA_ATTN_OP,
q=q,
kv_c_normed=kv_c_normed,
k_pe=k_pe,
output=output_attn,
layer_name=_ln,
output_scale=None,
output_block_scale=None,
kv_cache_dummy_dep=kv_cache_dummy_dep,
)
at2 = auto_functionalized(
self._QUANT_OP,
input=at1[1],
input_scale=input_scale,
is_sf_swizzled_layout=True,
output=output_quant,
output_scale=output_scale,
)
output_scale_view = torch.ops.aten.view.dtype(at2[2], FP8_DTYPE)
return at2[1], output_scale_view
return _pattern
@property
def replacement(
self,
) -> Callable[..., tuple[torch.Tensor, torch.Tensor]]:
_ln = _encode_layer_name(self._layer_name)
if _USE_LAYERNAME:
def _replacement_with_ln( # type: ignore[misc]
q,
kv_c_normed,
k_pe,
output_attn,
_output_quant,
output_scale,
input_scale,
kv_cache_dummy_dep,
layer_name,
):
# MLA output in quant_dtype (FP4 packed as uint8)
output_attn = torch.empty(
[q.shape[0], self._output_dim // 2],
dtype=FP4_DTYPE,
device=q.device,
)
output_scale_view = torch.ops.aten.view.dtype(output_scale, FP8_DTYPE)
at2 = auto_functionalized(
MLA_ATTN_OP,
q=q,
kv_c_normed=kv_c_normed,
k_pe=k_pe,
output=output_attn,
layer_name=layer_name,
output_scale=input_scale,
output_block_scale=output_scale_view,
kv_cache_dummy_dep=kv_cache_dummy_dep,
)
return at2[1], at2[2]
return _replacement_with_ln
def _replacement(
q,
kv_c_normed,
k_pe,
output_attn,
_output_quant,
output_scale,
input_scale,
kv_cache_dummy_dep,
):
# MLA output in quant_dtype (FP4 packed as uint8)
output_attn = torch.empty(
[q.shape[0], self._output_dim // 2],
dtype=FP4_DTYPE,
device=q.device,
)
output_scale_view = torch.ops.aten.view.dtype(output_scale, FP8_DTYPE)
at2 = auto_functionalized(
MLA_ATTN_OP,
q=q,
kv_c_normed=kv_c_normed,
k_pe=k_pe,
output=output_attn,
layer_name=_ln,
output_scale=input_scale,
output_block_scale=output_scale_view,
kv_cache_dummy_dep=kv_cache_dummy_dep,
)
return at2[1], at2[2]
return _replacement
def get_inputs(self) -> list[torch.Tensor]:
inputs: list = [
self.empty(5, self._num_heads, self._qk_head_dim, dtype=self._dtype),
self.empty(5, self._kv_lora_rank, dtype=self._dtype),
self.empty(5, 1, self._qk_rope_head_dim, dtype=self._dtype),
self.empty(5, self._output_dim, dtype=self._dtype),
self.empty(5, self._output_dim // 2, dtype=FP4_DTYPE),
self.empty_i32(128, round_up(self._output_dim // 16, 4)),
self.empty_fp32(1, 1),
self.empty(0, dtype=self._dtype),
]
if _USE_LAYERNAME:
inputs.append(_encode_layer_name(self._layer_name))
return inputs
class MLAAttnFp8GroupQuantPattern(
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor]]
):
"""Fusion for MLA Attention+Fp8GroupQuant (per-group dynamic FP8).
Matches the pattern: MLA attention -> per_token_group_fp8_quant, and
replaces it with MLA attention(output_block_scale=group_scale_buffer).
Used by models with block FP8 quantization (e.g. DeepSeek V3).
"""
def __init__(
self,
layer: MLAAttention,
dtype: torch.dtype,
quant_key: QuantKey,
has_col_major_scales: bool,
is_e8m0: bool,
is_tma_aligned: bool,
) -> None:
self._layer_name = layer.layer_name
self._num_heads = layer.num_heads
self._v_head_dim = layer.v_head_dim
self._kv_lora_rank = layer.kv_lora_rank
self._qk_rope_head_dim = layer.qk_rope_head_dim
self._qk_head_dim = layer.qk_nope_head_dim + layer.qk_rope_head_dim
self._output_dim = layer.num_heads * layer.v_head_dim
self._dtype = dtype
self._layer = layer
self._group_size = quant_key.scale.group_shape[1]
self._has_col_major_scales = has_col_major_scales
self._is_e8m0 = is_e8m0
self._is_tma_aligned = is_tma_aligned
self._quant_matcher = MatcherQuantFP8(
quant_key,
has_col_major_scales=has_col_major_scales,
is_e8m0=is_e8m0,
is_tma_aligned=is_tma_aligned,
)
@property
def pattern(
self,
) -> Callable[..., tuple[torch.Tensor, torch.Tensor]]:
_ln = _encode_layer_name(self._layer_name)
if _USE_LAYERNAME:
def _pattern_with_ln( # type: ignore[misc]
q,
kv_c_normed,
k_pe,
output_attn,
kv_cache_dummy_dep,
scale,
layer_name,
):
at1 = auto_functionalized(
MLA_ATTN_OP,
q=q,
kv_c_normed=kv_c_normed,
k_pe=k_pe,
output=output_attn,
layer_name=layer_name,
output_scale=None,
output_block_scale=None,
kv_cache_dummy_dep=kv_cache_dummy_dep,
)
attn_out = at1[1]
result = torch.empty(
attn_out.shape, device=attn_out.device, dtype=FP8_DTYPE
)
finfo = torch.finfo(FP8_DTYPE)
_, result, scale = auto_functionalized(
self._quant_matcher.QUANT_OP,
input=attn_out,
output_q=result,
output_s=scale,
group_size=self._group_size,
eps=1e-10,
fp8_min=finfo.min,
fp8_max=finfo.max,
scale_ue8m0=self._is_e8m0,
dummy_is_scale_transposed=self._has_col_major_scales,
dummy_is_tma_aligned=self._is_tma_aligned,
)
return result, scale
return _pattern_with_ln
def _pattern(
q: torch.Tensor,
kv_c_normed: torch.Tensor,
k_pe: torch.Tensor,
output_attn: torch.Tensor,
kv_cache_dummy_dep: torch.Tensor,
scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
at1 = auto_functionalized(
MLA_ATTN_OP,
q=q,
kv_c_normed=kv_c_normed,
k_pe=k_pe,
output=output_attn,
layer_name=_ln,
output_scale=None,
output_block_scale=None,
kv_cache_dummy_dep=kv_cache_dummy_dep,
)
attn_out = at1[1]
result = torch.empty(
attn_out.shape, device=attn_out.device, dtype=FP8_DTYPE
)
finfo = torch.finfo(FP8_DTYPE)
_, result, scale = auto_functionalized(
self._quant_matcher.QUANT_OP,
input=attn_out,
output_q=result,
output_s=scale,
group_size=self._group_size,
eps=1e-10,
fp8_min=finfo.min,
fp8_max=finfo.max,
scale_ue8m0=self._is_e8m0,
dummy_is_scale_transposed=self._has_col_major_scales,
dummy_is_tma_aligned=self._is_tma_aligned,
)
return result, scale
return _pattern
@property
def replacement(
self,
) -> Callable[..., tuple[torch.Tensor, torch.Tensor]]:
_ln = _encode_layer_name(self._layer_name)
if _USE_LAYERNAME:
def _replacement_with_ln( # type: ignore[misc]
q,
kv_c_normed,
k_pe,
output_attn,
kv_cache_dummy_dep,
scale,
layer_name,
):
output_attn = torch.empty(
[q.shape[0], self._output_dim],
dtype=FP8_DTYPE,
device=q.device,
)
at1 = auto_functionalized(
MLA_ATTN_OP,
q=q,
kv_c_normed=kv_c_normed,
k_pe=k_pe,
output=output_attn,
layer_name=layer_name,
output_scale=None,
output_block_scale=scale,
kv_cache_dummy_dep=kv_cache_dummy_dep,
quant_group_size=self._group_size,
quant_scale_ue8m0=self._is_e8m0,
quant_col_major=self._has_col_major_scales,
quant_tma_aligned=self._is_tma_aligned,
)
return at1[1], at1[2]
return _replacement_with_ln
def _replacement(q, kv_c_normed, k_pe, output_attn, kv_cache_dummy_dep, scale):
output_attn = torch.empty(
[q.shape[0], self._output_dim],
dtype=FP8_DTYPE,
device=q.device,
)
at1 = auto_functionalized(
MLA_ATTN_OP,
q=q,
kv_c_normed=kv_c_normed,
k_pe=k_pe,
output=output_attn,
layer_name=_ln,
output_scale=None,
output_block_scale=scale,
kv_cache_dummy_dep=kv_cache_dummy_dep,
quant_group_size=self._group_size,
quant_scale_ue8m0=self._is_e8m0,
quant_col_major=self._has_col_major_scales,
quant_tma_aligned=self._is_tma_aligned,
)
return at1[1], at1[2]
return _replacement
def get_inputs(self) -> list[torch.Tensor]:
inputs: list = [
self.empty(5, self._num_heads, self._qk_head_dim, dtype=self._dtype),
self.empty(5, self._kv_lora_rank, dtype=self._dtype),
self.empty(5, 1, self._qk_rope_head_dim, dtype=self._dtype),
self.empty(5, self._output_dim, dtype=self._dtype),
self.empty(0, dtype=self._dtype),
self._quant_matcher.empty_f32(1, 1),
]
if _USE_LAYERNAME:
inputs.append(_encode_layer_name(self._layer_name))
return inputs
class MLAAttnQuantFusionPass(VllmFusionPatternMatcherPass):
"""This pass fuses post-attention quantization onto MLA attention if supported.
It uses the pattern matcher and matches each MLA layer manually, as strings
cannot be wildcarded. This also lets us check support on attention layers
upon registration instead of during pattern matching.
"""
def __init__(self, config: VllmConfig) -> None:
super().__init__(config, "mla_attn_quant_fusion")
dtype = config.model_config.dtype
layers = list(get_layers_from_vllm_config(config, MLAAttention).values())
if len(layers) == 0:
logger.warning(
"MLA attention + quant fusion is enabled, but no MLA "
"attention layers were found in "
"CompilationConfig.static_forward_context "
"so no fusion patterns were registered."
)
# When _USE_LAYERNAME is enabled, layer_name is a wildcard so all
# layers produce the same pattern — register once then break.
for layer in layers:
if layer.impl.fused_output_quant_supported(kFp8StaticTensorSym):
self.register(MLAAttnFp8StaticQuantPattern(layer, dtype))
if _USE_LAYERNAME:
break
if current_platform.is_cuda() and hasattr(torch.ops._C, "scaled_fp4_quant"):
for layer in layers:
if layer.impl.fused_output_quant_supported(kNvfp4Dynamic):
self.register(MLAAttnNvfp4QuantPattern(layer, dtype))
if _USE_LAYERNAME:
break
# Per-group FP8 (block quant) — register all flag combinations.
if current_platform.is_cuda():
for quant_key in [kFp8Dynamic128Sym, kFp8Dynamic64Sym]:
for col_major in [True, False]:
for is_e8m0 in [True, False]:
for tma_aligned in [False, True]:
for layer in layers:
if layer.impl.fused_output_quant_supported(quant_key):
self.register(
MLAAttnFp8GroupQuantPattern(
layer,
dtype,
quant_key,
col_major,
is_e8m0,
tma_aligned,
)
)
if _USE_LAYERNAME:
break
self.dump_patterns(config, self.pm_pass)
@@ -1,274 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from torch._higher_order_ops.auto_functionalize import auto_functionalized
import vllm._custom_ops as ops
from vllm.config import VllmConfig, get_layers_from_vllm_config
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import MLAAttention
from vllm.model_executor.layers.attention.attention import get_attention_context
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
from vllm.utils.torch_utils import (
_USE_LAYERNAME,
LayerNameType,
_encode_layer_name,
_resolve_layer_name,
direct_register_custom_op,
)
from ..vllm_inductor_pass import VllmFusionPatternMatcherPass, VllmPatternReplacement
from .matcher_utils import MatcherDeepseekScalingRotaryEmbedding, MatcherRotaryEmbedding
logger = init_logger(__name__)
def fused_rope_unified_mla_kv_cache_update_impl(
positions: torch.Tensor,
q_pe: torch.Tensor,
k_pe: torch.Tensor,
kv_c: torch.Tensor,
cos_sin_cache: torch.Tensor,
is_neox: bool,
kv_cache_dtype: str,
kv_cache_scale: torch.Tensor,
layer_name: LayerNameType,
) -> torch.Tensor:
layer_name = _resolve_layer_name(layer_name)
attn_metadata, attn_layer, kv_cache, layer_slot_mapping = get_attention_context(
layer_name
)
attn_layer.prepare_kv_cache_update(attn_metadata)
if layer_slot_mapping is not None:
ops.concat_and_cache_mla_rope_fused(
positions,
q_pe,
k_pe,
kv_c,
cos_sin_cache,
is_neox,
layer_slot_mapping,
kv_cache.squeeze(1),
kv_cache_dtype,
kv_cache_scale,
)
return torch.empty(0, device=kv_c.device, dtype=kv_c.dtype)
def fused_rope_unified_mla_kv_cache_update_fake(
positions: torch.Tensor,
q_pe: torch.Tensor,
k_pe: torch.Tensor,
kv_c: torch.Tensor,
cos_sin_cache: torch.Tensor,
is_neox: bool,
kv_cache_dtype: str,
kv_cache_scale: torch.Tensor,
layer_name: LayerNameType,
) -> torch.Tensor:
return torch.empty(0, dtype=kv_c.dtype, device=kv_c.device)
direct_register_custom_op(
op_name="fused_rope_unified_mla_kv_cache_update",
op_func=fused_rope_unified_mla_kv_cache_update_impl,
fake_impl=fused_rope_unified_mla_kv_cache_update_fake,
mutates_args=["q_pe", "k_pe"],
)
class MLARoPEKVCacheCatPattern(VllmPatternReplacement):
FUSED_OP = torch.ops.vllm.fused_rope_unified_mla_kv_cache_update.default
def __init__(
self,
layer: MLAAttention,
is_neox: bool,
use_flashinfer: bool = False,
use_deepseek_scaling: bool = False,
) -> None:
self.layer_name = layer.layer_name
self.kv_cache_dtype = layer.kv_cache_dtype
self.num_heads = layer.num_heads
self.num_kv_heads = layer.num_kv_heads
self.kv_lora_rank = layer.kv_lora_rank
self.qk_rope_head_dim = layer.qk_rope_head_dim
self.is_neox = is_neox
self.use_flashinfer = use_flashinfer
self._ln = _encode_layer_name(self.layer_name)
if use_deepseek_scaling:
self.rope_matcher = MatcherDeepseekScalingRotaryEmbedding(
is_neox=self.is_neox,
head_size=self.qk_rope_head_dim,
num_heads=self.num_heads,
num_kv_heads=self.num_kv_heads,
use_flashinfer=self.use_flashinfer,
)
else:
self.rope_matcher = MatcherRotaryEmbedding( # type: ignore
is_neox=self.is_neox,
head_size=self.qk_rope_head_dim,
num_heads=self.num_heads,
num_kv_heads=self.num_kv_heads,
use_flashinfer=self.use_flashinfer,
)
def get_inputs(self) -> list[torch.Tensor]:
T = 5
L = 4096
q_pe = self.empty_bf16(T, self.num_heads, self.qk_rope_head_dim)
k_pe = self.empty_bf16(T, self.qk_rope_head_dim)
kv_c_normed = self.empty_bf16(T, self.kv_lora_rank)
cos_sin_cache = self.empty_bf16(L, self.qk_rope_head_dim)
positions = self.empty(T, dtype=torch.int64)
k_scale = self.empty(0, dtype=torch.float32)
inputs = [
q_pe,
k_pe,
kv_c_normed,
positions,
cos_sin_cache,
k_scale,
]
if _USE_LAYERNAME:
inputs.append(self._ln)
return inputs
@property
def pattern(self):
_ln = self._ln
if _USE_LAYERNAME:
def _pattern_with_ln(
q_pe: torch.Tensor,
k_pe: torch.Tensor,
kv_c_normed: torch.Tensor,
positions: torch.Tensor,
cos_sin_cache: torch.Tensor,
k_scale: torch.Tensor,
layer_name: LayerNameType,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
k_pe_unsqueezed = k_pe.unsqueeze(1)
q_pe, k_pe = self.rope_matcher(
positions, q_pe, k_pe_unsqueezed, cos_sin_cache
)
dummy = torch.ops.vllm.unified_mla_kv_cache_update(
kv_c_normed, k_pe, layer_name, self.kv_cache_dtype, k_scale
)
return dummy, q_pe, k_pe
return _pattern_with_ln
def _pattern(
q_pe: torch.Tensor,
k_pe: torch.Tensor,
kv_c_normed: torch.Tensor,
positions: torch.Tensor,
cos_sin_cache: torch.Tensor,
k_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
k_pe_unsqueezed = k_pe.unsqueeze(1)
q_pe, k_pe = self.rope_matcher(
positions, q_pe, k_pe_unsqueezed, cos_sin_cache
)
dummy = torch.ops.vllm.unified_mla_kv_cache_update(
kv_c_normed, k_pe, _ln, self.kv_cache_dtype, k_scale
)
return dummy, q_pe, k_pe
return _pattern
@property
def replacement(self):
_ln = self._ln
if _USE_LAYERNAME:
def _replacement_with_ln(
q_pe: torch.Tensor,
k_pe: torch.Tensor,
kv_c_normed: torch.Tensor,
positions: torch.Tensor,
cos_sin_cache: torch.Tensor,
k_scale: torch.Tensor,
layer_name: LayerNameType,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
at = auto_functionalized(
self.FUSED_OP,
positions=positions,
q_pe=q_pe,
k_pe=k_pe,
kv_c=kv_c_normed,
cos_sin_cache=cos_sin_cache,
is_neox=self.is_neox,
kv_cache_dtype=self.kv_cache_dtype,
kv_cache_scale=k_scale,
layer_name=layer_name,
)
dummy, q_pe, k_pe_squeezed = at
k_pe = k_pe_squeezed.unsqueeze(1)
return dummy, q_pe, k_pe
return _replacement_with_ln
def _replacement(
q_pe: torch.Tensor,
k_pe: torch.Tensor,
kv_c_normed: torch.Tensor,
positions: torch.Tensor,
cos_sin_cache: torch.Tensor,
k_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
at = auto_functionalized(
self.FUSED_OP,
positions=positions,
q_pe=q_pe,
k_pe=k_pe,
kv_c=kv_c_normed,
cos_sin_cache=cos_sin_cache,
is_neox=self.is_neox,
kv_cache_dtype=self.kv_cache_dtype,
kv_cache_scale=k_scale,
layer_name=_ln,
)
dummy, q_pe, k_pe_squeezed = at
k_pe = k_pe_squeezed.unsqueeze(1)
return dummy, q_pe, k_pe
return _replacement
class MLARoPEKVCacheCatFusionPass(VllmFusionPatternMatcherPass):
def __init__(self, config: VllmConfig) -> None:
super().__init__(config, "mla_rope_kv_cache_fusion_pass")
attn_layers = get_layers_from_vllm_config(config, MLAAttention)
for _, layer in attn_layers.items():
for is_neox in [False, True]:
for use_deepseek_scaling in [False, True]:
if RotaryEmbedding.enabled():
for use_flashinfer in [False, True]:
self.register(
MLARoPEKVCacheCatPattern(
layer,
is_neox,
use_flashinfer,
use_deepseek_scaling,
)
)
else:
self.register(
MLARoPEKVCacheCatPattern(
layer,
is_neox,
use_deepseek_scaling=use_deepseek_scaling,
)
)
if _USE_LAYERNAME:
break
self.dump_patterns(config, self.pm_pass)
-6
View File
@@ -40,8 +40,6 @@ if current_platform.is_cuda_alike() or current_platform.is_xpu():
if current_platform.is_cuda_alike():
from .fusion.act_quant_fusion import ActivationQuantFusionPass
from .fusion.attn_quant_fusion import AttnQuantFusionPass
from .fusion.mla_attn_quant_fusion import MLAAttnQuantFusionPass
from .fusion.mla_rope_kvcache_cat_fusion import MLARoPEKVCacheCatFusionPass
from .fusion.qk_norm_rope_kvcache_fusion import QkNormRopeKvCacheFusionPass
from .fusion.rms_quant_fusion import RMSNormQuantFusionPass
from .fusion.rope_kvcache_fusion import RopeKVCacheFusionPass
@@ -206,12 +204,8 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
self.passes += [ScatterSplitReplacementPass(config)]
self.passes += [RopeKVCacheFusionPass(config)]
if self.pass_config.fuse_rope_kvcache_cat_mla:
self.passes += [MLARoPEKVCacheCatFusionPass(config)]
if self.pass_config.fuse_attn_quant:
self.passes += [AttnQuantFusionPass(config)]
self.passes += [MLAAttnQuantFusionPass(config)]
if self.pass_config.enable_qk_norm_rope_fusion:
self.passes += [SplitCoalescingPass(config)]
+5 -23
View File
@@ -1072,30 +1072,12 @@ class ModelConfig:
self, mm_device_do_normalize: bool | None
) -> bool:
if mm_device_do_normalize is None:
if envs.VLLM_USE_RUST_FRONTEND:
logger.debug(
"VLLM_USE_RUST_FRONTEND is set. "
"Rust frontend does not currently support mm_device_do_normalize, "
"forcing mm_device_do_normalize = False."
)
mm_device_do_normalize = False
else:
mm_device_do_normalize = (
self._model_info.supports_mm_device_do_normalize
)
logger.debug(
"mm_device_do_normalize is %s by default.",
"enabled" if mm_device_do_normalize else "disabled",
)
mm_device_do_normalize = self._model_info.supports_mm_device_do_normalize
logger.debug(
"mm_device_do_normalize is %s by default.",
"enabled" if mm_device_do_normalize else "disabled",
)
else:
if mm_device_do_normalize and envs.VLLM_USE_RUST_FRONTEND:
logger.warning(
"VLLM_USE_RUST_FRONTEND is set. "
"Rust frontend does not currently support mm_device_do_normalize, "
"forcing mm_device_do_normalize = False."
)
mm_device_do_normalize = False
if (
mm_device_do_normalize
and not self._model_info.supports_mm_device_do_normalize
@@ -1,457 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any
import safetensors
import torch
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorMetadata,
KVConnectorRole,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.attention.mla_attention import MLACommonMetadata
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.hashing import safe_hash
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.sched.output import SchedulerOutput
if TYPE_CHECKING:
from vllm.forward_context import ForwardContext
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.request import Request
logger = init_logger(__name__)
@dataclass
class ReqMeta:
# Request tokens
token_ids: torch.Tensor
# Slot mappings, should have the same length as token_ids
slot_mapping: torch.Tensor
# Is store or load
is_store: bool
mm_hashes: list[str]
@staticmethod
def make_meta(
token_ids: list[int],
block_ids: list[int],
block_size: int,
is_store: bool,
mm_hashes: list[str],
) -> "ReqMeta":
valid_num_tokens = align_to_block_size(len(token_ids), block_size)
token_ids_tensor = torch.tensor(token_ids)[:valid_num_tokens]
block_ids_tensor = torch.tensor(block_ids)
num_blocks = block_ids_tensor.shape[0]
block_offsets = torch.arange(0, block_size)
slot_mapping = (
block_offsets.reshape((1, block_size))
+ block_ids_tensor.reshape((num_blocks, 1)) * block_size
)
slot_mapping = slot_mapping.flatten()[:valid_num_tokens]
return ReqMeta(
token_ids=token_ids_tensor,
slot_mapping=slot_mapping,
is_store=is_store,
mm_hashes=mm_hashes,
)
@dataclass
class ExampleConnectorMetadata(KVConnectorMetadata):
requests: list[ReqMeta] = field(default_factory=list)
def add_request(
self,
token_ids: list[int],
block_ids: list[int],
block_size: int,
is_store: bool,
mm_hashes: list[str],
) -> None:
self.requests.append(
ReqMeta.make_meta(token_ids, block_ids, block_size, is_store, mm_hashes)
)
class ExampleConnector(KVConnectorBase_V1):
# NOTE: This is Simple debug implementation of the KV connector.
# It save / load the KV cache to / from the disk.
# It does extra work which will overwrite the existing prefix-cache in GPU
# - to remove the overhead, need to add some "mask" in the ReqMeta class
def __init__(
self,
vllm_config: "VllmConfig",
role: KVConnectorRole,
kv_cache_config: "KVCacheConfig",
):
super().__init__(
vllm_config=vllm_config,
role=role,
kv_cache_config=kv_cache_config,
)
self._block_size = vllm_config.cache_config.block_size
self._requests_need_load: dict[str, Request] = {}
self._storage_path = self._kv_transfer_config.get_from_extra_config(
"shared_storage_path", "/tmp"
)
logger.info(self._kv_transfer_config)
logger.info("Shared storage path is %s", self._storage_path)
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
"""Start loading the KV cache from the connector buffer to vLLM's
paged KV buffer.
Args:
forward_context (ForwardContext): the forward context.
**kwargs: additional arguments for the load operation
Note:
The number of elements in kv_caches and layer_names should be
the same.
"""
def inject_kv_into_layer(
dst_kv_cache_layer: torch.Tensor,
src_kv_cache: torch.Tensor,
slot_mapping: torch.Tensor,
attn_metadata: AttentionMetadata,
) -> None:
"""Inject the KV cache into the layer.
Args:
dst_kv_cache_layer (torch.Tensor): the destination KV cache layer,
a standardized [B, H, N, C] per-layer view (H == 1 for MLA).
src_kv_cache (torch.Tensor): the source KV cache.
slot_mapping (torch.Tensor): the slot mapping. In shape
[num_tokens].
attn_metadata (AttentionMetadata): attention metadata for the
current forward pass.
"""
# Deliberate bulk transfers in this reference connector.
with gpu_sync_allowed():
slot_mapping = slot_mapping.to(
dst_kv_cache_layer.device, non_blocking=True
)
if isinstance(attn_metadata, MLACommonMetadata):
# [B, 1, N, C] -> [B * N, C]; slot_mapping indexes B * N slots.
dst_kv_cache_layer = dst_kv_cache_layer.reshape(
-1, dst_kv_cache_layer.shape[-1]
)
dst_kv_cache_layer[slot_mapping, ...] = src_kv_cache
else:
block_idxs = slot_mapping // self._block_size
offsets = slot_mapping % self._block_size
dst_kv_cache_layer[block_idxs, :, offsets] = src_kv_cache
# Get the metadata
metadata: KVConnectorMetadata = self._get_connector_metadata()
assert isinstance(metadata, ExampleConnectorMetadata)
attn_metadata = forward_context.attn_metadata
if attn_metadata is None:
logger.warning("In connector.start_load_kv, but the attn_metadata is None")
return
# Load the KV for each request each layer
for request in metadata.requests:
if request.is_store:
continue
logger.info(
"Inject KV cache of %d tokens to the paged memory",
len(request.slot_mapping),
)
for layer_name in forward_context.no_compile_layers:
layer = forward_context.no_compile_layers[layer_name]
# Only process layers that have kv_cache
# attribute (attention layers) Skip non-attention
# layers like FusedMoEFactory/MLP etc.
kv_cache_layer = getattr(layer, "kv_cache", None)
if kv_cache_layer is None:
continue
filename = self._generate_filename_debug(
layer_name, request.token_ids, request.mm_hashes
)
# Deliberate bulk load in this reference connector.
with gpu_sync_allowed():
kv_cache_cpu = safetensors.torch.load_file(filename)["kv_cache"]
kv_cache = kv_cache_cpu.to(kv_cache_layer.device, non_blocking=True)
if isinstance(attn_metadata, dict):
inject_kv_into_layer(
kv_cache_layer,
kv_cache,
request.slot_mapping,
attn_metadata[layer_name],
)
def wait_for_layer_load(self, layer_name: str) -> None:
"""Blocking until the KV for a specific layer is loaded into vLLM's
paged buffer.
This interface will be useful for layer-by-layer pipelining.
Args:
layer_name: the name of that layer
"""
return
def save_kv_layer(
self,
layer_name: str,
kv_layer: torch.Tensor,
attn_metadata: AttentionMetadata,
**kwargs: Any,
) -> None:
"""Start saving the KV cache of the layer from vLLM's paged buffer
to the connector.
Args:
layer_name (str): the name of the layer.
kv_layer (torch.Tensor): the paged KV buffer of the current
layer in vLLM.
attn_metadata (AttentionMetadata): the attention metadata.
**kwargs: additional arguments for the save operation.
"""
def extract_kv_from_layer(
layer: torch.Tensor,
slot_mapping: torch.Tensor,
) -> torch.Tensor:
"""Extract the KV cache from the layer.
The layer is a standardized [B, H, N, C] per-layer view (H == 1 for MLA).
"""
# Deliberate bulk transfer in this reference connector.
with gpu_sync_allowed():
slot_mapping = slot_mapping.to(layer.device, non_blocking=True)
if isinstance(attn_metadata, MLACommonMetadata):
# [B, 1, N, C] -> [B * N, C]; slot_mapping indexes B * N slots.
return layer.reshape(-1, layer.shape[-1])[slot_mapping, ...]
block_idxs = slot_mapping // self._block_size
offsets = slot_mapping % self._block_size
return layer[block_idxs, :, offsets]
connector_metadata = self._get_connector_metadata()
assert isinstance(connector_metadata, ExampleConnectorMetadata)
for request in connector_metadata.requests:
if request.is_store:
filename = self._generate_filename_debug(
layer_name, request.token_ids, request.mm_hashes
)
kv_cache = extract_kv_from_layer(kv_layer, request.slot_mapping)
with gpu_sync_allowed():
tensors = {"kv_cache": kv_cache.detach().cpu()}
safetensors.torch.save_file(tensors, filename)
def wait_for_save(self):
return
def get_num_new_matched_tokens(
self,
request: "Request",
num_computed_tokens: int,
) -> tuple[int | None, bool]:
"""Get number of new tokens that can be loaded from the
external KV cache beyond the num_computed_tokens.
Args:
request (Request): the request object.
num_computed_tokens (int): the number of locally
computed tokens for this request
Returns:
the number of tokens that can be loaded from the
external KV cache beyond what is already computed.
"""
# NOTE: in this debug implementation, we assume that the prompt is
# cached_prompt + newly_generated_single_token
# Therefore, we use prompt_token_ids[:-1] to determine the folder name
# NOTE: in current v1 scheduler, the num_computed_tokens is aligned
# with the block granularity. And it expects the returned blocks and
# num_computed_tokens to also be aligned with the block granularity.
if not self._found_match_for_request(request):
return 0, False
logger.info("External Cache Hit!")
# Now, first num_tokens_to_check tokens are hit, we need to prepare
# the metadata for the worker connector to correctly load the KV
token_ids = request.prompt_token_ids or []
num_tokens_to_check = align_to_block_size(len(token_ids) - 1, self._block_size)
return num_tokens_to_check - num_computed_tokens, False
def update_state_after_alloc(
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
):
"""Update KVConnector state after block allocation.
If blocks were allocated, add to _requests_need_load,
such that we load the KVs in the next forward pass.
"""
if num_external_tokens > 0:
self._requests_need_load[request.request_id] = request
def build_connector_meta(
self,
scheduler_output: SchedulerOutput,
) -> KVConnectorMetadata:
"""Build the connector metadata for this step.
This function should NOT modify any fields in the scheduler_output.
Also, calling this function will reset the state of the connector.
Args:
scheduler_output (SchedulerOutput): the scheduler output object.
"""
meta = ExampleConnectorMetadata()
total_need_load = 0
for new_req in scheduler_output.scheduled_new_reqs:
token_ids = new_req.prompt_token_ids or []
mm_hashes = [f.identifier for f in new_req.mm_features]
if new_req.req_id in self._requests_need_load:
meta.add_request(
token_ids=token_ids,
block_ids=new_req.block_ids[0],
block_size=self._block_size,
is_store=False,
mm_hashes=mm_hashes,
)
total_need_load += 1
else:
# NOTE: here, we set the store and load being exclusive,
# but a single request can have both store and load.
# NOTE(rob): for this debug implementation, we only cache
# the original prompt tokens.
if not self._found_match_for_prompt(token_ids, mm_hashes):
meta.add_request(
token_ids=token_ids,
block_ids=new_req.block_ids[0],
block_size=self._block_size,
is_store=True,
mm_hashes=mm_hashes,
)
cached_reqs = scheduler_output.scheduled_cached_reqs
for i, req_id in enumerate(cached_reqs.req_ids):
resumed_from_preemption = req_id in cached_reqs.resumed_req_ids
if not resumed_from_preemption or req_id not in self._requests_need_load:
continue
num_computed_tokens = cached_reqs.num_computed_tokens[i]
num_new_tokens = scheduler_output.num_scheduled_tokens[req_id]
new_block_ids = cached_reqs.new_block_ids[i]
# NOTE(rob): cached_req_data does not have the full
# list of token ids (only new tokens). So we look it
# up in the actual request object.
request = self._requests_need_load[req_id]
total_tokens = num_computed_tokens + num_new_tokens
token_ids = request.all_token_ids[:total_tokens]
# NOTE(rob): For resumed req, new_block_ids is all
# of the block_ids for the request.
assert new_block_ids is not None
block_ids = new_block_ids[0]
meta.add_request(
token_ids=token_ids,
block_ids=block_ids,
block_size=self._block_size,
is_store=False,
mm_hashes=[f.identifier for f in request.mm_features],
)
total_need_load += 1
assert total_need_load == len(self._requests_need_load)
self._requests_need_load.clear()
return meta
# ==============================
# Helper functions
# ==============================
def _found_match_for_request(
self,
request: "Request",
) -> bool:
"""Check if the cache is hit for the request."""
return self._found_match_for_prompt(
list(request.prompt_token_ids or []),
[f.identifier for f in request.mm_features],
)
def _found_match_for_prompt(
self,
prompt_token_ids: list[int],
mm_hashes: list[str],
) -> bool:
num_tokens_to_check = align_to_block_size(
len(prompt_token_ids) - 1, self._block_size
)
foldername = self._generate_foldername_debug(
torch.tensor(prompt_token_ids)[:num_tokens_to_check],
mm_hashes,
create_folder=False,
)
return os.path.exists(foldername)
def _generate_foldername_debug(
self,
token_ids: torch.Tensor,
mm_hashes: list[str],
create_folder=False,
) -> str:
"""Generate a folder name based on the hash of the bytes of the input
ids.
"""
token_bytes = token_ids.numpy().tobytes()
# Add mm_hashes to the bytes being hashed to avoid path traversal and
# to create a canonical key.
if mm_hashes:
mm_str = "-".join(mm_hashes)
token_bytes += mm_str.encode("utf-8")
input_ids_hash = safe_hash(token_bytes, usedforsecurity=False).hexdigest()
foldername = os.path.join(self._storage_path, input_ids_hash)
if create_folder:
os.makedirs(foldername, exist_ok=True)
return foldername
def _generate_filename_debug(
self,
layer_name: str,
token_ids: torch.Tensor,
mm_hashes: list[str],
) -> str:
"""Generate a file name based on the layer name and the hash
of the bytes of the input ids.
"""
foldername = self._generate_foldername_debug(
token_ids, mm_hashes=mm_hashes, create_folder=True
)
return os.path.join(foldername, f"{layer_name}.safetensors")
def align_to_block_size(num_tokens: int, block_size) -> int:
"""Align the number of tokens to the block size."""
return (num_tokens - 1) // block_size * block_size
-27
View File
@@ -1,27 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
from vllm.entrypoints.cli.types import CLISubcommand
from vllm.utils.argparse_utils import FlexibleArgumentParser
class BenchmarkSubcommandBase(CLISubcommand):
"""The base class of subcommands for `vllm bench`."""
help: str
@classmethod
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
"""Add the CLI arguments to the parser."""
raise NotImplementedError
@staticmethod
def cmd(args: argparse.Namespace) -> None:
"""Run the benchmark.
Args:
args: The arguments to the command.
"""
raise NotImplementedError
-22
View File
@@ -1,22 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
from vllm.benchmarks.forward.main import add_cli_args, main
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
from vllm.utils.argparse_utils import FlexibleArgumentParser
class BenchmarkForwardSubcommand(BenchmarkSubcommandBase):
"""The `forward` subcommand for `vllm bench`."""
name = "forward"
help = "Benchmark the model forward of exact batches as CUDA graphs."
@classmethod
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
add_cli_args(parser)
@staticmethod
def cmd(args: argparse.Namespace) -> None:
main(args)
-22
View File
@@ -1,22 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
from vllm.benchmarks.latency import add_cli_args, main
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
from vllm.utils.argparse_utils import FlexibleArgumentParser
class BenchmarkLatencySubcommand(BenchmarkSubcommandBase):
"""The `latency` subcommand for `vllm bench`."""
name = "latency"
help = "Benchmark the latency of a single batch of requests."
@classmethod
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
add_cli_args(parser)
@staticmethod
def cmd(args: argparse.Namespace) -> None:
main(args)
-100
View File
@@ -1,100 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
import os
import sys
import typing
from vllm import envs
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
from vllm.entrypoints.cli.types import CLISubcommand
from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
from vllm.logger import init_logger
if typing.TYPE_CHECKING:
from vllm.utils.argparse_utils import FlexibleArgumentParser
else:
FlexibleArgumentParser = argparse.ArgumentParser
logger = init_logger(__name__)
def maybe_exec_rust_bench() -> None:
if sys.argv[1:3] != ["bench", "serve"] or not envs.VLLM_USE_RUST_BENCH:
return
rust_cli = envs.VLLM_RUST_FRONTEND_PATH
if rust_cli is None:
raise RuntimeError(
"VLLM_USE_RUST_BENCH=1 requires VLLM_RUST_FRONTEND_PATH "
"to resolve to the vllm-rs binary."
)
logger.info("Delegating `vllm bench serve` to Rust binary at %s.", rust_cli)
os.execv(rust_cli, [rust_cli, "bench", "serve", *sys.argv[3:]])
def _import_bench_subcommand_modules() -> None:
# Imported lazily so `BenchmarkSubcommandBase` subclasses register only
# when `vllm bench` is actually invoked.
import vllm.entrypoints.cli.benchmark.forward # noqa: F401
import vllm.entrypoints.cli.benchmark.latency # noqa: F401
import vllm.entrypoints.cli.benchmark.mm_processor # noqa: F401
import vllm.entrypoints.cli.benchmark.serve # noqa: F401
import vllm.entrypoints.cli.benchmark.startup # noqa: F401
import vllm.entrypoints.cli.benchmark.sweep # noqa: F401
import vllm.entrypoints.cli.benchmark.throughput # noqa: F401
class BenchmarkSubcommand(CLISubcommand):
"""The `bench` subcommand for the vLLM CLI."""
name = "bench"
help = "vLLM bench subcommand."
@staticmethod
def cmd(args: argparse.Namespace) -> None:
args.dispatch_function(args)
def validate(self, args: argparse.Namespace) -> None:
pass
def subparser_init(
self, subparsers: argparse._SubParsersAction
) -> FlexibleArgumentParser:
bench_parser = subparsers.add_parser(
self.name,
help=self.help,
description=self.help,
usage=f"vllm {self.name} <bench_type> [options]",
)
bench_subparsers = bench_parser.add_subparsers(required=True, dest="bench_type")
# Only build the nested bench subparsers when the user is actually
# invoking `bench`; otherwise we'd drag in imports
# unnecessarily on every `vllm --help` and `vllm serve`.
# Scan for the first positional arg so global flags (e.g. `-v`)
# before the subcommand don't break detection.
first_positional = next(
(arg for arg in sys.argv[1:] if not arg.startswith("-")), None
)
if first_positional == self.name:
_import_bench_subcommand_modules()
for cmd_cls in BenchmarkSubcommandBase.__subclasses__():
cmd_subparser = bench_subparsers.add_parser(
cmd_cls.name,
help=cmd_cls.help,
description=cmd_cls.help,
usage=f"vllm {self.name} {cmd_cls.name} [options]",
)
cmd_subparser.set_defaults(dispatch_function=cmd_cls.cmd)
cmd_cls.add_cli_args(cmd_subparser)
cmd_subparser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(
subcmd=f"{self.name} {cmd_cls.name}"
)
return bench_parser
def cmd_init() -> list[CLISubcommand]:
return [BenchmarkSubcommand()]
@@ -1,69 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
import inspect
from vllm.benchmarks.mm_processor import add_cli_args, main
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
from vllm.utils.argparse_utils import FlexibleArgumentParser
class BenchmarkMMProcessorSubcommand(BenchmarkSubcommandBase):
r"""`vllm bench mm-processor` profiles the multimodal input processor pipeline of
vision-language models. It measures per-stage latency from the HuggingFace
processor through to the encoder forward pass, helping you identify
preprocessing bottlenecks and understand how different image resolutions or
item counts affect end-to-end request time.
The benchmark supports two data sources: synthetic random multimodal inputs
(`random-mm`) and HuggingFace datasets (`hf`). Warmup requests are run before
measurement to ensure stable results.
## Quick Start
```bash
vllm bench mm-processor \
--model Qwen/Qwen2-VL-7B-Instruct \
--dataset-name random-mm \
--num-prompts 50 \
--random-input-len 300 \
--random-output-len 40 \
--random-mm-base-items-per-request 2 \
--random-mm-limit-mm-per-prompt '{"image": 3, "video": 0}' \
--random-mm-bucket-config '{(256, 256, 1): 0.7, (720, 1280, 1): 0.3}'
```
## Measured Stages
| Stage | Description |
| ----- | ----------- |
| `get_mm_hashes_secs` | Time spent hashing multimodal inputs |
| `get_cache_missing_items_secs` | Time spent looking up the processor cache |
| `apply_hf_processor_secs` | Time spent in the HuggingFace processor |
| `merge_mm_kwargs_secs` | Time spent merging multimodal kwargs |
| `apply_prompt_updates_secs` | Time spent updating prompt tokens |
| `preprocessor_total_secs` | Total preprocessing time |
| `encoder_forward_secs` | Time spent in the encoder model forward pass |
| `num_encoder_calls` | Number of encoder invocations per request |
The benchmark also reports end-to-end latency (TTFT + decode time) per
request. Use `--metric-percentiles` to select which percentiles to report
(default: p99) and `--output-json` to save results.
For more examples (HF datasets, warmup, JSON output), see the
[multimodal processor benchmark guide](https://docs.vllm.ai/en/latest/benchmarking/cli/#multimodal-processor-benchmark).
"""
name = "mm-processor"
help = "Benchmark multimodal processor latency across different configurations."
@classmethod
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
# The class docstring is the page overview / `--help` description.
if cls.__doc__:
parser.description = inspect.cleandoc(cls.__doc__)
add_cli_args(parser)
@staticmethod
def cmd(args: argparse.Namespace) -> None:
main(args)
-22
View File
@@ -1,22 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
from vllm.benchmarks.serve import add_cli_args, main
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
from vllm.utils.argparse_utils import FlexibleArgumentParser
class BenchmarkServingSubcommand(BenchmarkSubcommandBase):
"""The `serve` subcommand for `vllm bench`."""
name = "serve"
help = "Benchmark the online serving throughput."
@classmethod
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
add_cli_args(parser)
@staticmethod
def cmd(args: argparse.Namespace) -> None:
main(args)
-22
View File
@@ -1,22 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
from vllm.benchmarks.startup import add_cli_args, main
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
from vllm.utils.argparse_utils import FlexibleArgumentParser
class BenchmarkStartupSubcommand(BenchmarkSubcommandBase):
"""The `startup` subcommand for `vllm bench`."""
name = "startup"
help = "Benchmark the startup time of vLLM models."
@classmethod
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
add_cli_args(parser)
@staticmethod
def cmd(args: argparse.Namespace) -> None:
main(args)
-22
View File
@@ -1,22 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
from vllm.benchmarks.sweep.cli import add_cli_args, main
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
from vllm.utils.argparse_utils import FlexibleArgumentParser
class BenchmarkSweepSubcommand(BenchmarkSubcommandBase):
"""The `sweep` subcommand for `vllm bench`."""
name = "sweep"
help = "Benchmark for a parameter sweep."
@classmethod
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
add_cli_args(parser)
@staticmethod
def cmd(args: argparse.Namespace) -> None:
main(args)
@@ -1,22 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
from vllm.benchmarks.throughput import add_cli_args, main
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
from vllm.utils.argparse_utils import FlexibleArgumentParser
class BenchmarkThroughputSubcommand(BenchmarkSubcommandBase):
"""The `throughput` subcommand for `vllm bench`."""
name = "throughput"
help = "Benchmark offline inference throughput."
@classmethod
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
add_cli_args(parser)
@staticmethod
def cmd(args: argparse.Namespace) -> None:
main(args)
-20
View File
@@ -40,7 +40,6 @@ def main():
if maybe_run_omni():
return
import vllm.entrypoints.cli.benchmark.main
import vllm.entrypoints.cli.collect_env
import vllm.entrypoints.cli.download_kernels
import vllm.entrypoints.cli.launch
@@ -59,7 +58,6 @@ def main():
vllm.entrypoints.cli.openai,
vllm.entrypoints.cli.serve,
vllm.entrypoints.cli.launch,
vllm.entrypoints.cli.benchmark.main,
vllm.entrypoints.cli.collect_env,
vllm.entrypoints.cli.download_kernels,
vllm.entrypoints.cli.preload,
@@ -70,24 +68,6 @@ def main():
if sys.argv[1:2] != ["snapshot"]:
cli_env_setup()
vllm.entrypoints.cli.benchmark.main.maybe_exec_rust_bench()
# For 'vllm bench *': use CPU instead of UnspecifiedPlatform by default
if len(sys.argv) > 1 and sys.argv[1] == "bench":
logger.debug(
"Bench command detected, must ensure current platform is not "
"UnspecifiedPlatform to avoid device type inference error"
)
from vllm import platforms
if platforms.current_platform.is_unspecified():
from vllm.platforms.cpu import CpuPlatform
platforms.current_platform = CpuPlatform()
logger.info(
"Unspecified platform detected, switching to CPU Platform instead."
)
parser = FlexibleArgumentParser(
description="vLLM CLI",
epilog=VLLM_SUBCMD_PARSER_EPILOG.format(subcmd="[subcommand]"),
+13 -65
View File
@@ -17,7 +17,6 @@ from vllm.entrypoints.launchers.cli_args import (
validate_parsed_serve_args,
)
from vllm.entrypoints.launchers.dp_supervisor import run_dp_supervisor
from vllm.entrypoints.launchers.launcher import create_server_socket
from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
from vllm.logger import init_logger
from vllm.reasoning import ReasoningParserManager
@@ -30,7 +29,6 @@ from vllm.v1.executor.multiproc_executor import MultiprocExecutor
from vllm.v1.metrics.prometheus import setup_multiprocess_prometheus
from vllm.v1.utils import (
APIServerProcessManager,
RustFrontendProcessManager,
wait_for_completion_or_failure,
)
@@ -62,10 +60,6 @@ class ServeSubcommand(CLISubcommand):
uvloop.run(serve_grpc(args))
return
rust_frontend_path = (
envs.VLLM_RUST_FRONTEND_PATH if envs.VLLM_USE_RUST_FRONTEND else None
)
if args.headless:
if args.api_server_count is not None and args.api_server_count > 0:
raise ValueError(
@@ -106,12 +100,11 @@ class ServeSubcommand(CLISubcommand):
# Default api_server_count if not explicitly set.
# - Multi-port: 1 (supervisor spawns one server per local DP rank)
# - Rust frontend: 1 (not applicable as it's multithreaded)
# - External LB: 1 (external LB handles distribution)
# - Hybrid LB: Use local DP size (internal LB for local ranks only)
# - Internal LB: Use full DP size
if args.api_server_count is None:
if is_multi_port or is_external_lb or rust_frontend_path:
if is_multi_port or is_external_lb:
args.api_server_count = 1
elif is_hybrid_lb:
args.api_server_count = args.data_parallel_size_local or 1
@@ -128,12 +121,6 @@ class ServeSubcommand(CLISubcommand):
"Defaulting api_server_count to data_parallel_size (%d).",
args.api_server_count,
)
elif rust_frontend_path and args.api_server_count > 1:
logger.warning(
"Ignoring --api-server-count=%d when using rust front-end process",
args.api_server_count,
)
args.api_server_count = 1
# Elastic EP currently only supports running with at most one API server.
if getattr(args, "enable_elastic_ep", False) and args.api_server_count > 1:
@@ -148,7 +135,7 @@ class ServeSubcommand(CLISubcommand):
run_dp_supervisor(args)
elif args.api_server_count < 1:
run_headless(args)
elif args.api_server_count > 1 or rust_frontend_path:
elif args.api_server_count > 1:
run_multi_api_server(args)
else:
# Single API server (this process).
@@ -268,17 +255,9 @@ def run_headless(args: argparse.Namespace):
def run_multi_api_server(args: argparse.Namespace):
assert not args.headless
rust_frontend_path = (
envs.VLLM_RUST_FRONTEND_PATH if envs.VLLM_USE_RUST_FRONTEND else None
)
num_api_servers: int = args.api_server_count
assert num_api_servers > 0
if rust_frontend_path and num_api_servers > 1:
raise ValueError(
"VLLM_RUST_FRONTEND_PATH does not support api_server_count > 1"
)
if num_api_servers > 1:
setup_multiprocess_prometheus()
@@ -296,13 +275,6 @@ def run_multi_api_server(args: argparse.Namespace):
signal.signal(signal.SIGINT, signal_handler)
listen_address, sock = setup_server(args, reuse_port=num_api_servers > 1)
# `--grpc-port` is only accepted for the Rust frontend, which inherits this
# listener like the HTTP one. gRPC follows the HTTP TCP host, or IPv4
# loopback when HTTP uses a Unix socket.
grpc_sock = None
if args.grpc_port is not None:
grpc_host = "127.0.0.1" if args.uds else (args.host or "")
grpc_sock = create_server_socket((grpc_host, args.grpc_port), reuse_port=False)
engine_args = vllm.AsyncEngineArgs.from_cli_args(args)
propagate_flash_late_interaction(args, engine_args)
@@ -324,9 +296,7 @@ def run_multi_api_server(args: argparse.Namespace):
dp_rank = parallel_config.data_parallel_rank
assert parallel_config.local_engines_only or dp_rank == 0
api_server_manager: APIServerProcessManager | RustFrontendProcessManager | None = (
None
)
api_server_manager: APIServerProcessManager | None = None
from vllm.v1.engine.utils import bind_engine_zmq_listeners
@@ -343,38 +313,16 @@ def run_multi_api_server(args: argparse.Namespace):
coordinator.get_stats_publish_address() if coordinator else None
)
if rust_frontend_path:
if parallel_config.local_engines_only:
expected_engine_start_index = parallel_config.data_parallel_rank
expected_engine_count = parallel_config.data_parallel_size_local
else:
expected_engine_start_index = 0
expected_engine_count = parallel_config.data_parallel_size
# Start rust front-end process.
api_server_manager = RustFrontendProcessManager(
binary_path=rust_frontend_path,
sock=sock,
grpc_sock=grpc_sock,
args=args,
input_listener=zmq_listeners.inputs[0],
output_listener=zmq_listeners.outputs[0],
engine_start_index=expected_engine_start_index,
engine_count=expected_engine_count,
data_parallel_size=parallel_config.data_parallel_size,
stats_update_address=stats_update_address,
)
else:
# Start API server(s).
api_server_manager = APIServerProcessManager(
listen_address=listen_address,
sock=sock,
args=args,
num_servers=num_api_servers,
input_listeners=zmq_listeners.inputs,
output_listeners=zmq_listeners.outputs,
stats_update_address=stats_update_address,
tensor_queue=engine_launch.tensor_queue,
)
api_server_manager = APIServerProcessManager(
listen_address=listen_address,
sock=sock,
args=args,
num_servers=num_api_servers,
input_listeners=zmq_listeners.inputs,
output_listeners=zmq_listeners.outputs,
stats_update_address=stats_update_address,
tensor_queue=engine_launch.tensor_queue,
)
# Set frontend processes to watch during engine startup.
# If any of these processes exit before the engines are up, the engine startup
@@ -18,7 +18,7 @@ from vllm.plugins.endpoint_plugins.interface import init_endpoint_plugins_state
from vllm.renderers.chat_utils import load_chat_template
from vllm.renderers.online_derenderer import OnlineDerenderer
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.tasks import FALLBACK_SUPPORTED_TASKS, POOLING_TASKS, SupportedTask
from vllm.tasks import FALLBACK_SUPPORTED_TASKS, SupportedTask
async def init_app_state(
@@ -152,11 +152,6 @@ async def init_app_state(
engine_client, state, args, request_logger, supported_tasks
)
if any(task in POOLING_TASKS for task in supported_tasks):
from vllm.entrypoints.pooling.factories import init_pooling_state
init_pooling_state(engine_client, state, args, request_logger, supported_tasks)
from vllm.entrypoints.generate.structured_decisions.api_router import (
init_structured_decisions_state,
)
@@ -6,7 +6,7 @@ from fastapi import FastAPI
from vllm import envs
from vllm.config import ModelConfig
from vllm.tasks import POOLING_TASKS, SupportedTask
from vllm.tasks import SupportedTask
def register_api_routers(
@@ -77,11 +77,6 @@ def register_api_routers(
register_speech_to_text_api_routers(app, supported_tasks)
if any(task in POOLING_TASKS for task in supported_tasks):
from vllm.entrypoints.pooling.factories import register_pooling_api_routers
register_pooling_api_routers(app, supported_tasks, model_config)
if getattr(args, "enable_fault_tolerance", False):
from vllm.entrypoints.serve.fault_tolerance.api_router import (
register_fault_tolerance_api_router,
-39
View File
@@ -285,11 +285,6 @@ class FrontendArgs(BaseFrontendArgs):
"""Host name."""
port: int = 8000
"""Port number."""
grpc_port: int | None = None
"""Enable the Rust frontend's additional gRPC Inference and Control services
on this port. Requires `VLLM_USE_RUST_FRONTEND=1 vllm serve`; HTTP remains on
`--port`. Binds on `--host`, or 127.0.0.1 with `--uds`. Cannot be combined
with the Python gRPC server's `--grpc` flag."""
data_parallel_supervisor_port: int = 9256
"""HTTP port for aggregated health endpoints in multi-port external LB
mode."""
@@ -468,38 +463,6 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
return parser
def validate_grpc_port_arg(args: argparse.Namespace) -> None:
"""Validate the Rust frontend's optional gRPC listener."""
if getattr(args, "grpc_port", None) is None:
return
if args.grpc:
raise ValueError(
"--grpc and --grpc-port are mutually exclusive. Use --grpc "
"--port for the Python gRPC server, or VLLM_USE_RUST_FRONTEND=1 "
"with --grpc-port for the Rust frontend."
)
if not envs.VLLM_USE_RUST_FRONTEND or getattr(args, "subparser", None) != "serve":
raise ValueError("--grpc-port requires VLLM_USE_RUST_FRONTEND=1 vllm serve")
if args.headless or (
args.api_server_count is not None and args.api_server_count <= 0
):
raise ValueError(
"--grpc-port requires a Rust frontend; remove --headless "
"and non-positive --api-server-count"
)
if args.data_parallel_multi_port_external_lb:
raise ValueError(
"--grpc-port is incompatible with "
"--data-parallel-multi-port-external-lb: "
"its frontends would share the same gRPC port"
)
if not 0 <= args.grpc_port <= 65535:
raise ValueError("--grpc-port must be between 0 and 65535")
if args.grpc_port != 0 and args.grpc_port == args.port and not args.uds:
raise ValueError("--grpc-port must differ from --port when HTTP uses TCP")
def validate_parsed_serve_args(args: argparse.Namespace):
"""Quick checks for model serve args that raise prior to loading."""
# `vllm launch <component>` builds its parser with make_arg_parser too (see
@@ -518,8 +481,6 @@ def validate_parsed_serve_args(args: argparse.Namespace):
"the checkpoint's output format."
)
validate_grpc_port_arg(args)
# Enable auto tool needs a tool call parser to be valid
if args.enable_auto_tool_choice and not args.tool_call_parser:
raise TypeError("Error: --enable-auto-tool-choice requires --tool-call-parser")
+1 -11
View File
@@ -22,7 +22,6 @@ import uvicorn
import uvloop
from fastapi import FastAPI, Response
import vllm.envs as envs
from vllm.logger import configure_logging_from_args, init_logger
from vllm.utils.system_utils import (
decorate_logs,
@@ -243,12 +242,6 @@ def _run_python_vllm_dp_server(child_args: argparse.Namespace) -> None:
uvloop.run(run_server(child_args))
def _run_rust_vllm_dp_server(child_args: argparse.Namespace) -> None:
from vllm.entrypoints.cli.serve import run_multi_api_server
run_multi_api_server(child_args)
def _run_vllm_dp_server(child_args: argparse.Namespace) -> None:
"""Entrypoint function for the vLLM DP Server."""
configure_logging_from_args(child_args)
@@ -260,10 +253,7 @@ def _run_vllm_dp_server(child_args: argparse.Namespace) -> None:
name = f"APIServer_DP{child_args.data_parallel_rank}"
set_process_title(name)
decorate_logs(name)
if envs.VLLM_USE_RUST_FRONTEND and envs.VLLM_RUST_FRONTEND_PATH:
_run_rust_vllm_dp_server(child_args)
else:
_run_python_vllm_dp_server(child_args)
_run_python_vllm_dp_server(child_args)
class DPSupervisor:
+3 -51
View File
@@ -35,16 +35,6 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
ChatCompletionResponse,
)
from vllm.entrypoints.pooling.embed.protocol import (
EmbeddingRequest,
EmbeddingResponse,
)
from vllm.entrypoints.pooling.scoring.protocol import (
RerankRequest,
RerankResponse,
ScoreRequest,
ScoreResponse,
)
from vllm.entrypoints.serve import create_error_response
from vllm.entrypoints.serve.engine.protocol import (
ErrorInfo,
@@ -147,12 +137,7 @@ class BatchTranslationRequest(TranslationRequest):
BatchRequestInputBody: TypeAlias = (
ChatCompletionRequest
| EmbeddingRequest
| ScoreRequest
| RerankRequest
| BatchTranscriptionRequest
| BatchTranslationRequest
ChatCompletionRequest | BatchTranscriptionRequest | BatchTranslationRequest
)
@@ -184,12 +169,6 @@ class BatchRequestInput(OpenAIBaseModel):
url: str = info.data["url"]
if url == "/v1/chat/completions":
return ChatCompletionRequest.model_validate(value)
if url == "/v1/embeddings":
return TypeAdapter(EmbeddingRequest).validate_python(value)
if url.endswith("/score"):
return TypeAdapter(ScoreRequest).validate_python(value)
if url.endswith("/rerank"):
return RerankRequest.model_validate(value)
if url == "/v1/audio/transcriptions":
return BatchTranscriptionRequest.model_validate(value)
if url == "/v1/audio/translations":
@@ -199,9 +178,6 @@ class BatchRequestInput(OpenAIBaseModel):
AllResponse: TypeAlias = (
ChatCompletionResponse
| EmbeddingResponse
| ScoreResponse
| RerankResponse
| TranscriptionResponseVariant
| TranslationResponse
| TranslationResponseVerbose
@@ -738,8 +714,6 @@ async def build_endpoint_registry(
openai_serving_chat = getattr(state, "openai_serving_chat", None)
openai_serving_transcription = getattr(state, "openai_serving_transcription", None)
openai_serving_translation = getattr(state, "openai_serving_translation", None)
serving_embedding = getattr(state, "serving_embedding", None)
serving_scores = getattr(state, "serving_scores", None)
allowed_media_domains = getattr(args, "allowed_media_domains", None)
@@ -754,27 +728,6 @@ async def build_endpoint_registry(
),
"wrapper_fn": None,
},
"embeddings": {
"url_matcher": lambda url: url == "/v1/embeddings",
"handler_getter": lambda: (
serving_embedding if serving_embedding is not None else None
),
"wrapper_fn": None,
},
"score": {
"url_matcher": lambda url: url.endswith("/score"),
"handler_getter": lambda: (
serving_scores if serving_scores is not None else None
),
"wrapper_fn": None,
},
"rerank": {
"url_matcher": lambda url: url.endswith("/rerank"),
"handler_getter": lambda: (
serving_scores if serving_scores is not None else None
),
"wrapper_fn": None,
},
"transcriptions": {
"url_matcher": lambda url: url == "/v1/audio/transcriptions",
"handler_getter": lambda: (
@@ -859,9 +812,8 @@ async def run_batch(
make_async_error_request_output(
request,
error_msg=f"URL {request.url} was used. "
"Supported endpoints: /v1/chat/completions, /v1/embeddings,"
" /v1/audio/transcriptions, /v1/audio/translations, /score, "
" /rerank.",
"Supported endpoints: /v1/chat/completions,"
" /v1/audio/transcriptions, /v1/audio/translations.",
)
)
+1 -6
View File
@@ -30,7 +30,6 @@ from vllm.config.quantization import QuantizationConfigArgs
from vllm.engine.arg_utils import EngineArgs
from vllm.entrypoints.common.offline import _O, _R, OfflineInferenceMixin
from vllm.entrypoints.generate.beam_search.offline import BeamSearchOfflineMixin
from vllm.entrypoints.pooling.offline import PoolingOfflineMixin
from vllm.entrypoints.rl.offline import RLOfflineMixin
from vllm.entrypoints.serve.utils.api_utils import log_non_default_args
from vllm.inputs import PromptType
@@ -60,9 +59,7 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
class LLM(
BeamSearchOfflineMixin, PoolingOfflineMixin, RLOfflineMixin, OfflineInferenceMixin
):
class LLM(BeamSearchOfflineMixin, RLOfflineMixin, OfflineInferenceMixin):
"""An LLM for generating texts from given prompts and sampling parameters.
This class includes a tokenizer, a language model (possibly distributed
@@ -373,8 +370,6 @@ class LLM(
self.model_config.renderer_num_workers,
)
PoolingOfflineMixin.__init__(self)
# Cache for __repr__ to avoid repeated collective_rpc calls
self._cached_repr: str | None = None
@@ -1,356 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
from concurrent.futures import Executor
from typing import Any, Final, cast
from vllm import (
PoolingParams,
PoolingRequestOutput,
)
from vllm.config import VllmConfig
from vllm.exceptions import VLLMValidationError
from vllm.lora.request import LoRARequest
from vllm.renderers import BaseRenderer, merge_kwargs
from vllm.renderers.chat_utils import (
ChatTemplateConfig,
)
from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq
from vllm.utils.async_utils import make_async
from vllm.utils.mistral import is_mistral_tokenizer
from ..typing import (
AnyOfflineInputsContext,
AnyRenderParam,
EncodeChatRenderParams,
EncodeCMPLRenderParams,
OfflineEncodeInputsContext,
OfflineOutputsContext,
PoolingChatLikeRequest,
PoolingCompletionLikeRequest,
PoolingEngineInput,
PoolingServeContext,
RequestFactory,
RequestGenerator,
ScoringRenderParams,
)
class PoolingIOProcessor:
"""Processor for handling preprocessing & postprocessing ops for pooling requests.
This class manages both online (serving) and offline (batch) processing of pooling
requests, handling chat and completion formats.
"""
name: str
def __init__(
self,
vllm_config: VllmConfig,
renderer: BaseRenderer,
chat_template_config: ChatTemplateConfig,
):
self.vllm_config = vllm_config
self.model_config = vllm_config.model_config
self.renderer = renderer
self.chat_template = chat_template_config.chat_template
self.chat_template_content_format: Final = (
chat_template_config.chat_template_content_format
)
self.trust_request_chat_template = (
chat_template_config.trust_request_chat_template
)
self.template_kwargs = None
self.tool_dicts = None
# Shared thread pool executor for preprocessing
self._executor: Executor = self.renderer._executor
self.render_async = make_async(self.render, executor=self._executor)
#######################################
# online APIs
def create_pooling_params(self, request):
return request.to_pooling_params()
def get_request_factory_online(
self, ctx: PoolingServeContext
) -> Sequence[AnyRenderParam]:
request = ctx.request
renderer = self.renderer
requests: Sequence[AnyRenderParam]
if isinstance(request, PoolingChatLikeRequest):
self._validate_chat_template(
request_chat_template=request.chat_template,
chat_template_kwargs=request.chat_template_kwargs,
trust_request_chat_template=self.trust_request_chat_template,
)
num_requests = 1
default_template_kwargs = merge_kwargs(
self.template_kwargs,
dict(
tools=self.tool_dicts,
tokenize=is_mistral_tokenizer(renderer.tokenizer),
),
)
mm_config = self.model_config.multimodal_config
tok_params = request.build_tok_params(self.model_config)
chat_params = request.build_chat_params(
self.chat_template, self.chat_template_content_format
).with_defaults(
default_template_kwargs,
default_media_io_kwargs=(
mm_config.media_io_kwargs if mm_config else None
),
)
params_seq = self._params_to_seq(ctx.pooling_params, num_requests)
seq_lora_requests = self._lora_request_to_seq(
ctx.lora_request, num_requests
)
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
requests = [
EncodeChatRenderParams(
conversations=request.messages,
chat_params=chat_params,
tok_params=tok_params,
prompt_extras=ctx.prompt_extras,
skip_mm_cache=False,
params=params_seq[i],
lora_requests=seq_lora_requests[i],
priorities=seq_priority[i],
)
for i in range(num_requests)
]
return requests
elif isinstance(request, PoolingCompletionLikeRequest):
model_config = self.model_config
prompts_seq = prompt_to_seq(request.input)
num_requests = len(prompts_seq)
parsed_prompts = [
(
prompt
if isinstance(prompt, bytes)
else parse_model_prompt(model_config, prompt)
)
for prompt in prompts_seq
]
tok_params = request.build_tok_params(model_config)
params_seq = self._params_to_seq(ctx.pooling_params, num_requests)
seq_lora_requests = self._lora_request_to_seq(
ctx.lora_request, num_requests
)
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
requests = [
EncodeCMPLRenderParams(
prompts=parsed_prompts[i],
tok_params=tok_params,
prompt_extras=ctx.prompt_extras,
skip_mm_cache=False,
params=params_seq[i],
lora_requests=seq_lora_requests[i],
priorities=seq_priority[i],
)
for i in range(num_requests)
]
return requests
else:
raise ValueError(f"Invalid {self.name} request type")
def post_process_online(
self,
ctx: PoolingServeContext,
):
pass
#######################################
# offline APIs
def get_request_factory_offline(
self, ctx: AnyOfflineInputsContext
) -> tuple[RequestFactory, int]:
assert isinstance(ctx, OfflineEncodeInputsContext)
prompts_seq = prompt_to_seq(ctx.prompts)
num_requests = len(prompts_seq)
pooling_task = ctx.pooling_task
parsed_prompts = [
(
prompt
if isinstance(prompt, bytes)
else parse_model_prompt(self.model_config, prompt)
)
for prompt in prompts_seq
]
tok_params = self.renderer.default_cmpl_tok_params.with_kwargs(
**(ctx.tokenization_kwargs or {})
)
pooling_params: PoolingParams | Sequence[PoolingParams]
if ctx.pooling_params is None:
pooling_params = PoolingParams()
else:
pooling_params = ctx.pooling_params
# Clone to avoid modifying the caller's pooling parameters.
params_seq = [
param.clone() for param in self._params_to_seq(pooling_params, num_requests)
]
for param in params_seq:
if param.task is None:
param.task = pooling_task
elif pooling_task == "plugin":
# `plugin` task uses io_processor.parse_data to verify inputs.
# We actually allow plugin to overwrite pooling_task.
pass
elif param.task != pooling_task:
msg = f"You cannot overwrite {param.task=!r} with {pooling_task=!r}!"
raise VLLMValidationError(msg)
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
def request_factory() -> RequestGenerator:
for i in range(num_requests):
yield EncodeCMPLRenderParams(
prompts=parsed_prompts[i],
tok_params=tok_params,
prompt_extras=None,
skip_mm_cache=False,
params=params_seq[i],
lora_requests=seq_lora_requests[i],
priorities=seq_priority[i],
)
return request_factory, num_requests
def post_process_offline(
self,
ctx: OfflineOutputsContext,
) -> list[PoolingRequestOutput]:
return ctx.outputs
#######################################
# helpers
def render(
self,
render_params: EncodeCMPLRenderParams
| EncodeChatRenderParams
| ScoringRenderParams,
) -> PoolingEngineInput:
if "conversations" in render_params:
render_params = cast(EncodeChatRenderParams, render_params)
(_,), engine_input = self.renderer.render_chat(
conversations=[render_params["conversations"]],
chat_params=render_params["chat_params"],
tok_params=render_params["tok_params"],
prompt_extras=render_params["prompt_extras"],
skip_mm_cache=render_params["skip_mm_cache"],
)
elif "prompts" in render_params:
render_params = cast(EncodeCMPLRenderParams, render_params)
engine_input = self.renderer.render_cmpl(
prompts=[render_params["prompts"]],
tok_params=render_params["tok_params"],
prompt_extras=render_params["prompt_extras"],
skip_mm_cache=render_params["skip_mm_cache"],
)
else:
raise ValueError(
f"Unsupported render_params type {render_params.__class__.__name__}"
)
return PoolingEngineInput(
prompts=engine_input[0],
params=render_params["params"],
lora_requests=render_params["lora_requests"],
priorities=render_params["priorities"],
)
def _validate_chat_template(
self,
request_chat_template: str | None,
chat_template_kwargs: dict[str, Any] | None,
trust_request_chat_template: bool,
):
if not trust_request_chat_template and (
request_chat_template is not None
or (
chat_template_kwargs
and chat_template_kwargs.get("chat_template") is not None
)
):
raise VLLMValidationError(
"Chat template is passed with request, but "
"--trust-request-chat-template is not set. "
"Refused request with untrusted chat template."
)
return None
def _params_to_seq(
self,
params: PoolingParams | Sequence[PoolingParams],
num_requests: int,
) -> Sequence[PoolingParams]:
if isinstance(params, Sequence):
if len(params) != num_requests:
raise VLLMValidationError(
f"The lengths of prompts ({num_requests}) "
f"and params ({len(params)}) must be the same."
)
return params
return [params] * num_requests
def _lora_request_to_seq(
self,
lora_request: LoRARequest | None | Sequence[LoRARequest | None],
num_requests: int,
) -> Sequence[LoRARequest | None]:
if isinstance(lora_request, Sequence):
if len(lora_request) != num_requests:
raise VLLMValidationError(
f"The lengths of prompts ({num_requests}) "
f"and lora_request ({len(lora_request)}) must be the same."
)
return lora_request
return [lora_request] * num_requests
def _priority_to_seq(
self,
priority: int | Sequence[int] | None,
num_requests: int,
) -> Sequence[int]:
if priority is not None:
if isinstance(priority, int):
return [priority] * num_requests
if len(priority) != num_requests:
raise ValueError(
f"The lengths of prompts ({num_requests}) "
f"and priority ({len(priority)}) must be the same."
)
return priority
return [0] * num_requests
-366
View File
@@ -1,366 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Annotated, Any, Literal
from pydantic import Field, model_validator
from vllm.config import ModelConfig
from vllm.entrypoints.generate.base.protocol import validate_cache_salt
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
from vllm.exceptions import VLLMValidationError
from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
from vllm.renderers.chat_utils import (
ChatCompletionMessageParam,
ChatTemplateContentFormatOption,
)
from vllm.tasks import check_removed_pooling_task
from vllm.utils import random_uuid
from vllm.utils.serial_utils import EmbedDType, EncodingFormat, Endianness
def reject_removed_pooling_parameters(data):
if not isinstance(data, dict):
return data
if "normalize" in data:
raise VLLMValidationError(
"Parameter `normalize` was removed; use `use_activation` instead.",
parameter="normalize",
)
check_removed_pooling_task(data.get("task"))
return data
class PoolingBasicRequestMixin(OpenAIBaseModel):
# --8<-- [start:pooling-common-params]
model: str | None = None
user: str | None = None
# --8<-- [end:pooling-common-params]
# --8<-- [start:pooling-common-extra-params]
truncate_prompt_tokens: Annotated[int, Field(ge=-1)] | None = None
padding: Literal["max_length", "do_not_pad"] | None = Field(
default=None,
description=(
"Whether to pad the prompt, using the same names as the "
"Transformers tokenizer. 'max_length' pads to the maximum input "
"length, 'do_not_pad' leaves the prompt as is. Models trained "
"with a fixed sequence length and no attention mask, such as "
"SigLIP, need 'max_length' to match training, otherwise their "
"embeddings are not comparable."
),
)
truncation_side: Literal["left", "right"] | None = Field(
default=None,
description=(
"Which side to truncate from when truncate_prompt_tokens is active. "
"'right' keeps the first N tokens. "
"'left' keeps the last N tokens."
),
)
request_id: str = Field(
default_factory=random_uuid,
description=(
"The request_id related to this request. If the caller does "
"not set it, a random_uuid will be generated. This id is used "
"through out the inference process and return in response."
),
)
priority: int = Field(
default=0,
ge=-(2**63),
le=2**63 - 1,
description=(
"The priority of the request (lower means earlier handling; "
"default: 0). Any priority other than 0 will raise an error "
"if the served model does not use priority scheduling."
),
)
mm_processor_kwargs: dict[str, Any] | None = Field(
default=None,
description="Additional kwargs to pass to the HF processor.",
)
cache_salt: str | None = Field(
default=None,
min_length=1,
max_length=1024,
description=(
"If specified, the prefix cache will be salted with the provided "
"string to prevent an attacker to guess prompts in multi-user "
"environments. The salt should be random, protected from "
"access by 3rd parties, and long enough to be "
"unpredictable (e.g., 43 characters base64-encoded, corresponding "
"to 256 bit)."
),
)
# --8<-- [end:pooling-common-extra-params]
@model_validator(mode="before")
@classmethod
def check_cache_salt_support(cls, data: Any) -> Any:
if not isinstance(data, dict):
return data
validate_cache_salt(data.get("cache_salt"))
return data
def _build_pooling_tok_params(
self,
model_config: ModelConfig,
*,
add_special_tokens: bool,
max_total_tokens: int | None,
max_output_tokens: int,
max_total_tokens_param: str = "max_model_len",
max_output_tokens_param: str | None = None,
) -> TokenizeParams:
encoder_config = model_config.encoder_config or {}
if max_output_tokens_param is None:
tok_params = TokenizeParams(
max_total_tokens=max_total_tokens,
max_output_tokens=max_output_tokens,
truncate_prompt_tokens=self.truncate_prompt_tokens,
truncation_side=self.truncation_side,
do_lower_case=encoder_config.get("do_lower_case", False),
add_special_tokens=add_special_tokens,
max_total_tokens_param=max_total_tokens_param,
)
else:
tok_params = TokenizeParams(
max_total_tokens=max_total_tokens,
max_output_tokens=max_output_tokens,
truncate_prompt_tokens=self.truncate_prompt_tokens,
truncation_side=self.truncation_side,
do_lower_case=encoder_config.get("do_lower_case", False),
add_special_tokens=add_special_tokens,
max_total_tokens_param=max_total_tokens_param,
max_output_tokens_param=max_output_tokens_param,
)
if self.padding is None:
return tok_params
return tok_params.with_kwargs(padding=self.padding)
class PoolingTokenizeParamsMixin:
add_special_tokens: bool
def _build_pooling_tok_params(
self,
model_config: ModelConfig,
*,
add_special_tokens: bool,
max_total_tokens: int | None,
max_output_tokens: int,
max_total_tokens_param: str = "max_model_len",
max_output_tokens_param: str | None = None,
) -> TokenizeParams:
raise NotImplementedError
class FixedMaxLenTokenizeParamsMixin(PoolingTokenizeParamsMixin):
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
return self._build_pooling_tok_params(
model_config,
add_special_tokens=self.add_special_tokens,
max_total_tokens=model_config.max_model_len,
max_output_tokens=0,
)
class EmbeddingTokenizeParamsMixin(PoolingTokenizeParamsMixin):
padding: Literal["max_length", "do_not_pad"] | None
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
default_max_total_tokens = model_config.max_model_len
max_total_tokens: int | None = default_max_total_tokens
max_output_tokens = 0
pooler_config = model_config.pooler_config
if pooler_config is not None:
if pooler_config.enable_chunked_processing:
max_total_tokens = pooler_config.max_embed_len
else:
max_total_tokens = (
pooler_config.max_embed_len or default_max_total_tokens
)
tok_params = self._build_pooling_tok_params(
model_config,
add_special_tokens=self.add_special_tokens,
max_total_tokens=max_total_tokens,
max_output_tokens=max_output_tokens,
)
if (
self.padding == "max_length"
and pooler_config is not None
and pooler_config.enable_chunked_processing
):
max_padding_len = pooler_config.max_embed_len
if max_padding_len is None:
max_padding_len = default_max_total_tokens
return tok_params.with_kwargs(
pad_prompt_tokens=min(default_max_total_tokens, max_padding_len)
)
return tok_params
class CompletionRequestMixin(OpenAIBaseModel):
# --8<-- [start:completion-params]
input: list[int] | list[list[int]] | str | list[str]
# --8<-- [end:completion-params]
# --8<-- [start:completion-extra-params]
add_special_tokens: bool = Field(
default=True,
description=(
"If true (the default), special tokens (e.g. BOS) will be added to "
"the prompt."
),
)
# --8<-- [end:completion-extra-params]
class ChatRequestOptionsMixin(OpenAIBaseModel):
# --8<-- [start:chat-extra-params]
add_generation_prompt: bool = Field(
default=False,
description=(
"If true, the generation prompt will be added to the chat template. "
"This is a parameter used by chat template in tokenizer config of the "
"model."
),
)
continue_final_message: bool = Field(
default=False,
description=(
"If this is set, the chat will be formatted so that the final "
"message in the chat is open-ended, without any EOS tokens. The "
"model will continue this message rather than starting a new one. "
'This allows you to "prefill" part of the model\'s response for it. '
"Cannot be used at the same time as `add_generation_prompt`."
),
)
add_special_tokens: bool = Field(
default=False,
description=(
"If true, special tokens (e.g. BOS) will be added to the prompt "
"on top of what is added by the chat template. "
"For most models, the chat template takes care of adding the "
"special tokens so this should be set to false (as is the "
"default)."
),
)
chat_template: str | None = Field(
default=None,
description=(
"A Jinja template to use for this conversion. "
"As of transformers v4.44, default chat template is no longer "
"allowed, so you must provide a chat template if the tokenizer "
"does not define one."
),
)
chat_template_kwargs: dict[str, Any] | None = Field(
default=None,
description=(
"Additional keyword args to pass to the template renderer. "
"Will be accessible by the chat template."
),
)
media_io_kwargs: dict[str, dict[str, Any]] | None = Field(
default=None,
description=(
"Additional kwargs to pass to the media IO connectors, "
"keyed by modality. Merged with engine-level media_io_kwargs."
),
)
# --8<-- [end:chat-extra-params]
@model_validator(mode="before")
@classmethod
def check_generation_prompt(cls, data):
if not isinstance(data, dict):
return data
if data.get("continue_final_message") and data.get("add_generation_prompt"):
raise VLLMValidationError(
"Cannot set both `continue_final_message` and "
"`add_generation_prompt` to True.",
)
return data
def build_chat_params(
self,
default_template: str | None,
default_template_content_format: ChatTemplateContentFormatOption,
) -> ChatParams:
return ChatParams(
chat_template=self.chat_template or default_template,
chat_template_content_format=default_template_content_format,
chat_template_kwargs=merge_kwargs(
self.chat_template_kwargs,
dict(
add_generation_prompt=self.add_generation_prompt,
continue_final_message=self.continue_final_message,
),
),
media_io_kwargs=self.media_io_kwargs,
)
class ChatRequestMixin(ChatRequestOptionsMixin):
# --8<-- [start:chat-params]
messages: list[ChatCompletionMessageParam]
# --8<-- [end:chat-params]
class EncodingRequestMixin(OpenAIBaseModel):
# --8<-- [start:encoding-params]
encoding_format: EncodingFormat = "float"
# --8<-- [end:encoding-params]
# --8<-- [start:encoding-extra-params]
embed_dtype: EmbedDType = Field(
default="float32",
description=(
"What dtype to use for encoding. Default to using float32 for base64 "
"encoding to match the OpenAI python client behavior. "
"This parameter will affect base64 and binary_response."
),
)
endianness: Endianness = Field(
default="native",
description=(
"What endianness to use for encoding. Default to using native for "
"base64 encoding to match the OpenAI python client behavior."
"This parameter will affect base64 and binary_response."
),
)
# --8<-- [end:encoding-extra-params]
class EmbedRequestMixin(EncodingRequestMixin):
# --8<-- [start:embed-params]
dimensions: int | None = None
# --8<-- [end:embed-params]
# --8<-- [start:embed-extra-params]
use_activation: bool | None = Field(
default=None,
description="Whether to use activation for the pooler outputs. "
"`None` uses the pooler's default, which is `True` in most cases.",
)
# --8<-- [end:embed-extra-params]
class ClassifyRequestMixin(OpenAIBaseModel):
# --8<-- [start:classify-extra-params]
use_activation: bool | None = Field(
default=None,
description="Whether to use activation for the pooler outputs. "
"`None` uses the pooler's default, which is `True` in most cases.",
)
# --8<-- [end:classify-extra-params]
-280
View File
@@ -1,280 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
from abc import ABC, abstractmethod
from collections.abc import AsyncGenerator
from concurrent.futures import Executor
from http import HTTPStatus
from typing import ClassVar
import torch
from fastapi import Request
from fastapi.responses import Response
from vllm import PoolingRequestOutput, envs
from vllm.config import VllmConfig
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.generate.base.protocol import validate_request_mm_kwargs
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.engine.serving import BaseServing
from vllm.entrypoints.serve.engine.typing import AnyRequest
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.exceptions import GenerationError, RetryableRequestError
from vllm.lora.request import LoRARequest
from vllm.renderers.base import BaseRenderer
from vllm.renderers.chat_utils import ChatTemplateConfig
from vllm.utils.async_utils import make_async, merge_async_iterators
from ...serve.engine.protocol import ErrorResponse
from ..typing import AnyPoolingRequest, PoolingServeContext
from .io_processor import PoolingIOProcessor
class PoolingBaseServing(ABC, BaseServing):
request_id_prefix: ClassVar[str]
def __init__(
self,
engine_client: EngineClient,
models: OpenAIServingModels,
*,
request_logger: RequestLogger | None,
chat_template_config: ChatTemplateConfig,
return_tokens_as_token_ids: bool = False,
log_error_stack: bool = False,
):
super().__init__(
models=models,
model_config=models.model_config,
request_logger=request_logger,
)
self.engine_client = engine_client
self.renderer = engine_client.renderer
self.vllm_config = engine_client.vllm_config
self.max_model_len = self.model_config.max_model_len
self.return_tokens_as_token_ids = return_tokens_as_token_ids
self.log_error_stack = log_error_stack
self.chat_template_config = chat_template_config
self.trust_request_mm_kwargs = chat_template_config.trust_request_mm_kwargs
# Shared thread pool executor for preprocessing and postprocessing.
self._executor: Executor = self.renderer._executor
self._postprocessing_async = make_async(
self._postprocessing, executor=self._executor
)
async def __call__(
self,
request: AnyPoolingRequest,
raw_request: Request | None = None,
) -> Response:
io_processor = self.get_io_processor(request)
ctx = await self._init_ctx(io_processor, request, raw_request)
await self._preprocessing(io_processor, ctx)
await self._prepare_generators(ctx)
await self._collect_batch(ctx)
return await self._postprocessing_async(io_processor, ctx)
@abstractmethod
def get_io_processor(self, request: AnyPoolingRequest) -> PoolingIOProcessor:
raise NotImplementedError
async def _preprocessing(
self, io_processor: PoolingIOProcessor, ctx: PoolingServeContext
):
requests = io_processor.get_request_factory_online(ctx)
if len(requests) == 0:
raise ValueError("You must pass at least one prompt")
ctx.engine_inputs = await asyncio.gather(
*[io_processor.render_async(request) for request in requests]
)
@torch.inference_mode()
def _postprocessing(
self, io_processor: PoolingIOProcessor, ctx: PoolingServeContext
):
io_processor.post_process_online(ctx)
return self._build_response(ctx)
async def _init_ctx(
self,
io_processor: PoolingIOProcessor,
request: AnyPoolingRequest,
raw_request: Request | None = None,
):
validate_request_mm_kwargs(
mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),
media_io_kwargs=getattr(request, "media_io_kwargs", None),
trust_request_mm_kwargs=self.trust_request_mm_kwargs,
)
base_request_id = self._base_request_id(
raw_request, getattr(request, "request_id", None)
)
request_id = f"{self.request_id_prefix}-{base_request_id}"
await self._check_model(request)
pooling_params = io_processor.create_pooling_params(request)
lora_request = self._maybe_get_adapters(request)
model_name = self.models.model_name(lora_request)
priorities = getattr(request, "priority", 0)
prompt_extras = {
k: v
for k in ("mm_processor_kwargs", "cache_salt", "chat_template_kwargs")
if (v := getattr(request, k, None)) is not None
}
ctx = PoolingServeContext(
request=request,
raw_request=raw_request,
model_name=model_name,
pooling_params=pooling_params,
request_id=request_id,
lora_request=lora_request,
priorities=priorities,
prompt_extras=prompt_extras,
)
self._validate_request(ctx)
return ctx
async def _prepare_generators(
self,
ctx: PoolingServeContext,
):
if ctx.engine_inputs is None:
raise ValueError("Engine inputs not available")
generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
trace_headers = (
None
if ctx.raw_request is None
else await self._get_trace_headers(ctx.raw_request.headers)
)
assert ctx.pooling_params is not None
pooling_params = ctx.pooling_params
pooling_params.verify(self.model_config)
for i, engine_input in enumerate(ctx.engine_inputs):
prompt_request_id = (
f"{ctx.request_id}-{i}"
if ctx.prompt_request_ids is None
else ctx.prompt_request_ids[i]
)
self._log_inputs(
prompt_request_id,
engine_input["prompts"],
params=engine_input["params"],
lora_request=ctx.lora_request,
)
generator = self.engine_client.encode(
request_id=prompt_request_id,
prompt=engine_input["prompts"],
pooling_params=engine_input["params"],
lora_request=engine_input["lora_requests"],
priority=engine_input["priorities"],
trace_headers=trace_headers,
)
generators.append(generator)
ctx.result_generator = merge_async_iterators(*generators)
async def _collect_batch(
self,
ctx: PoolingServeContext,
):
if ctx.engine_inputs is None:
raise ValueError("Engine prompts not available")
if ctx.result_generator is None:
raise ValueError("Result generator not available")
num_inputs = len(ctx.engine_inputs)
final_res_batch: list[PoolingRequestOutput | None]
final_res_batch = [None] * num_inputs
async for i, res in ctx.result_generator:
if res.error is not None:
if res.error.retryable:
raise RetryableRequestError(res.error.message)
raise GenerationError(res.error.message)
final_res_batch[i] = res
if None in final_res_batch:
raise ValueError("Failed to generate results for all prompts")
ctx.final_res_batch = [res for res in final_res_batch if res is not None]
@abstractmethod
def _build_response(
self,
ctx: PoolingServeContext,
) -> Response:
raise NotImplementedError
async def _check_model(
self,
request: AnyRequest | AnyPoolingRequest,
) -> ErrorResponse | None:
if self._is_model_supported(request.model):
return None
if request.model in self.models.lora_requests:
return None
if (
envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING
and request.model
and (load_result := await self.models.resolve_lora(request.model))
):
if isinstance(load_result, LoRARequest):
return None
if (
isinstance(load_result, ErrorResponse)
and load_result.error.code == HTTPStatus.BAD_REQUEST.value
):
raise ValueError(load_result.error.message)
return None
def _validate_request(self, ctx: PoolingServeContext) -> None:
truncate_prompt_tokens = getattr(ctx.request, "truncate_prompt_tokens", None)
if (
truncate_prompt_tokens is not None
and truncate_prompt_tokens > self.max_model_len
):
raise ValueError(
"truncate_prompt_tokens value is "
"greater than max_model_len."
" Please request a smaller truncation size."
)
return None
class PoolingServing(PoolingBaseServing, ABC):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.io_processor = self.init_io_processor(
vllm_config=self.vllm_config,
renderer=self.renderer,
chat_template_config=self.chat_template_config,
)
@abstractmethod
def init_io_processor(
self,
vllm_config: VllmConfig,
renderer: BaseRenderer,
chat_template_config: ChatTemplateConfig,
) -> PoolingIOProcessor:
raise NotImplementedError
def get_io_processor(self, request: AnyPoolingRequest) -> PoolingIOProcessor:
return self.io_processor
@@ -1,33 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from fastapi import APIRouter, Depends, Request
from fastapi.responses import Response
from vllm.entrypoints.serve.utils.api_utils import (
load_aware_call,
validate_json_request,
with_cancellation,
)
from .protocol import ClassificationRequest
from .serving import ServingClassification
router = APIRouter()
def classify(request: Request) -> ServingClassification:
handler = getattr(request.app.state, "serving_classification", None)
if handler is None:
raise NotImplementedError("The model does not support Classification API")
return handler
@router.post("/classify", dependencies=[Depends(validate_json_request)])
@with_cancellation
@load_aware_call
async def create_classify(
request: ClassificationRequest, raw_request: Request
) -> Response:
handler = classify(raw_request)
return await handler(request, raw_request)
@@ -1,12 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from ..base.io_processor import PoolingIOProcessor
class ClassifyIOProcessor(PoolingIOProcessor):
name = "classify"
class TokenClassifyIOProcessor(PoolingIOProcessor):
name = "token_classify"
@@ -1,71 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from typing import Annotated, TypeAlias
from pydantic import BeforeValidator, Field
from vllm import PoolingParams
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
from vllm.logger import init_logger
from vllm.utils import random_uuid
from ..base.protocol import (
ChatRequestMixin,
ClassifyRequestMixin,
CompletionRequestMixin,
FixedMaxLenTokenizeParamsMixin,
PoolingBasicRequestMixin,
reject_removed_pooling_parameters,
)
logger = init_logger(__name__)
class ClassificationCompletionRequest(
PoolingBasicRequestMixin,
CompletionRequestMixin,
ClassifyRequestMixin,
FixedMaxLenTokenizeParamsMixin,
):
def to_pooling_params(self):
return PoolingParams(
task="classify",
use_activation=self.use_activation,
)
class ClassificationChatRequest(
PoolingBasicRequestMixin,
ChatRequestMixin,
ClassifyRequestMixin,
FixedMaxLenTokenizeParamsMixin,
):
def to_pooling_params(self):
return PoolingParams(
task="classify",
use_activation=self.use_activation,
)
ClassificationRequest: TypeAlias = Annotated[
ClassificationCompletionRequest | ClassificationChatRequest,
BeforeValidator(reject_removed_pooling_parameters),
]
class ClassificationData(OpenAIBaseModel):
index: int
label: str | None
probs: list[float]
num_classes: int
class ClassificationResponse(OpenAIBaseModel):
id: str = Field(default_factory=lambda: f"classify-{random_uuid()}")
object: str = "list"
created: int = Field(default_factory=lambda: int(time.time()))
model: str
data: list[ClassificationData]
usage: UsageInfo
@@ -1,72 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TypeAlias
import numpy as np
from fastapi.responses import JSONResponse
from vllm.entrypoints.serve.engine.protocol import UsageInfo
from vllm.logger import init_logger
from vllm.outputs import ClassificationOutput
from ..base.serving import PoolingServing
from ..typing import PoolingServeContext
from .io_processor import ClassifyIOProcessor
from .protocol import (
ClassificationData,
ClassificationRequest,
ClassificationResponse,
)
logger = init_logger(__name__)
ClassificationServeContext: TypeAlias = PoolingServeContext[ClassificationRequest]
class ServingClassification(PoolingServing):
request_id_prefix = "classify"
def init_io_processor(self, *args, **kwargs) -> ClassifyIOProcessor:
return ClassifyIOProcessor(*args, **kwargs)
def _build_response(
self,
ctx: ClassificationServeContext,
) -> JSONResponse:
id2label = getattr(self.model_config.hf_config, "id2label", {})
num_prompt_tokens = 0
items: list[ClassificationData] = []
for idx, final_res in enumerate(ctx.final_res_batch):
classify_res = ClassificationOutput.from_base(final_res.outputs)
probs = classify_res.probs
predicted_index = int(np.argmax(probs))
label = id2label.get(predicted_index)
item = ClassificationData(
index=idx,
label=label,
probs=probs,
num_classes=len(probs),
)
items.append(item)
prompt_token_ids = final_res.prompt_token_ids
num_prompt_tokens += len(prompt_token_ids)
usage = UsageInfo(
prompt_tokens=num_prompt_tokens,
total_tokens=num_prompt_tokens,
)
response = ClassificationResponse(
id=ctx.request_id,
created=ctx.created_time,
model=ctx.model_name,
data=items,
usage=usage,
)
return JSONResponse(content=response.model_dump())
@@ -1,61 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from http import HTTPStatus
from fastapi import APIRouter, Depends, Request
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.utils.api_utils import (
load_aware_call,
validate_json_request,
with_cancellation,
)
from .protocol import CohereEmbedRequest, EmbeddingRequest
from .serving import ServingEmbedding
router = APIRouter()
def embedding(request: Request) -> ServingEmbedding:
handler = getattr(request.app.state, "serving_embedding", None)
if handler is None:
raise NotImplementedError("The model does not support Embeddings API")
return handler
@router.post(
"/v1/embeddings",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
@with_cancellation
@load_aware_call
async def create_embedding(
request: EmbeddingRequest,
raw_request: Request,
):
handler = embedding(raw_request)
return await handler(request, raw_request)
@router.post(
"/v2/embed",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
@with_cancellation
@load_aware_call
async def create_cohere_embedding(
request: CohereEmbedRequest,
raw_request: Request,
):
handler = embedding(raw_request)
return await handler(request, raw_request)
@@ -1,777 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
from dataclasses import dataclass
from typing import Any, Literal, cast
import torch
from openai.types.chat import (
ChatCompletionContentPartImageParam,
ChatCompletionContentPartTextParam,
)
from vllm import PoolingParams
from vllm.exceptions import VLLMValidationError
from vllm.inputs import tokens_input
from vllm.logger import init_logger
from vllm.outputs import PoolingOutput, PoolingRequestOutput
from vllm.renderers import merge_kwargs
from vllm.renderers.chat_utils import (
ChatCompletionContentPartParam,
ChatCompletionMessageParam,
CustomChatCompletionMessageParam,
ImageURL,
)
from vllm.renderers.hf import resolve_chat_template
from vllm.utils.collection_utils import chunk_list
from vllm.utils.mistral import is_mistral_tokenizer
from ..base.io_processor import PoolingIOProcessor
from ..late_chunking import (
attach_late_chunking_metadata,
build_late_chunking_metadata,
prepare_late_chunking_input,
)
from ..scoring.io_processor import JinaRankingIOProcessorMixin
from ..typing import (
AnyOfflineInputsContext,
AnyRenderParam,
ChunkedEmbeddingMetadata,
EncodeChatRenderParams,
LateChunkingSlot,
OfflineEncodeInputsContext,
OfflineOutputsContext,
PoolingChatLikeRequest,
PoolingCompletionLikeRequest,
PoolingEngineInput,
PoolingServeContext,
RequestFactory,
RequestGenerator,
)
from .protocol import (
CohereEmbedContent,
CohereEmbedInput,
CohereEmbedRequest,
EmbeddingBatchChatInputRequest,
EmbeddingBatchChatRequest,
EmbeddingChatInputRequest,
EmbeddingChatRequest,
EmbeddingCompletionRequest,
)
logger = init_logger(__name__)
@dataclass
class _ChunkedPromptAggregator:
weighted_sum: torch.Tensor | None = None
total_weight: int = 0
num_chunks: int = 0
all_chunks_l2_normalized: bool = True
class EmbedIOProcessor(PoolingIOProcessor):
name = "embed"
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
assert self.model_config.pooler_config is not None
self.pooler_config = self.model_config.pooler_config
self.enable_chunked_processing = self.pooler_config.enable_chunked_processing
# Load task instructions from HF config or sentence-transformers config
self.task_instructions: dict[str, str] | None = self._load_task_instructions(
self.model_config.hf_config
) or self._load_st_prompts(self.model_config.model, self.model_config.revision)
if self.task_instructions:
logger.info(
"Loaded prompt prefixes for input_type: %s",
list(self.task_instructions.keys()),
)
def get_request_factory_online(
self, ctx: PoolingServeContext
) -> Sequence[AnyRenderParam]:
if isinstance(ctx.request, CohereEmbedRequest):
requests = self._get_request_factory_cohere_online(ctx)
elif isinstance(
ctx.request,
(
EmbeddingChatRequest,
EmbeddingBatchChatRequest,
EmbeddingChatInputRequest,
EmbeddingBatchChatInputRequest,
),
):
requests = self._get_request_factory_chat_input_online(ctx)
else:
requests = super().get_request_factory_online(ctx)
return requests
def post_process_online(
self,
ctx: PoolingServeContext,
):
if ctx.final_res_batch is None:
raise ValueError("Final response batch not available")
if not self.enable_chunked_processing:
self._enforce_cohere_max_tokens(ctx)
return super().post_process_online(ctx)
self._post_process_chunked(ctx)
self._enforce_cohere_max_tokens(ctx)
#################################################################
# Long Text Embedding with Chunked Processing
# PTAL: examples/pooling/embed/openai_embedding_long_text
#################################################################
def maybe_pre_process_chunked(self, ctx: PoolingServeContext) -> None:
if not self.enable_chunked_processing:
return None
if ctx.engine_inputs is None:
raise ValueError("Engine prompts not available")
ctx.original_engine_inputs = ctx.engine_inputs
request_id = ctx.request_id
max_model_len = self.model_config.max_model_len
chunked_engine_inputs: list[PoolingEngineInput] = []
prompt_request_ids: list[str] = []
chunked_embedding_metadata: list[ChunkedEmbeddingMetadata] = []
for prompt_idx, engine_input in enumerate(ctx.engine_inputs):
token_ids = engine_input["prompts"].get("prompt_token_ids", None)
if token_ids is None:
raise NotImplementedError(
"Long Text Embedding with Chunked Processing does "
"not support EmbedsPrompt and EncoderDecoderInput."
)
prompt_token_ids = cast(list[int], token_ids)
cache_salt = cast(str | None, engine_input["prompts"].get("cache_salt"))
for chunk_idx, chunk_tokens in enumerate(
chunk_list(prompt_token_ids, max_model_len)
):
chunked_engine_inputs.append(
PoolingEngineInput(
prompts=tokens_input(
prompt_token_ids=chunk_tokens, cache_salt=cache_salt
),
params=engine_input["params"],
lora_requests=engine_input["lora_requests"],
priorities=engine_input["priorities"],
)
)
prompt_request_ids.append(
f"{request_id}-prompt-{prompt_idx}-chunk-{chunk_idx}"
)
chunked_embedding_metadata.append(
ChunkedEmbeddingMetadata(
prompt_index=prompt_idx,
chunk_index=chunk_idx,
)
)
ctx.engine_inputs = chunked_engine_inputs
ctx.prompt_request_ids = prompt_request_ids
ctx.chunked_embedding_metadata = chunked_embedding_metadata
return None
def _post_process_chunked(self, ctx: PoolingServeContext) -> None:
# Online aggregation for chunked requests to
# minimize memory usage
# Track aggregation state for each prompt
if ctx.chunked_embedding_metadata is None:
raise ValueError("Chunked embedding metadata not available")
if len(ctx.chunked_embedding_metadata) != len(ctx.final_res_batch):
raise ValueError(
"Chunked embedding metadata count does not match result count"
)
prompt_aggregators: dict[int, _ChunkedPromptAggregator] = {}
for result, chunk_metadata in zip(
ctx.final_res_batch, ctx.chunked_embedding_metadata
):
prompt_idx = chunk_metadata.prompt_index
aggregator = prompt_aggregators.setdefault(
prompt_idx, _ChunkedPromptAggregator()
)
# MEAN pooling with online weighted averaging
# Ensure result is PoolingRequestOutput
# for embedding processing
if not isinstance(result, PoolingRequestOutput):
raise ValueError(
f"Expected PoolingRequestOutput for "
f"chunked embedding, got "
f"{type(result).__name__}"
)
if result.prompt_token_ids is None:
raise ValueError(
"prompt_token_ids cannot be None for chunked processing"
)
weight = len(result.prompt_token_ids)
embedding_data = result.outputs.data
float_embedding = embedding_data.to(dtype=torch.float32)
if ctx.pooling_params.use_activation:
# Custom activations need not L2-normalize; allow BF16 rounding.
embedding_norm = torch.linalg.vector_norm(float_embedding)
aggregator.all_chunks_l2_normalized &= torch.allclose(
embedding_norm,
torch.ones_like(embedding_norm),
rtol=1e-2,
atol=1e-5,
)
weighted_embedding = float_embedding * weight
if aggregator.weighted_sum is None:
# First chunk
aggregator.weighted_sum = weighted_embedding
else:
# Accumulate
aggregator.weighted_sum += weighted_embedding
aggregator.total_weight += weight
aggregator.num_chunks += 1
if ctx.original_engine_inputs is None:
raise ValueError("Original engine inputs not available")
original_engine_inputs = ctx.original_engine_inputs
num_prompts = len(original_engine_inputs)
# Finalize aggregated results
final_res_batch: list[PoolingRequestOutput] = []
for prompt_idx in range(num_prompts):
if prompt_idx in prompt_aggregators:
# Finalize MEAN aggregation for this chunked prompt
aggregator = prompt_aggregators[prompt_idx]
weighted_sum = aggregator.weighted_sum
total_weight = aggregator.total_weight
if (
weighted_sum is not None
and isinstance(weighted_sum, torch.Tensor)
and isinstance(total_weight, (int, float))
and total_weight > 0
):
# Compute final mean embedding
final_embedding = weighted_sum / total_weight
if (
ctx.pooling_params.use_activation
and aggregator.num_chunks > 1
and aggregator.all_chunks_l2_normalized
):
final_embedding = torch.nn.functional.normalize(
final_embedding, dim=-1
)
# Create a PoolingRequestOutput
# for the aggregated result
pooling_output_data = PoolingOutput(data=final_embedding)
# Get original prompt token IDs for this prompt
original_prompt = original_engine_inputs[prompt_idx]
token_ids = original_prompt["prompts"].get("prompt_token_ids", None)
if token_ids is None:
raise NotImplementedError(
"Long Text Embedding with Chunked Processing does "
"not support EmbedsPrompt and EncoderDecoderInput."
)
original_token_ids = cast(list[int], token_ids)
pooling_request_output = PoolingRequestOutput(
request_id=f"{ctx.request_id}-prompt-{prompt_idx}",
prompt_token_ids=original_token_ids,
outputs=pooling_output_data,
num_cached_tokens=0,
finished=True,
)
final_res_batch.append(pooling_request_output)
else:
raise ValueError(
f"Failed to aggregate chunks for prompt {prompt_idx}"
)
else:
raise ValueError(f"Result not found for prompt {prompt_idx}")
ctx.final_res_batch = final_res_batch
return None
#################################################################
# Chat input Request Preprocessing & Postprocessing
#################################################################
def _get_request_factory_chat_input_online(
self,
ctx: PoolingServeContext,
) -> Sequence[AnyRenderParam]:
request = ctx.request
renderer = self.renderer
self._validate_chat_template(
request_chat_template=request.chat_template,
chat_template_kwargs=request.chat_template_kwargs,
trust_request_chat_template=self.trust_request_chat_template,
)
if isinstance(
request, (EmbeddingBatchChatRequest, EmbeddingBatchChatInputRequest)
):
all_messages = request.messages
else:
all_messages = [request.messages]
num_requests = len(all_messages)
default_template_kwargs = merge_kwargs(
self.template_kwargs,
dict(
tools=self.tool_dicts,
tokenize=is_mistral_tokenizer(renderer.tokenizer),
),
)
mm_config = self.model_config.multimodal_config
tok_params = request.build_tok_params(self.model_config)
chat_params = request.build_chat_params(
self.chat_template, self.chat_template_content_format
).with_defaults(
default_template_kwargs,
default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
)
params_seq = self._params_to_seq(ctx.pooling_params, num_requests)
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
requests = [
EncodeChatRenderParams(
conversations=all_messages[i],
chat_params=chat_params,
tok_params=tok_params,
prompt_extras=ctx.prompt_extras,
skip_mm_cache=False,
params=params_seq[i],
lora_requests=seq_lora_requests[i],
priorities=seq_priority[i],
)
for i in range(num_requests)
]
return requests
#################################################################
# Cohere Request Preprocessing & Postprocessing
#################################################################
@staticmethod
def _load_task_instructions(hf_config: Any) -> dict[str, str] | None:
"""Extract ``task_instructions`` from the HF model config."""
ti = getattr(hf_config, "task_instructions", None)
if not isinstance(ti, dict) or not ti:
return None
return {k: v for k, v in ti.items() if isinstance(v, str)}
@staticmethod
def _load_st_prompts(
model: str | Any,
revision: str | None,
) -> dict[str, str] | None:
"""Load ``task_instructions`` from ``config_sentence_transformers.json``."""
from vllm.transformers_utils.repo_utils import get_hf_file_to_dict
try:
cfg = get_hf_file_to_dict(
"config_sentence_transformers.json", str(model), revision
)
except (ValueError, OSError):
return None
if cfg is None:
return None
prompts = cfg.get("prompts")
if not isinstance(prompts, dict) or not prompts:
return None
return {k: v for k, v in prompts.items() if isinstance(v, str)}
@staticmethod
def _mixed_input_to_messages(
inp: CohereEmbedInput,
*,
task_prefix: str | None = None,
) -> list[ChatCompletionMessageParam]:
"""Build chat messages from a mixed text+image input.
When *task_prefix* is given, it is used as the system prompt.
"""
messages: list[ChatCompletionMessageParam] = []
if task_prefix is not None:
messages.append(
CustomChatCompletionMessageParam(
role="system",
content=[
ChatCompletionContentPartTextParam(
type="text", text=task_prefix
)
],
)
)
parts: list[ChatCompletionContentPartParam] = []
for item in inp.content:
if item.type == "text" and item.text is not None:
parts.append(
ChatCompletionContentPartTextParam(type="text", text=item.text)
)
elif item.type == "image_url" and item.image_url is not None:
parts.append(
ChatCompletionContentPartImageParam(
type="image_url",
image_url=ImageURL(url=item.image_url["url"]),
)
)
messages.append(CustomChatCompletionMessageParam(role="user", content=parts))
return messages
@staticmethod
def _check_cohere_max_tokens(
outputs: list[PoolingRequestOutput],
max_tokens_check: int | None,
) -> None:
"""Raise if any output exceeds *max_tokens_check* tokens.
Used to enforce ``truncate=NONE`` with an explicit ``max_tokens``:
the pipeline runs without truncation and we reject afterwards.
"""
if max_tokens_check is None:
return
for out in outputs:
n = len(out.prompt_token_ids)
if n > max_tokens_check:
raise ValueError(
f"Input of {n} tokens exceeds max_tokens={max_tokens_check} "
"with truncate=NONE. Set truncate to END or START to "
"allow truncation."
)
@staticmethod
def _resolve_cohere_truncation(
request: CohereEmbedRequest,
) -> tuple[int | None, Literal["left", "right"] | None]:
"""Return ``(truncate_prompt_tokens, truncation_side)``."""
if request.truncate == "NONE":
return None, None
if request.truncate == "START":
tokens = request.max_tokens if request.max_tokens is not None else -1
return tokens, "left"
if request.max_tokens is not None:
return request.max_tokens, None
return -1, None
def create_pooling_params(self, request):
if isinstance(request, CohereEmbedRequest):
return PoolingParams(
task="embed",
dimensions=request.output_dimension,
)
return super().create_pooling_params(request)
def _get_request_factory_cohere_online(
self, ctx: PoolingServeContext
) -> Sequence[AnyRenderParam]:
"""Convert a ``CohereEmbedRequest`` into engine prompts.
If a model has a chat template the task instruction are rendered
as a system prompt. Otherwise they are just prepended to the input text.
Images and mixed inputs are always batch-rendered through the chat
template in one ``render_chat`` call.
"""
request = ctx.request
assert isinstance(request, CohereEmbedRequest)
if request.texts is None and request.images is None and request.inputs is None:
raise ValueError("One of texts, images, or inputs must be provided")
truncate_prompt_tokens, truncation_side = self._resolve_cohere_truncation(
request
)
input_type = request.input_type
self._validate_input_type(input_type)
if request.images is not None:
input: list[CohereEmbedInput] = [
CohereEmbedInput(
content=[
CohereEmbedContent(type="image_url", image_url={"url": uri})
]
)
for uri in request.images
]
elif request.inputs is not None:
input = request.inputs
else:
texts = request.texts or []
task_prefix = self._get_task_instruction_prefix(input_type)
if task_prefix is None:
return self._get_request_factory_cohere_text_completion(
ctx,
texts,
truncate_prompt_tokens,
truncation_side,
)
all_messages = [
self._mixed_input_to_messages(
CohereEmbedInput(
content=[CohereEmbedContent(type="text", text=text)]
),
task_prefix=task_prefix,
)
for text in texts
]
if self._has_chat_template():
return self._batch_render_chat(
ctx,
all_messages,
truncate_prompt_tokens,
truncation_side,
)
else:
return self._get_request_factory_cohere_text_completion(
ctx,
self._apply_task_instruction(texts, input_type),
truncate_prompt_tokens,
truncation_side,
)
task_prefix = self._get_task_instruction_prefix(input_type)
all_messages = [
self._mixed_input_to_messages(inp, task_prefix=task_prefix) for inp in input
]
return self._batch_render_chat(
ctx,
all_messages,
truncate_prompt_tokens,
truncation_side,
)
def _has_chat_template(self) -> bool:
return (
resolve_chat_template(
self.renderer.tokenizer,
chat_template=self.chat_template,
tools=None,
model_config=self.model_config,
)
is not None
)
def _get_request_factory_cohere_text_completion(
self,
ctx: PoolingServeContext,
texts: list[str],
truncate_prompt_tokens: int | None,
truncation_side: Literal["left", "right"] | None,
) -> Sequence[AnyRenderParam]:
request = ctx.request
proxy = EmbeddingCompletionRequest(
model=request.model,
input=texts,
dimensions=request.output_dimension,
encoding_format="float",
truncate_prompt_tokens=truncate_prompt_tokens,
truncation_side=truncation_side,
)
ctx.request = proxy
requests = super().get_request_factory_online(ctx)
ctx.request = request
return requests
def _batch_render_chat(
self,
ctx: PoolingServeContext,
all_messages: Sequence[list[ChatCompletionMessageParam]],
truncate_prompt_tokens: int | None,
truncation_side: Literal["left", "right"] | None,
) -> Sequence[AnyRenderParam]:
"""Batch-render multiple conversations through the chat template."""
request = ctx.request
proxy = EmbeddingBatchChatRequest(
model=request.model,
messages=all_messages,
dimensions=request.output_dimension,
encoding_format="float",
truncate_prompt_tokens=truncate_prompt_tokens,
truncation_side=truncation_side,
)
ctx.request = proxy
requests = self._get_request_factory_chat_input_online(ctx)
ctx.request = request
return requests
def _validate_input_type(self, input_type: str | None) -> None:
"""Raise if *input_type* is not supported by this model."""
if input_type is None:
return
if self.task_instructions is None:
raise ValueError(
f"Unsupported input_type {input_type!r}. "
"This model does not define any input_type task instructions."
)
if input_type not in self.task_instructions:
supported = ", ".join(sorted(self.task_instructions))
raise ValueError(
f"Unsupported input_type {input_type!r}. Supported values: {supported}"
)
def _apply_task_instruction(
self,
texts: list[str],
input_type: str | None,
) -> list[str]:
"""Prepend the task-instruction prefix for *input_type*.
Returns *texts* unchanged when no matching prefix is configured.
"""
prefix = self._get_task_instruction_prefix(input_type)
if not prefix:
return texts
return [prefix + t for t in texts]
def _get_task_instruction_prefix(self, input_type: str | None) -> str | None:
"""Return the task-instruction prefix for *input_type*, or ``None``."""
if not self.task_instructions or input_type is None:
return None
return self.task_instructions.get(input_type) or None
def _enforce_cohere_max_tokens(self, ctx: PoolingServeContext) -> None:
if isinstance(ctx.request, CohereEmbedRequest):
request = ctx.request
if request.truncate == "NONE" and request.max_tokens is not None:
self._check_cohere_max_tokens(ctx.final_res_batch, request.max_tokens)
class TokenEmbedIOProcessor(PoolingIOProcessor):
name = "token_embed"
def get_request_factory_offline(
self, ctx: AnyOfflineInputsContext
) -> tuple[RequestFactory, int]:
factory, num_requests = super().get_request_factory_offline(ctx)
assert isinstance(ctx, OfflineEncodeInputsContext)
if ctx.pooling_params is None or not any(
p.late_chunking_params is not None
for p in self._params_to_seq(ctx.pooling_params, num_requests)
):
return factory, num_requests
if (ctx.tokenization_kwargs or {}).get("padding") not in (
None,
False,
"do_not_pad",
):
raise VLLMValidationError("Late chunking does not support input padding")
def request_factory() -> RequestGenerator:
for request in factory():
slot = None
if request["params"].late_chunking_params is not None:
slot = LateChunkingSlot()
request["late_chunking"] = slot
ctx.late_chunking.append(slot)
yield request
return request_factory, num_requests
def render(self, render_params: AnyRenderParam) -> PoolingEngineInput:
late_chunking = render_params["params"].late_chunking_params
if late_chunking is None:
return super().render(render_params)
text, tok_params = prepare_late_chunking_input(self.vllm_config, render_params)
render_params = render_params.copy()
render_params["tok_params"] = tok_params
result = super().render(render_params)
prompt = result["prompts"]
if prompt["type"] != "token":
raise VLLMValidationError("Late chunking requires tokenized text")
render_params["late_chunking"].metadata = build_late_chunking_metadata(
text,
len(prompt["prompt_token_ids"]),
prompt.pop("prompt_token_offsets", None),
late_chunking.chunk_size,
)
return result
def post_process_offline(self, ctx: OfflineOutputsContext):
if ctx.late_chunking:
for output, slot in zip(ctx.outputs, ctx.late_chunking, strict=True):
if slot is not None:
assert slot.metadata is not None
attach_late_chunking_metadata(output, slot.metadata)
return ctx.outputs
class JinaRankingTokenEmbedIOProcessor(
TokenEmbedIOProcessor, JinaRankingIOProcessorMixin
):
def render(self, render_params: AnyRenderParam) -> PoolingEngineInput:
if render_params["params"].late_chunking_params is not None:
raise VLLMValidationError(
"JinaForRanking returns document/query vectors and does not support "
"late chunking"
)
return super().render(render_params)
def get_request_factory_online(
self, ctx: PoolingServeContext
) -> Sequence[AnyRenderParam]:
request = ctx.request
if isinstance(request, PoolingCompletionLikeRequest):
prompts = request.input
if not isinstance(prompts, Sequence) or len(prompts) < 2:
raise ValueError("The JinaForRanking model requires at least 2 inputs.")
text_prompts = self.ensure_str(prompts)
# The JinaForRanking model concatenates docs first, then query.
# Let's stay consistent with this novel design.
prompt_input = self.format_docs_prompts_func(
query=text_prompts[-1], docs=text_prompts[:-1]
)
request.input = prompt_input
return super().get_request_factory_online(ctx)
elif isinstance(request, PoolingChatLikeRequest):
raise ValueError("The JinaForRanking does not support chat Request.")
else:
raise ValueError(f"Invalid {self.name} request type")
def get_request_factory_offline(
self, ctx: AnyOfflineInputsContext
) -> tuple[RequestFactory, int]:
assert isinstance(ctx, OfflineEncodeInputsContext)
if not isinstance(ctx.prompts, Sequence) or len(ctx.prompts) < 2:
raise ValueError("The JinaForRanking model requires at least 2 inputs.")
text_prompts = self.ensure_str(ctx.prompts)
# The JinaForRanking model concatenates docs first, then query.
# Let's stay consistent with this novel design.
ctx.prompts = self.format_docs_prompts_func(
query=text_prompts[-1], docs=text_prompts[:-1]
)
return super().get_request_factory_offline(ctx)
-354
View File
@@ -1,354 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Embedding API protocol models for OpenAI and Cohere formats.
OpenAI: https://platform.openai.com/docs/api-reference/embeddings
Cohere: https://docs.cohere.com/reference/embed
"""
import builtins
import struct
import time
from collections.abc import Sequence
from typing import Annotated, Any, Literal, TypeAlias
import numpy as np
import pybase64 as base64
from pydantic import BaseModel, BeforeValidator, Field, model_validator
from vllm import PoolingParams
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
from vllm.renderers.chat_utils import ChatCompletionMessageParam
from vllm.utils import random_uuid
from ..base.protocol import (
ChatRequestMixin,
ChatRequestOptionsMixin,
CompletionRequestMixin,
EmbeddingTokenizeParamsMixin,
EmbedRequestMixin,
PoolingBasicRequestMixin,
reject_removed_pooling_parameters,
)
class EmbeddingCompletionRequest(
PoolingBasicRequestMixin,
CompletionRequestMixin,
EmbedRequestMixin,
EmbeddingTokenizeParamsMixin,
):
def to_pooling_params(self):
return PoolingParams(
task="embed",
dimensions=self.dimensions,
use_activation=self.use_activation,
)
def _is_chat_message(value: Any) -> bool:
return isinstance(value, dict) and isinstance(value.get("role"), str)
def _is_chat_messages(value: Any) -> bool:
return (
isinstance(value, list)
and bool(value)
and all(_is_chat_message(item) for item in value)
)
def _is_batched_chat_messages(value: Any) -> bool:
return (
isinstance(value, list)
and bool(value)
and all(_is_chat_messages(item) for item in value)
)
class EmbeddingChatRequest(
PoolingBasicRequestMixin,
ChatRequestMixin,
EmbedRequestMixin,
EmbeddingTokenizeParamsMixin,
):
"""OpenAI embeddings request with one top-level chat conversation."""
def to_pooling_params(self):
return PoolingParams(
task="embed",
dimensions=self.dimensions,
use_activation=self.use_activation,
)
class EmbeddingBatchChatRequest(
PoolingBasicRequestMixin,
ChatRequestOptionsMixin,
EmbedRequestMixin,
EmbeddingTokenizeParamsMixin,
):
"""OpenAI embeddings request with batched top-level chat conversations.
Mirrors ``BatchChatCompletionRequest`` by keeping batched conversations in
``messages`` instead of introducing a separate batch-specific field.
"""
messages: Sequence[
Annotated[list[ChatCompletionMessageParam], Field(min_length=1)]
] = Field(..., min_length=1)
def to_pooling_params(self):
return PoolingParams(
task="embed",
dimensions=self.dimensions,
use_activation=self.use_activation,
)
class EmbeddingChatInputRequest(
EmbeddingChatRequest,
):
"""OpenAI embeddings request with one chat conversation in ``input``."""
input: list[ChatCompletionMessageParam]
@model_validator(mode="before")
@classmethod
def normalize_input_messages(cls, data):
if not isinstance(data, dict):
return data
if "messages" in data or "input" not in data:
return data
input_data = data["input"]
if not _is_chat_messages(input_data):
return data
normalized = dict(data)
normalized["messages"] = input_data
return normalized
class EmbeddingBatchChatInputRequest(EmbeddingBatchChatRequest):
"""OpenAI embeddings request with batched chat conversations in ``input``."""
input: Sequence[
Annotated[list[ChatCompletionMessageParam], Field(min_length=1)]
] = Field(..., min_length=1)
@model_validator(mode="before")
@classmethod
def normalize_input_messages(cls, data):
if not isinstance(data, dict):
return data
if "messages" in data or "input" not in data:
return data
input_data = data["input"]
if not _is_batched_chat_messages(input_data):
return data
normalized = dict(data)
normalized["messages"] = input_data
return normalized
EmbeddingRequest: TypeAlias = Annotated[
EmbeddingCompletionRequest
| EmbeddingChatRequest
| EmbeddingBatchChatRequest
| EmbeddingChatInputRequest
| EmbeddingBatchChatInputRequest,
BeforeValidator(reject_removed_pooling_parameters),
]
# ---------------------------------------------------------------------------
# OpenAI /v1/embeddings — response models
# ---------------------------------------------------------------------------
class EmbeddingResponseData(OpenAIBaseModel):
index: int
object: str = "embedding"
embedding: list[float] | str
class EmbeddingResponse(OpenAIBaseModel):
id: str = Field(default_factory=lambda: f"embd-{random_uuid()}")
object: str = "list"
created: int = Field(default_factory=lambda: int(time.time()))
model: str | None = None
data: list[EmbeddingResponseData]
usage: UsageInfo
class EmbeddingBytesResponse(OpenAIBaseModel):
content: list[bytes]
headers: dict[str, str] | None = None
media_type: str = "application/octet-stream"
# ---------------------------------------------------------------------------
# Cohere /v2/embed — request models
# ---------------------------------------------------------------------------
CohereEmbeddingType = Literal[
"float",
"binary",
"ubinary",
"base64",
]
CohereTruncate = Literal["NONE", "START", "END"]
class CohereEmbedContent(BaseModel):
type: Literal["text", "image_url"]
text: str | None = None
image_url: dict[str, str] | None = None
@model_validator(mode="after")
def validate_content_payload(self):
if self.type == "text":
if self.text is None:
raise ValueError("CohereEmbedContent with type='text' requires text")
elif not self.image_url or not self.image_url.get("url"):
raise ValueError(
"CohereEmbedContent with type='image_url' requires image_url.url"
)
return self
class CohereEmbedInput(BaseModel):
content: list[CohereEmbedContent]
class CohereEmbedRequest(BaseModel):
model: str | None = None
input_type: str | None = None
texts: list[str] | None = None
images: list[str] | None = None
inputs: list[CohereEmbedInput] | None = None
output_dimension: int | None = None
embedding_types: list[CohereEmbeddingType] | None = None
truncate: CohereTruncate = "END"
max_tokens: int | None = None
priority: int = Field(default=0, ge=-(2**63), le=2**63 - 1)
@model_validator(mode="after")
def validate_input_fields(self):
input_fields = (self.texts, self.images, self.inputs)
provided_fields = [field for field in input_fields if field is not None]
if len(provided_fields) != 1 or not provided_fields[0]:
raise ValueError(
"Exactly one of texts, images, or inputs must be provided, "
"and it must be non-empty"
)
return self
# ---------------------------------------------------------------------------
# Cohere /v2/embed — response models
# ---------------------------------------------------------------------------
class CohereApiVersion(BaseModel):
version: str = "2"
class CohereBilledUnits(BaseModel):
input_tokens: int | None = None
image_tokens: int | None = None
class CohereMeta(BaseModel):
api_version: CohereApiVersion = Field(default_factory=CohereApiVersion)
billed_units: CohereBilledUnits | None = None
class CohereEmbedByTypeEmbeddings(BaseModel):
# The field name ``float`` shadows the builtin type, so the annotation
# must use ``builtins.float`` to avoid a self-referential type error.
float: list[list[builtins.float]] | None = None
binary: list[list[int]] | None = None
ubinary: list[list[int]] | None = None
base64: list[str] | None = None
class CohereEmbedResponse(BaseModel):
id: str = Field(default_factory=lambda: f"embd-{random_uuid()}")
embeddings: CohereEmbedByTypeEmbeddings
texts: list[str] | None = None
meta: CohereMeta | None = None
response_type: Literal["embeddings_by_type"] = "embeddings_by_type"
# ---------------------------------------------------------------------------
# Cohere embedding type conversion helpers
# ---------------------------------------------------------------------------
_UNSIGNED_TO_SIGNED_DIFF = 1 << 7 # 128
def _pack_binary_embeddings(
float_embeddings: list[list[float]],
signed: bool,
) -> list[list[int]]:
"""Bit-pack float embeddings: positive -> 1, negative -> 0.
Bits are packed MSB-first, eight per byte.
"""
if not float_embeddings:
return []
array = np.asarray(float_embeddings, dtype=np.float64)
if array.ndim != 2:
raise ValueError(
f"Expected a 2D batch of embeddings, but got {array.ndim}D input."
)
dim = array.shape[1]
if dim % 8 != 0:
raise ValueError(
"Embedding dimension must be a multiple of 8 for binary "
f"embedding types, but got {dim}."
)
packed = np.packbits(array >= 0, axis=-1)
if signed:
packed = packed.astype(np.int16) - _UNSIGNED_TO_SIGNED_DIFF
return packed.tolist()
def _encode_base64_embeddings(
float_embeddings: list[list[float]],
) -> list[str]:
"""Encode float embeddings as base64 (little-endian float32)."""
result: list[str] = []
for embedding in float_embeddings:
buf = struct.pack(f"<{len(embedding)}f", *embedding)
result.append(base64.b64encode(buf).decode("utf-8"))
return result
def build_typed_embeddings(
float_embeddings: list[list[float]],
embedding_types: Sequence[str],
) -> CohereEmbedByTypeEmbeddings:
"""Convert float embeddings to all requested Cohere embedding types."""
result = CohereEmbedByTypeEmbeddings()
for emb_type in embedding_types:
if emb_type == "float":
result.float = float_embeddings
elif emb_type == "binary":
result.binary = _pack_binary_embeddings(float_embeddings, signed=True)
elif emb_type == "ubinary":
result.ubinary = _pack_binary_embeddings(float_embeddings, signed=False)
elif emb_type == "base64":
result.base64 = _encode_base64_embeddings(float_embeddings)
return result
-220
View File
@@ -1,220 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TypeAlias, assert_never, cast
from fastapi.responses import JSONResponse, Response, StreamingResponse
from vllm.logger import init_logger
from vllm.outputs import PoolingRequestOutput
from vllm.utils.serial_utils import EmbedDType, Endianness
from ..base.io_processor import PoolingIOProcessor
from ..base.serving import PoolingServing
from ..typing import PoolingServeContext
from ..utils import (
BytesEncodingFormat,
JsonEncodingFormat,
build_pooling_bytes_streaming_response,
encode_pooling_output_float,
encode_pooling_output_float_or_ndarray,
get_json_response_cls,
get_pooling_output_encoder,
get_pooling_usage,
)
from .io_processor import EmbedIOProcessor
from .protocol import (
CohereBilledUnits,
CohereEmbedRequest,
CohereEmbedResponse,
CohereMeta,
EmbeddingRequest,
EmbeddingResponse,
EmbeddingResponseData,
build_typed_embeddings,
)
logger = init_logger(__name__)
EmbeddingServeContext: TypeAlias = PoolingServeContext[EmbeddingRequest]
class ServingEmbedding(PoolingServing):
"""Embedding API supporting both OpenAI and Cohere formats."""
request_id_prefix = "embd"
io_processor: EmbedIOProcessor
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.json_response_cls = get_json_response_cls()
def init_io_processor(self, *args, **kwargs) -> EmbedIOProcessor:
return EmbedIOProcessor(*args, **kwargs)
async def _preprocessing(
self, io_processor: PoolingIOProcessor, ctx: PoolingServeContext
):
await super()._preprocessing(io_processor, ctx)
self.io_processor.maybe_pre_process_chunked(ctx)
def _build_response(
self,
ctx: PoolingServeContext,
) -> Response:
if isinstance(ctx.request, CohereEmbedRequest):
return self._build_cohere_response_from_ctx(ctx)
return self._build_openai_response(ctx)
def _build_openai_response(
self,
ctx: EmbeddingServeContext,
) -> JSONResponse | StreamingResponse:
encoding_format = ctx.request.encoding_format
embed_dtype = ctx.request.embed_dtype
endianness = ctx.request.endianness
if encoding_format == "float" or encoding_format == "base64":
return self._openai_json_response(
ctx.final_res_batch,
ctx.request_id,
ctx.created_time,
ctx.model_name,
encoding_format,
embed_dtype,
endianness,
)
if encoding_format == "bytes" or encoding_format == "bytes_only":
return self._openai_bytes_response(
ctx.final_res_batch,
ctx.request_id,
ctx.created_time,
ctx.model_name,
encoding_format,
embed_dtype,
endianness,
)
assert_never(encoding_format)
def _openai_json_response(
self,
final_res_batch: list[PoolingRequestOutput],
request_id: str,
created_time: int,
model_name: str,
encoding_format: JsonEncodingFormat,
embed_dtype: EmbedDType,
endianness: Endianness,
) -> JSONResponse:
use_ndarray_response = (
encoding_format == "float"
and self.json_response_cls.__name__ == "ORJSONResponse"
)
if use_ndarray_response:
ndarray_items: list[dict[str, object]] = []
for idx, final_res in enumerate(final_res_batch):
item_dict = EmbeddingResponseData(
index=idx,
embedding=[],
).model_dump()
item_dict["embedding"] = encode_pooling_output_float_or_ndarray(
final_res
)
ndarray_items.append(item_dict)
ndarray_response = EmbeddingResponse(
id=request_id,
created=created_time,
model=model_name,
data=[], # type: ignore[arg-type]
usage=get_pooling_usage(final_res_batch),
).model_dump()
ndarray_response["data"] = ndarray_items
return self.json_response_cls(content=ndarray_response)
encode_fn = get_pooling_output_encoder(
encoding_format=encoding_format,
embed_dtype=embed_dtype,
endianness=endianness,
)
items: list[EmbeddingResponseData] = []
for idx, final_res in enumerate(final_res_batch):
item = EmbeddingResponseData(
index=idx,
embedding=cast(list[float] | str, encode_fn(final_res)),
)
items.append(item)
response = EmbeddingResponse(
id=request_id,
created=created_time,
model=model_name,
data=items,
usage=get_pooling_usage(final_res_batch),
)
return self.json_response_cls(content=response.model_dump())
def _openai_bytes_response(
self,
final_res_batch: list[PoolingRequestOutput],
request_id: str,
created_time: int,
model_name: str,
encoding_format: BytesEncodingFormat,
embed_dtype: EmbedDType,
endianness: Endianness,
) -> StreamingResponse:
return build_pooling_bytes_streaming_response(
pooling_outputs=final_res_batch,
request_id=request_id,
created_time=created_time,
model_name=model_name,
encoding_format=encoding_format,
embed_dtype=embed_dtype,
endianness=endianness,
)
def _build_cohere_response_from_ctx(
self,
ctx: PoolingServeContext,
) -> JSONResponse:
request = ctx.request
assert isinstance(request, CohereEmbedRequest)
all_floats = [
cast(list[float], encode_pooling_output_float(out))
for out in ctx.final_res_batch
]
total_tokens = get_pooling_usage(ctx.final_res_batch).prompt_tokens
has_image_input = request.images is not None or any(
content.type == "image_url"
for input_item in request.inputs or []
for content in input_item.content
)
image_tokens = total_tokens if has_image_input else 0
texts_echo = request.texts
embedding_types = request.embedding_types or ["float"]
embeddings_obj = build_typed_embeddings(all_floats, embedding_types)
input_tokens = total_tokens - image_tokens
response = CohereEmbedResponse(
id=ctx.request_id,
embeddings=embeddings_obj,
texts=texts_echo,
meta=CohereMeta(
billed_units=CohereBilledUnits(
input_tokens=input_tokens,
image_tokens=image_tokens,
),
),
)
return self.json_response_cls(content=response.model_dump(exclude_none=True))
-272
View File
@@ -1,272 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
from fastapi import FastAPI
from vllm.config import ModelConfig, VllmConfig
from vllm.logger import init_logger
from vllm.plugins.io_processors import has_io_processor
from vllm.renderers import BaseRenderer
from vllm.renderers.chat_utils import ChatTemplateConfig
from vllm.tasks import POOLING_TASKS, SCORE_TYPE_MAP, SupportedTask
from .base.io_processor import PoolingIOProcessor
from .utils import enable_scoring_api
if TYPE_CHECKING:
from argparse import Namespace
from starlette.datastructures import State
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.serve.sagemaker.api_router import (
EndpointFn,
GetHandlerFn,
RequestType,
)
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
else:
RequestLogger = object
logger = init_logger(__name__)
def init_pooling_io_processors(
supported_tasks: tuple[SupportedTask, ...],
vllm_config: VllmConfig,
renderer: BaseRenderer,
chat_template_config: ChatTemplateConfig,
) -> dict[str, PoolingIOProcessor]:
model_config = vllm_config.model_config
processors: dict[str, type[PoolingIOProcessor]] = {}
pooling_task = model_config.get_pooling_task(supported_tasks)
if pooling_task == "classify":
from .classify.io_processor import ClassifyIOProcessor
processors["classify"] = ClassifyIOProcessor
if pooling_task == "token_classify":
from .classify.io_processor import TokenClassifyIOProcessor
processors["token_classify"] = TokenClassifyIOProcessor
if pooling_task == "embed":
from .embed.io_processor import EmbedIOProcessor
processors["embed"] = EmbedIOProcessor
if pooling_task == "token_embed":
from .embed.io_processor import TokenEmbedIOProcessor
processors["token_embed"] = TokenEmbedIOProcessor
if pooling_task == "embed&token_classify":
from .pooling.io_processor import UnsupportedCombinedTaskIOProcessor
processors[pooling_task] = UnsupportedCombinedTaskIOProcessor
has_plugin = has_io_processor(
vllm_config,
model_config.io_processor_plugin,
)
if has_plugin:
from .pooling.io_processor import PluginWithIOProcessorPlugins
processors["plugin"] = PluginWithIOProcessorPlugins
elif pooling_task == "plugin":
from .pooling.io_processor import PluginWithoutIOProcessorPlugins
processors["plugin"] = PluginWithoutIOProcessorPlugins
if enable_scoring_api(supported_tasks, model_config):
from .scoring.io_processor import ScoringIOProcessors
score_type: str | None = SCORE_TYPE_MAP.get(pooling_task, None) # type: ignore[arg-type]
if score_type is not None and score_type in ScoringIOProcessors:
processors[score_type] = ScoringIOProcessors[score_type]
if model_config.architecture == "JinaForRanking":
from .embed.io_processor import JinaRankingTokenEmbedIOProcessor
from .scoring.io_processor import ScoringIOProcessors
processors["token_embed"] = JinaRankingTokenEmbedIOProcessor
processors["late-interaction"] = ScoringIOProcessors["jina-reranking-scoring"]
return {
task: processor_cls(
vllm_config=vllm_config,
renderer=renderer,
chat_template_config=chat_template_config,
)
for task, processor_cls in processors.items()
}
def register_pooling_api_routers(
app: FastAPI,
supported_tasks: tuple["SupportedTask", ...],
model_config: ModelConfig | None = None,
):
if model_config is None:
return
pooling_task = model_config.get_pooling_task(supported_tasks)
if pooling_task is not None:
from .pooling.api_router import router as pooling_router
app.include_router(pooling_router)
if "classify" in supported_tasks:
from .classify.api_router import (
router as classify_router,
)
app.include_router(classify_router)
if "embed" in supported_tasks:
from .embed.api_router import router as embed_router
app.include_router(embed_router)
if enable_scoring_api(supported_tasks, model_config):
from .scoring.api_router import router as score_router
app.include_router(score_router)
def init_pooling_state(
engine_client: "EngineClient",
state: "State",
args: "Namespace",
request_logger: RequestLogger | None,
supported_tasks: tuple["SupportedTask", ...],
):
model_config = engine_client.model_config
if model_config is None:
return
from vllm.renderers.chat_utils import load_chat_template
from vllm.tasks import POOLING_TASKS
from .classify.serving import ServingClassification
from .embed.serving import ServingEmbedding
from .pooling.serving import ServingPooling
from .scoring.serving import ServingScores
resolved_chat_template = load_chat_template(args.chat_template)
pooling_task = model_config.get_pooling_task(supported_tasks)
chat_template_config = ChatTemplateConfig(
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
trust_request_chat_template=args.trust_request_chat_template,
trust_request_mm_kwargs=args.trust_request_mm_kwargs,
)
state.serving_pooling = (
(
ServingPooling(
engine_client,
state.openai_serving_models,
supported_tasks=supported_tasks,
request_logger=request_logger,
chat_template_config=chat_template_config,
)
)
if any(t in supported_tasks for t in POOLING_TASKS)
else None
)
state.serving_embedding = (
ServingEmbedding(
engine_client,
state.openai_serving_models,
request_logger=request_logger,
chat_template_config=chat_template_config,
)
if pooling_task == "embed"
else None
)
state.serving_classification = (
ServingClassification(
engine_client,
state.openai_serving_models,
request_logger=request_logger,
chat_template_config=chat_template_config,
)
if pooling_task == "classify"
else None
)
state.serving_scores = (
ServingScores(
engine_client,
state.openai_serving_models,
supported_tasks=supported_tasks,
request_logger=request_logger,
chat_template_config=chat_template_config,
enable_flash_late_interaction=getattr(
args, "enable_flash_late_interaction", True
),
)
if enable_scoring_api(supported_tasks, model_config)
else None
)
def get_pooling_invocation_types(
supported_tasks: tuple["SupportedTask", ...],
model_config: ModelConfig | None = None,
):
# NOTE: Items defined earlier take higher priority
invocation_types: list[tuple[RequestType, tuple[GetHandlerFn, EndpointFn]]] = []
if model_config is None:
return invocation_types
pooling_task = model_config.get_pooling_task(supported_tasks)
if pooling_task == "embed":
from .embed.api_router import create_embedding, embedding
from .embed.protocol import EmbeddingRequest
invocation_types += [
(EmbeddingRequest, (embedding, create_embedding)),
]
if pooling_task == "classify":
from .classify.api_router import classify, create_classify
from .classify.protocol import ClassificationRequest
invocation_types += [
(ClassificationRequest, (classify, create_classify)),
]
if enable_scoring_api(supported_tasks, model_config):
from .scoring.api_router import do_rerank, rerank
from .scoring.protocol import RerankRequest
invocation_types += [
(RerankRequest, (rerank, do_rerank)),
]
from .scoring.api_router import create_score, score
from .scoring.protocol import ScoreRequest
invocation_types += [
(ScoreRequest, (score, create_score)),
]
if any(task in POOLING_TASKS for task in supported_tasks):
from .pooling.api_router import create_pooling, pooling
from .pooling.protocol import PoolingRequest
invocation_types += [
(PoolingRequest, (pooling, create_pooling)),
]
return invocation_types
-92
View File
@@ -1,92 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
from dataclasses import replace
from typing import cast
from vllm.config import VllmConfig
from vllm.exceptions import VLLMValidationError
from vllm.outputs import LateChunk, LateChunkingMetadata, PoolingRequestOutput
from vllm.renderers import TokenizeParams
from .typing import AnyRenderParam, EncodeCMPLRenderParams
def prepare_late_chunking_input(
config: VllmConfig, render_params: AnyRenderParam
) -> tuple[str, TokenizeParams]:
"""Validate full-document text input before requesting tokenizer offsets."""
params = render_params["params"]
params.verify(config.model_config)
if "prompts" not in render_params:
raise VLLMValidationError("Late chunking requires plain-text input")
prompt = cast(EncodeCMPLRenderParams, render_params)["prompts"]
if (
not isinstance(prompt, dict)
or set(prompt) - {"prompt", "cache_salt"}
or not isinstance(text := prompt.get("prompt"), str)
or not text
):
raise VLLMValidationError("Late chunking requires nonempty plain-text input")
tok_params = render_params["tok_params"]
if (
tok_params.pad_prompt_tokens is not None
or tok_params.truncate_prompt_tokens is not None
or tok_params.do_lower_case
):
raise VLLMValidationError(
"Late chunking does not support input padding, truncation or "
"renderer text normalization"
)
return text, replace(tok_params, return_token_offsets=True)
def build_late_chunking_metadata(
text: str,
num_tokens: int,
offsets: Sequence[tuple[int, int]] | None,
chunk_size: int,
) -> LateChunkingMetadata:
"""Map chunks to original text, keeping special-token-only chunks."""
if num_tokens <= 0 or offsets is None or len(offsets) != num_tokens:
raise VLLMValidationError(
"Late chunking requires a fast tokenizer with offsets aligned to "
"all input tokens"
)
previous_start = previous_end = 0
for start, end in offsets:
if not 0 <= start <= end <= len(text):
raise VLLMValidationError("Token offsets are not aligned to the input text")
if start < end:
if start < previous_start or end < previous_end:
raise VLLMValidationError("Token offsets are not in source-text order")
previous_start, previous_end = start, end
chunks = []
for start in range(0, num_tokens, chunk_size):
end = min(start + chunk_size, num_tokens)
source = [(a, b) for a, b in offsets[start:end] if a < b]
char_range = (
(min(a for a, _ in source), max(b for _, b in source)) if source else None
)
chunks.append(LateChunk(token_range=(start, end), char_range=char_range))
return LateChunkingMetadata(
chunk_size=chunk_size, input_tokens=num_tokens, chunks=chunks
)
def attach_late_chunking_metadata(
output: PoolingRequestOutput, metadata: LateChunkingMetadata
) -> None:
"""Attach ranges only to successful, complete chunk-vector results."""
if not output.finished or output.error is not None:
return
data = output.outputs.data
if (
data.ndim != 2
or data.shape[0] != len(metadata.chunks)
or len(output.prompt_token_ids) != metadata.input_tokens
):
raise ValueError("Late-chunking output does not match its token/source ranges")
output.late_chunking = metadata
-479
View File
@@ -1,479 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable, Sequence
from typing import Any
from tqdm.auto import tqdm
from vllm.entrypoints.common.offline import OfflineInferenceMixin
from vllm.inputs import DataPrompt, PromptType
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
from vllm.outputs import (
ClassificationRequestOutput,
EmbeddingRequestOutput,
PoolingRequestOutput,
ScoringRequestOutput,
)
from vllm.pooling_params import PoolingParams
from vllm.renderers.chat_utils import ChatTemplateConfig
from vllm.tasks import SCORE_TYPE_MAP, PoolingTask, SupportedTask
from .base.io_processor import PoolingIOProcessor
from .factories import init_pooling_io_processors
from .scoring.io_processor import ScoringIOProcessor
from .scoring.typing import ScoreInput
from .typing import (
AnyOfflineInputsContext,
OfflineEncodeInputsContext,
OfflineOutputsContext,
OfflinePluginInputsContext,
OfflineScoringInputsContext,
RequestFactory,
)
logger = init_logger(__name__)
class PoolingOfflineMixin(OfflineInferenceMixin):
"""Offline inference for pooling models."""
runner_type: str
chat_template: str | None
supported_tasks: tuple[SupportedTask, ...]
def __init__(self):
self.pooling_task = self.model_config.get_pooling_task(self.supported_tasks)
if self.pooling_task is not None:
logger.info("Supported pooling task: %s", self.pooling_task)
self.chat_template_config = ChatTemplateConfig(chat_template=self.chat_template)
self.pooling_io_processors = init_pooling_io_processors(
supported_tasks=self.supported_tasks,
vllm_config=self.llm_engine.vllm_config,
renderer=self.renderer,
chat_template_config=self.chat_template_config,
)
# Use thread pool executor to accelerate preprocessing.
self._executor = self.renderer._executor
def encode(
self,
prompts: PromptType | Sequence[PromptType] | DataPrompt,
pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,
*,
use_tqdm: bool | Callable[..., tqdm] = True,
lora_request: list[LoRARequest] | LoRARequest | None = None,
pooling_task: PoolingTask | None = None,
tokenization_kwargs: dict[str, Any] | None = None,
) -> list[PoolingRequestOutput]:
"""Apply pooling to the hidden states corresponding to the input
prompts.
This class automatically batches the given prompts, considering
the memory constraint. For the best performance, put all of your prompts
into a single list and pass it to this method.
Args:
prompts: The prompts to the LLM. You may pass a sequence of prompts
for batch inference. See [PromptType][vllm.inputs.PromptType]
for more details about the format of each prompt.
pooling_params: The pooling parameters for pooling. If None, we
use the default pooling parameters.
use_tqdm: If `True`, shows a tqdm progress bar.
If a callable (e.g., `functools.partial(tqdm, leave=False)`),
it is used to create the progress bar.
If `False`, no progress bar is created.
lora_request: LoRA request to use for generation, if any.
pooling_task: Override the pooling task to use.
tokenization_kwargs: Overrides for `tokenizer.encode`.
Returns:
A list of `PoolingRequestOutput` objects containing the
pooled hidden states in the same order as the input prompts.
"""
if isinstance(prompts, dict) and "data" in prompts and pooling_task != "plugin":
raise ValueError(
"The 'data' field is only supported for the 'plugin' pooling task."
)
self._verify_pooling_task(pooling_task)
assert pooling_task is not None and pooling_task in self.pooling_io_processors
io_processor = self.pooling_io_processors[pooling_task]
ctx: AnyOfflineInputsContext
if isinstance(prompts, dict) and "data" in prompts:
ctx = OfflinePluginInputsContext(
pooling_task=pooling_task,
prompts=prompts, # type: ignore[arg-type]
tokenization_kwargs=tokenization_kwargs,
pooling_params=pooling_params,
lora_request=lora_request,
priorities=None,
)
else:
ctx = OfflineEncodeInputsContext(
pooling_task=pooling_task,
prompts=prompts,
tokenization_kwargs=tokenization_kwargs,
pooling_params=pooling_params,
lora_request=lora_request,
priorities=None,
)
request_factory, num_requests = io_processor.get_request_factory_offline(ctx)
outputs = self._run_tiling_engine(
io_processor, request_factory, num_requests, use_tqdm=use_tqdm
)
outputs = io_processor.post_process_offline(
ctx=OfflineOutputsContext(outputs=outputs, late_chunking=ctx.late_chunking)
)
return outputs
def _verify_pooling_task(self, pooling_task: PoolingTask | None):
if self.runner_type != "pooling":
raise ValueError(
"LLM.encode() is only supported for pooling models. "
"Try passing `--runner pooling` to use the model as a "
"pooling model."
)
if pooling_task is None:
raise ValueError(
"""
pooling_task required for `LLM.encode`.
Please use one of the more specific methods or set the pooling_task when using `LLM.encode`:
- For embeddings, use `LLM.embed(...)` or `pooling_task="embed"`.
- For classification logits, use `LLM.classify(...)` or `pooling_task="classify"`.
- For similarity scores, use `LLM.score(...)`.
- For rewards, `pooling_task="classify"` or `pooling_task="token_classify"`.
- For token classification, use `pooling_task="token_classify"`.
- For multi-vector retrieval, use `pooling_task="token_embed"`.
""" # noqa: E501
)
if (
pooling_task in ("embed", "token_embed")
and pooling_task not in self.supported_tasks
):
raise ValueError(
"Embedding API is not supported by this model. "
"Try converting the model using `--convert embed`."
)
if (
pooling_task in ("classify", "token_classify")
and pooling_task not in self.supported_tasks
):
raise ValueError(
"Classification API is not supported by this model. "
"Try converting the model using `--convert classify`."
)
# plugin task uses io_processor.parse_data to verify inputs
if pooling_task != "plugin" and pooling_task != self.pooling_task:
if pooling_task not in self.supported_tasks:
raise ValueError(
f"Unsupported task: {pooling_task!r} "
f"Supported tasks: {self.supported_tasks}"
)
else:
raise ValueError(
f"Try switching the model's pooling_task "
f'via `PoolerConfig(task="{pooling_task}")`'
)
if pooling_task == "plugin" and "plugin" not in self.pooling_io_processors:
raise ValueError(
"No IOProcessor plugin installed. Please refer "
"to the documentation and to the "
"'prithvi_geospatial_mae_io_processor' "
"offline inference example for more details."
)
def embed(
self,
prompts: PromptType | Sequence[PromptType],
*,
use_tqdm: bool | Callable[..., tqdm] = True,
pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,
lora_request: list[LoRARequest] | LoRARequest | None = None,
tokenization_kwargs: dict[str, Any] | None = None,
) -> list[EmbeddingRequestOutput]:
"""Generate an embedding vector for each prompt.
This class automatically batches the given prompts, considering
the memory constraint. For the best performance, put all of your prompts
into a single list and pass it to this method.
Args:
prompts: The prompts to the LLM. You may pass a sequence of prompts
for batch inference. See [PromptType][vllm.inputs.PromptType]
for more details about the format of each prompt.
pooling_params: The pooling parameters for pooling. If None, we
use the default pooling parameters.
use_tqdm: If `True`, shows a tqdm progress bar.
If a callable (e.g., `functools.partial(tqdm, leave=False)`),
it is used to create the progress bar.
If `False`, no progress bar is created.
lora_request: LoRA request to use for generation, if any.
tokenization_kwargs: Overrides for `tokenizer.encode`.
Returns:
A list of `EmbeddingRequestOutput` objects containing the
embedding vectors in the same order as the input prompts.
"""
items = self.encode(
prompts,
use_tqdm=use_tqdm,
pooling_params=pooling_params,
lora_request=lora_request,
pooling_task="embed",
tokenization_kwargs=tokenization_kwargs,
)
return [EmbeddingRequestOutput.from_base(item) for item in items]
def classify(
self,
prompts: PromptType | Sequence[PromptType],
*,
pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,
use_tqdm: bool | Callable[..., tqdm] = True,
lora_request: list[LoRARequest] | LoRARequest | None = None,
tokenization_kwargs: dict[str, Any] | None = None,
) -> list[ClassificationRequestOutput]:
"""Generate class logits for each prompt.
This class automatically batches the given prompts, considering
the memory constraint. For the best performance, put all of your prompts
into a single list and pass it to this method.
Args:
prompts: The prompts to the LLM. You may pass a sequence of prompts
for batch inference. See [PromptType][vllm.inputs.PromptType]
for more details about the format of each prompt.
pooling_params: The pooling parameters for pooling. If None, we
use the default pooling parameters.
use_tqdm: If `True`, shows a tqdm progress bar.
If a callable (e.g., `functools.partial(tqdm, leave=False)`),
it is used to create the progress bar.
If `False`, no progress bar is created.
lora_request: LoRA request to use for generation, if any.
tokenization_kwargs: Overrides for `tokenizer.encode`.
Returns:
A list of `ClassificationRequestOutput` objects containing the
embedding vectors in the same order as the input prompts.
"""
items = self.encode(
prompts,
use_tqdm=use_tqdm,
pooling_params=pooling_params,
lora_request=lora_request,
pooling_task="classify",
tokenization_kwargs=tokenization_kwargs,
)
return [ClassificationRequestOutput.from_base(item) for item in items]
def score(
self,
data_1: ScoreInput | list[ScoreInput],
data_2: ScoreInput | list[ScoreInput],
/,
*,
use_tqdm: bool | Callable[..., tqdm] = True,
pooling_params: PoolingParams | None = None,
lora_request: list[LoRARequest] | LoRARequest | None = None,
tokenization_kwargs: dict[str, Any] | None = None,
chat_template: str | None = None,
) -> list[ScoringRequestOutput]:
"""Generate similarity scores for all pairs `<text,text_pair>` or
`<multi-modal data, multi-modal data pair>`.
The inputs can be `1 -> 1`, `1 -> N` or `N -> N`.
In the `1 - N` case the `data_1` input will be replicated `N`
times to pair with the `data_2` inputs.
The input pairs are used to build a list of prompts for the
cross encoder model. This class automatically batches the prompts,
considering the memory constraint. For the best performance, put all
of your inputs into a single list and pass it to this method.
Supports both text and multi-modal data (images, etc.) when used with
appropriate multi-modal models. For multi-modal inputs, ensure the
prompt structure matches the model's expected input format.
Args:
data_1: Can be a single prompt, a list of prompts or
`ScoreMultiModalParam`, which can contain either text or
multi-modal data. When a list, it must have the same length as
the `data_2` list.
data_2: The data to pair with the query to form the input to
the LLM. Can be text or multi-modal data. See [PromptType]
[vllm.inputs.PromptType] for more details about the format of
each prompt.
pooling_params: The pooling parameters for pooling. If None, we
use the default pooling parameters.
use_tqdm: If `True`, shows a tqdm progress bar.
If a callable (e.g., `functools.partial(tqdm, leave=False)`),
it is used to create the progress bar.
If `False`, no progress bar is created.
lora_request: LoRA request to use for generation, if any.
chat_template: The chat template to use for scoring. If None, use
the chat template passed to the LLM constructor.
tokenization_kwargs: Overrides for `tokenizer.encode`.
Returns:
A list of `ScoringRequestOutput` objects containing the
generated scores in the same order as the input prompts.
"""
if self.runner_type != "pooling":
raise ValueError(
"LLM.score() is only supported for pooling models. "
"Try passing `--runner pooling` to use the model as a "
"pooling model."
)
score_type: str | None = SCORE_TYPE_MAP.get(self.pooling_task, None) # type: ignore[arg-type]
if (
score_type == "cross-encoder"
and getattr(self.model_config.hf_config, "num_labels", 0) != 1
):
raise ValueError("Scoring API is only enabled for num_labels == 1.")
if score_type is None or score_type not in self.pooling_io_processors:
raise ValueError("This model does not support the Scoring API.")
io_processor = self.pooling_io_processors[score_type]
assert isinstance(io_processor, ScoringIOProcessor)
scoring_data = io_processor.valid_inputs(data_1, data_2)
n_queries = len(scoring_data.data_1)
if pooling_params is None:
pooling_params = PoolingParams()
assert isinstance(pooling_params, PoolingParams)
pooling_task = io_processor.pooling_task
# Clone to avoid modifying the caller's pooling parameters.
pooling_params = pooling_params.clone()
if pooling_params.task is None:
pooling_params.task = pooling_task
elif pooling_params.task != pooling_task:
msg = (
f"You cannot overwrite {pooling_params.task=!r} with {pooling_task=!r}!"
)
raise ValueError(msg)
ctx = OfflineScoringInputsContext(
pooling_task=pooling_task,
scoring_data=scoring_data,
pooling_params=pooling_params,
tokenization_kwargs=tokenization_kwargs,
lora_request=lora_request,
chat_template=chat_template,
priorities=None,
)
request_factory, num_requests = io_processor.get_request_factory_offline(ctx)
outputs = self._run_tiling_engine(
io_processor, request_factory, num_requests, use_tqdm=use_tqdm
)
outputs = io_processor.post_process_offline(
ctx=OfflineOutputsContext(outputs=outputs, n_queries=n_queries),
)
return [ScoringRequestOutput.from_base(item) for item in outputs]
def _run_tiling_engine(
self,
io_processor: PoolingIOProcessor,
request_factory: RequestFactory,
num_requests: int,
use_tqdm: bool | Callable[..., tqdm] = True,
):
if num_requests == 0:
raise ValueError("You must pass at least one prompt")
# Keeping max_num_seqs * 2 requests in the core can already saturate the core.
# Therefore, keep most requests waiting outside the core.
max_requests_in_core = (
self.llm_engine.vllm_config.scheduler_config.max_num_seqs * 2
)
num_requests_in_core = 0
num_waited_requests = num_requests
if use_tqdm:
tqdm_func = use_tqdm if callable(use_tqdm) else tqdm
pbar = tqdm_func(
total=num_requests,
desc="Processed prompts",
dynamic_ncols=True,
postfix=f"est. speed input: {0:.2f} toks/s, output: {0:.2f} toks/s",
)
outputs: list[PoolingRequestOutput] = []
added_request_ids: set[str] = set()
it = self._executor.map(io_processor.render, request_factory())
try:
while num_waited_requests or self.llm_engine.has_unfinished_requests():
requests = []
for _ in range(max_requests_in_core - num_requests_in_core):
if num_waited_requests == 0:
break
try:
request = next(it)
requests.append(request)
except StopIteration:
num_waited_requests = 0
break
num_waited_requests -= 1
num_requests_in_core += 1
if requests:
request_ids = self._render_and_add_requests(
prompts=[x["prompts"] for x in requests],
params=[x["params"] for x in requests],
lora_requests=[x["lora_requests"] for x in requests],
priorities=[x["priorities"] for x in requests],
)
for request_id in request_ids:
# undo assign_request_id
request_id = request_id.split("-", 1)[0]
added_request_ids.add(request_id)
step_outputs = self.llm_engine.step()
for output in step_outputs:
assert isinstance(output, PoolingRequestOutput)
assert output.finished
outputs.append(output)
added_request_ids.discard(output.request_id)
num_requests_in_core -= 1
if use_tqdm:
pbar.update(1)
if pbar.n == num_requests:
pbar.refresh()
except Exception:
if added_request_ids:
self.llm_engine.abort_request(list(added_request_ids))
raise
finally:
if use_tqdm:
pbar.close()
return sorted(outputs, key=lambda x: int(x.request_id))
@@ -1,39 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from http import HTTPStatus
from fastapi import APIRouter, Depends, Request
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.utils.api_utils import (
load_aware_call,
validate_json_request,
with_cancellation,
)
from .protocol import PoolingRequest
from .serving import ServingPooling
router = APIRouter()
def pooling(request: Request) -> ServingPooling:
handler = getattr(request.app.state, "serving_pooling", None)
if handler is None:
raise NotImplementedError("The model does not support Pooling API")
return handler
@router.post(
"/pooling",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
@with_cancellation
@load_aware_call
async def create_pooling(request: PoolingRequest, raw_request: Request):
handler = pooling(raw_request)
return await handler(request, raw_request)
@@ -1,187 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
from typing import Any
from vllm import PoolingParams, PoolingRequestOutput
from vllm.plugins.io_processors import get_io_processor
from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq
from ..base.io_processor import PoolingIOProcessor
from ..typing import (
AnyOfflineInputsContext,
AnyRenderParam,
EncodeCMPLRenderParams,
OfflineEncodeInputsContext,
OfflineOutputsContext,
OfflinePluginInputsContext,
PoolingServeContext,
RequestFactory,
)
from .protocol import IOProcessorRequest, IOProcessorResponse
class UnsupportedCombinedTaskIOProcessor(PoolingIOProcessor):
name = "embed&token_classify"
def create_pooling_params(self, request):
raise ValueError(
"The 'embed&token_classify' pooling task is only available "
"through an IO processor plugin. Send a plugin request with "
"a 'data' field, "
"or select a concrete task with --pooler-config.task."
)
class PluginWithoutIOProcessorPlugins(PoolingIOProcessor):
# Some models, such as Terratorch (tests/models/test_terratorch.py),
# use plugin tasks in the pooler but do not use IO Processor plugins.
name = "plugin"
class PluginWithIOProcessorPlugins(PoolingIOProcessor):
"""IO Processor plugins are a feature that allows pre- and post-processing
of the model input and output for pooling models."""
name = "plugin"
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
io_processor = get_io_processor(
self.vllm_config,
self.renderer,
self.model_config.io_processor_plugin,
)
assert io_processor is not None
self.io_processor = io_processor
#######################################
# online APIs
def get_request_factory_online(
self, ctx: PoolingServeContext
) -> Sequence[AnyRenderParam]:
assert isinstance(ctx.request, IOProcessorRequest)
validated_prompt = self.io_processor.parse_data(ctx.request.data)
raw_prompts = self.io_processor.pre_process(
prompt=validated_prompt, request_id=ctx.request_id
)
parsed_prompts = [
(
prompt
if isinstance(prompt, bytes)
else parse_model_prompt(self.model_config, prompt)
)
for prompt in prompt_to_seq(raw_prompts)
]
num_requests = len(parsed_prompts)
tok_params = ctx.request.build_tok_params(self.model_config)
pooling_params = self.io_processor.merge_pooling_params()
if pooling_params.task is None:
pooling_params.task = "plugin"
ctx.pooling_params = pooling_params
params_seq = self._params_to_seq(ctx.pooling_params, num_requests)
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
requests = [
EncodeCMPLRenderParams(
prompts=parsed_prompts[i],
tok_params=tok_params,
prompt_extras=ctx.prompt_extras,
skip_mm_cache=False,
params=params_seq[i],
lora_requests=seq_lora_requests[i],
priorities=seq_priority[i],
)
for i in range(num_requests)
]
return requests
def post_process_online(
self,
ctx: PoolingServeContext,
):
output = self.io_processor.post_process(
ctx.final_res_batch,
request_id=ctx.request_id,
)
ctx.response = IOProcessorResponse(request_id=ctx.request_id, data=output)
#######################################
# offline APIs
def get_request_factory_offline(
self, ctx: AnyOfflineInputsContext
) -> tuple[RequestFactory, int]:
assert isinstance(ctx, OfflinePluginInputsContext)
assert isinstance(ctx.prompts, dict) and "data" in ctx.prompts
# Validate the request data is valid for the loaded plugin
prompt_data = ctx.prompts.get("data")
if prompt_data is None:
raise ValueError(
"The 'data' field of the prompt is expected to contain "
"the prompt data and it cannot be None. "
"Refer to the documentation of the IOProcessor "
"in use for more details."
)
validated_prompt = self.io_processor.parse_data(prompt_data)
# obtain the actual model prompts from the pre-processor
prompts = self.io_processor.pre_process(prompt=validated_prompt)
prompts_seq = prompt_to_seq(prompts)
num_requests = len(prompts_seq)
pooling_params: PoolingParams | Sequence[PoolingParams]
if ctx.pooling_params is None:
pooling_params = PoolingParams()
else:
pooling_params = ctx.pooling_params
params_seq: list[PoolingParams] = [
self.io_processor.merge_pooling_params(param.clone())
for param in self._params_to_seq(
pooling_params,
num_requests,
)
]
for p in params_seq:
if p.task is None:
p.task = "plugin"
return super().get_request_factory_offline(
OfflineEncodeInputsContext(
prompts=prompts_seq,
pooling_params=params_seq,
pooling_task="plugin",
tokenization_kwargs=ctx.tokenization_kwargs,
lora_request=ctx.lora_request,
priorities=ctx.priorities,
)
)
def post_process_offline(
self,
ctx: OfflineOutputsContext,
) -> list[PoolingRequestOutput]:
processed_outputs = self.io_processor.post_process(ctx.outputs)
return [
PoolingRequestOutput[Any](
request_id="",
outputs=processed_outputs,
num_cached_tokens=getattr(processed_outputs, "num_cached_tokens", 0),
prompt_token_ids=[],
finished=True,
)
]
@@ -1,120 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from typing import Annotated, Generic, TypeAlias, TypeVar
from pydantic import BeforeValidator, Field
from vllm import PoolingParams
from vllm.config import ModelConfig
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
from vllm.renderers import TokenizeParams
from vllm.tasks import PoolingTask
from vllm.utils import random_uuid
from ..base.protocol import (
ChatRequestMixin,
ClassifyRequestMixin,
CompletionRequestMixin,
EmbedRequestMixin,
EncodingRequestMixin,
FixedMaxLenTokenizeParamsMixin,
PoolingBasicRequestMixin,
reject_removed_pooling_parameters,
)
class PoolingCompletionRequest(
PoolingBasicRequestMixin,
CompletionRequestMixin,
EmbedRequestMixin,
ClassifyRequestMixin,
FixedMaxLenTokenizeParamsMixin,
):
task: PoolingTask | None = None
def to_pooling_params(self):
return PoolingParams(
task=self.task,
use_activation=self.use_activation,
dimensions=self.dimensions,
)
class PoolingChatRequest(
PoolingBasicRequestMixin,
ChatRequestMixin,
EmbedRequestMixin,
ClassifyRequestMixin,
FixedMaxLenTokenizeParamsMixin,
):
task: PoolingTask | None = None
def to_pooling_params(self):
return PoolingParams(
task=self.task,
use_activation=self.use_activation,
dimensions=self.dimensions,
)
T = TypeVar("T")
class IOProcessorRequest(PoolingBasicRequestMixin, EncodingRequestMixin, Generic[T]):
data: T
task: PoolingTask = "plugin"
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
return self._build_pooling_tok_params(
model_config,
add_special_tokens=not model_config.is_encoder_decoder,
max_total_tokens=model_config.max_model_len,
max_output_tokens=0,
)
def to_pooling_params(self):
return PoolingParams(
task=self.task,
)
class IOProcessorResponse(OpenAIBaseModel, Generic[T]):
request_id: str | None = None
"""
The request_id associated with this response
"""
created_at: int = Field(default_factory=lambda: int(time.time()))
data: T
"""
When using plugins IOProcessor plugins, the actual output is generated
by the plugin itself. Hence, we use a generic type for the response data
"""
PoolingRequest: TypeAlias = Annotated[
PoolingCompletionRequest | PoolingChatRequest | IOProcessorRequest,
BeforeValidator(reject_removed_pooling_parameters),
]
class PoolingResponseData(OpenAIBaseModel):
index: int
object: str = "pooling"
data: list[list[float]] | list[float] | str
class PoolingResponse(OpenAIBaseModel):
id: str = Field(default_factory=lambda: f"pool-{random_uuid()}")
object: str = "list"
created: int = Field(default_factory=lambda: int(time.time()))
model: str
data: list[PoolingResponseData]
usage: UsageInfo
class PoolingBytesResponse(OpenAIBaseModel):
content: list[bytes]
headers: dict[str, str] | None = None
media_type: str = "application/octet-stream"
-191
View File
@@ -1,191 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import assert_never, cast
from fastapi.responses import JSONResponse, Response, StreamingResponse
from vllm.exceptions import VLLMValidationError
from vllm.logger import init_logger
from vllm.outputs import PoolingRequestOutput
from vllm.tasks import SupportedTask
from vllm.utils.serial_utils import EmbedDType, Endianness
from ..base.io_processor import PoolingIOProcessor
from ..base.serving import PoolingBaseServing
from ..factories import init_pooling_io_processors
from ..typing import AnyPoolingRequest, PoolingServeContext
from ..utils import (
BytesEncodingFormat,
JsonEncodingFormat,
build_pooling_bytes_streaming_response,
get_json_response_cls,
get_pooling_output_encoder,
get_pooling_usage,
)
from .protocol import (
IOProcessorRequest,
PoolingRequest,
PoolingResponse,
PoolingResponseData,
)
logger = init_logger(__name__)
class ServingPooling(PoolingBaseServing):
request_id_prefix = "pooling"
def __init__(
self,
*args,
supported_tasks: tuple[SupportedTask, ...],
**kwargs,
):
super().__init__(*args, **kwargs)
self.supported_tasks = supported_tasks
self.pooling_task = self.model_config.get_pooling_task(supported_tasks)
self.io_processors = init_pooling_io_processors(
supported_tasks=supported_tasks,
vllm_config=self.vllm_config,
renderer=self.renderer,
chat_template_config=self.chat_template_config,
)
self.json_response_cls = get_json_response_cls()
def get_io_processor(self, request: AnyPoolingRequest) -> PoolingIOProcessor:
request = cast(PoolingRequest, request)
pooling_task = self._verify_pooling_task(request)
return self.io_processors[pooling_task]
def _verify_pooling_task(self, request: PoolingRequest) -> str:
if getattr(request, "dimensions", None) is not None:
raise VLLMValidationError(
"dimensions is currently not supported", parameter="dimensions"
)
if request.task is None:
request.task = self.pooling_task
if isinstance(request, IOProcessorRequest):
request.task = "plugin"
assert request.task is not None
pooling_task = request.task
# plugin task uses io_processor.parse_data to verify inputs
if pooling_task != "plugin" and pooling_task != self.pooling_task:
if pooling_task not in self.supported_tasks:
raise VLLMValidationError(
f"Unsupported task: {pooling_task!r} "
f"Supported tasks: {self.supported_tasks}",
parameter="task",
)
else:
raise VLLMValidationError(
"Try switching the model's pooling_task "
f"via --pooler-config.task {request.task}.",
parameter="task",
)
if pooling_task == "plugin" and "plugin" not in self.io_processors:
raise VLLMValidationError(
"No IOProcessor plugin installed. Please refer "
"to the documentation and to the "
"'prithvi_geospatial_mae_io_processor' "
"offline inference example for more details.",
parameter="task",
)
return pooling_task
def _build_response(
self,
ctx: PoolingServeContext,
) -> Response:
if ctx.response is not None:
# for IOProcessorResponse
return self.json_response_cls(content=ctx.response.model_dump())
encoding_format = ctx.request.encoding_format
embed_dtype = ctx.request.embed_dtype
endianness = ctx.request.endianness
if encoding_format == "float" or encoding_format == "base64":
return self.request_output_to_pooling_json_response(
ctx.final_res_batch,
ctx.request_id,
ctx.created_time,
ctx.model_name,
encoding_format,
embed_dtype,
endianness,
)
if encoding_format == "bytes" or encoding_format == "bytes_only":
return self.request_output_to_pooling_bytes_response(
ctx.final_res_batch,
ctx.request_id,
ctx.created_time,
ctx.model_name,
encoding_format,
embed_dtype,
endianness,
)
assert_never(encoding_format)
def request_output_to_pooling_json_response(
self,
final_res_batch: list[PoolingRequestOutput],
request_id: str,
created_time: int,
model_name: str,
encoding_format: JsonEncodingFormat,
embed_dtype: EmbedDType,
endianness: Endianness,
) -> JSONResponse:
encode_fn = get_pooling_output_encoder(
encoding_format=encoding_format,
embed_dtype=embed_dtype,
endianness=endianness,
)
items: list[PoolingResponseData] = []
for idx, final_res in enumerate(final_res_batch):
item = PoolingResponseData(
index=idx,
data=encode_fn(final_res),
)
items.append(item)
response = PoolingResponse(
id=request_id,
created=created_time,
model=model_name,
data=items,
usage=get_pooling_usage(final_res_batch),
)
return self.json_response_cls(content=response.model_dump())
def request_output_to_pooling_bytes_response(
self,
final_res_batch: list[PoolingRequestOutput],
request_id: str,
created_time: int,
model_name: str,
encoding_format: BytesEncodingFormat,
embed_dtype: EmbedDType,
endianness: Endianness,
) -> StreamingResponse:
return build_pooling_bytes_streaming_response(
pooling_outputs=final_res_batch,
request_id=request_id,
created_time=created_time,
model_name=model_name,
encoding_format=encoding_format,
embed_dtype=embed_dtype,
endianness=endianness,
)
@@ -1,111 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from http import HTTPStatus
from fastapi import APIRouter, Depends, Request
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.utils.api_utils import (
load_aware_call,
validate_json_request,
with_cancellation,
)
from vllm.logger import init_logger
from .protocol import RerankRequest, ScoreRequest
from .serving import ServingScores
router = APIRouter()
logger = init_logger(__name__)
def score(request: Request) -> ServingScores:
handler = getattr(request.app.state, "serving_scores", None)
if handler is None:
raise NotImplementedError("The model does not support Score API")
return handler
def rerank(request: Request) -> ServingScores:
handler = getattr(request.app.state, "serving_scores", None)
if handler is None:
raise NotImplementedError("The model does not support Rerank (Score) API")
return handler
@router.post(
"/score",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
@with_cancellation
@load_aware_call
async def create_score(request: ScoreRequest, raw_request: Request):
handler = score(raw_request)
return await handler(request, raw_request)
@router.post(
"/v1/score",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def create_score_v1(request: ScoreRequest, raw_request: Request):
logger.warning(
"To indicate that Score API is not part of standard OpenAI API, we "
"have moved it to `/score`. Please update your client accordingly."
)
return await create_score(request, raw_request)
@router.post(
"/rerank",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
@with_cancellation
@load_aware_call
async def do_rerank(request: RerankRequest, raw_request: Request):
handler = rerank(raw_request)
return await handler(request, raw_request)
@router.post(
"/v1/rerank",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def do_rerank_v1(request: RerankRequest, raw_request: Request):
logger.warning_once(
"To indicate that the rerank API is not part of the standard OpenAI"
" API, we have located it at `/rerank`. Please update your client "
"accordingly. (Note: Conforms to JinaAI rerank API)"
)
return await do_rerank(request, raw_request)
@router.post(
"/v2/rerank",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def do_rerank_v2(request: RerankRequest, raw_request: Request):
return await do_rerank(request, raw_request)
@@ -1,978 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from collections.abc import Sequence
from typing import Any, TypeAlias, cast
import torch.nn.functional as F
from vllm import PoolingParams, PoolingRequestOutput, TokensPrompt
from vllm.exceptions import ChatTemplateResolutionError
from vllm.logger import init_logger
from vllm.renderers import TokenizeParams
from vllm.renderers.hf import safe_apply_chat_template
from vllm.renderers.inputs.preprocess import (
extract_target_prompt,
parse_model_prompt,
prompt_to_seq,
)
from vllm.tasks import PoolingTask
from vllm.utils.mistral import is_mistral_tokenizer
from ..base.io_processor import PoolingIOProcessor
from ..pooling.protocol import PoolingCompletionRequest
from ..typing import (
AnyOfflineInputsContext,
AnyPoolingRequest,
AnyRenderParam,
EncodeChatRenderParams,
EncodeCMPLRenderParams,
OfflineEncodeInputsContext,
OfflineOutputsContext,
OfflineScoringInputsContext,
PoolingEngineInput,
PoolingServeContext,
RequestFactory,
RequestGenerator,
ScoringRenderParams,
)
from .protocol import RerankRequest, ScoreRequest, ScoringRequest
from .typing import ScoreData, ScoreInput, ScoringData
from .utils import (
compress_token_type_ids,
compute_maxsim_score,
get_num_special_tokens_for_pair,
parse_score_data,
score_data_to_prompts,
truncate_text_to_tokens,
validate_score_input,
)
logger = init_logger(__name__)
ScoringServeContext: TypeAlias = PoolingServeContext[ScoringRequest]
def _apply_post_tokenization_to_token_type_ids(
tokenizer: Any,
tok_params: TokenizeParams,
token_type_ids: list[int],
) -> list[int]:
# Must stay in the same order as `TokenizeParams._validate_tokens`, which
# truncates before padding. These ids are parallel to the prompt tokens
# and locate the query/document boundary for the cross-encoder; applying
# the two steps in a different order silently misplaces that boundary.
max_length = tok_params.truncate_prompt_tokens
if max_length is not None and max_length < 0:
max_length = tok_params.max_input_tokens
if max_length is not None and max_length < len(token_type_ids):
if max_length == 0:
token_type_ids = token_type_ids[:0]
else:
side = tok_params.truncation_side or (
tokenizer.truncation_side if tokenizer is not None else None
)
token_type_ids = (
token_type_ids[-max_length:]
if side == "left"
else token_type_ids[:max_length]
)
pad_length = tok_params.pad_prompt_tokens
if pad_length is not None and pad_length < 0:
pad_length = tok_params.max_input_tokens
if pad_length is not None and pad_length > len(token_type_ids):
pad_token_type_id = getattr(tokenizer, "pad_token_type_id", 0) or 0
token_type_ids = token_type_ids + [pad_token_type_id] * (
pad_length - len(token_type_ids)
)
return token_type_ids
class ScoringIOProcessor(PoolingIOProcessor):
name: str
pooling_task: PoolingTask
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.tokenizer = self.renderer.get_tokenizer()
self.architecture = self.model_config.architecture
self.is_multimodal_model = self.model_config.is_multimodal_model
self.pad_token_id = self.tokenizer.pad_token_id
def create_pooling_params(self, request):
return request.to_pooling_params(self.pooling_task)
def _validate_token_limit(self, value: int, name: str) -> None:
if value < 0:
raise ValueError(f"{name} must be a non-negative integer")
if value >= self.model_config.max_model_len:
raise ValueError(
f"{name} ({value}) must be less "
f"than max_model_len ({self.model_config.max_model_len})."
)
def _get_token_limits(
self,
request: ScoringRequest | None = None,
pooling_params: PoolingParams | None = None,
) -> tuple[int, int]:
"""Extract and validate token limits from request or pooling_params."""
if request is not None:
max_tokens_per_query = getattr(request, "max_tokens_per_query", 0)
max_tokens_per_doc = getattr(request, "max_tokens_per_doc", 0)
else:
extra = (
(pooling_params.extra_kwargs or {})
if pooling_params is not None
else {}
)
max_tokens_per_query = extra.get("max_tokens_per_query", 0)
max_tokens_per_doc = extra.get("max_tokens_per_doc", 0)
if max_tokens_per_query != 0:
self._validate_token_limit(max_tokens_per_query, "max_tokens_per_query")
if max_tokens_per_doc != 0:
self._validate_token_limit(max_tokens_per_doc, "max_tokens_per_doc")
return max_tokens_per_query, max_tokens_per_doc
def _truncate_scoring_data(
self,
scoring_data: ScoringData,
max_tokens_per_query: int = 0,
max_tokens_per_doc: int = 0,
) -> ScoringData:
"""Truncate query/document texts to token limits."""
data_1 = scoring_data.data_1
data_2 = scoring_data.data_2
if max_tokens_per_query > 0:
data_1 = [
truncate_text_to_tokens(d, self.tokenizer, max_tokens_per_query)
if isinstance(d, str)
else d
for d in data_1
]
if max_tokens_per_doc > 0:
data_2 = [
truncate_text_to_tokens(d, self.tokenizer, max_tokens_per_doc)
if isinstance(d, str)
else d
for d in data_2
]
return ScoringData(data_1=data_1, data_2=data_2)
def valid_inputs(
self,
data_1: ScoreInput | list[ScoreInput],
data_2: ScoreInput | list[ScoreInput],
) -> ScoringData:
scoring_data = validate_score_input(
data_1,
data_2,
is_multimodal_model=self.is_multimodal_model,
architecture=self.architecture,
)
return scoring_data
def valid_inputs_online(self, request: AnyPoolingRequest):
if isinstance(request, ScoreRequest):
data_1 = request.data_1
data_2 = request.data_2
elif isinstance(request, RerankRequest):
data_1 = request.query
data_2 = request.documents
else:
raise ValueError(f"Invalid {request.__class__.__name__} request type")
scoring_data = self.valid_inputs(data_1, data_2)
return scoring_data
def _pair_error_output(
self,
output_1: PoolingRequestOutput,
output_2: PoolingRequestOutput,
) -> PoolingRequestOutput | None:
failed = output_1 if output_1.error is not None else output_2
if failed.error is None:
return None
padding = [self.pad_token_id] if self.pad_token_id is not None else []
return PoolingRequestOutput(
request_id=f"{output_1.request_id}_{output_2.request_id}",
outputs=failed.outputs,
prompt_token_ids=(
output_1.prompt_token_ids + padding + output_2.prompt_token_ids
),
num_cached_tokens=(output_1.num_cached_tokens + output_2.num_cached_tokens),
finished=failed.finished,
error=failed.error,
)
class BiEncoderIOProcessor(ScoringIOProcessor):
name = "bi-encoder"
pooling_task: PoolingTask = "embed"
#######################################
# online APIs
def get_request_factory_online(
self, ctx: PoolingServeContext
) -> Sequence[AnyRenderParam]:
request = ctx.request
scoring_data = self.valid_inputs_online(request)
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
request=request
)
if max_tokens_per_query > 0 or max_tokens_per_doc > 0:
scoring_data = self._truncate_scoring_data(
scoring_data, max_tokens_per_query, max_tokens_per_doc
)
data_1 = score_data_to_prompts(scoring_data.data_1, "query", self.model_config)
data_2 = score_data_to_prompts(
scoring_data.data_2, "document", self.model_config
)
prompts = data_1 + data_2
ctx.n_queries = len(data_1)
prompts_seq = prompt_to_seq(prompts)
parsed_prompts = [
parse_model_prompt(self.model_config, prompt) for prompt in prompts_seq
]
num_requests = len(parsed_prompts)
tok_params = request.build_tok_params(self.model_config)
params_seq = self._params_to_seq(ctx.pooling_params, num_requests)
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
requests = [
EncodeCMPLRenderParams(
prompts=parsed_prompts[i],
tok_params=tok_params,
prompt_extras=ctx.prompt_extras,
skip_mm_cache=False,
params=params_seq[i],
lora_requests=seq_lora_requests[i],
priorities=seq_priority[i],
)
for i in range(num_requests)
]
return requests
def post_process_online(
self,
ctx: ScoringServeContext,
):
assert ctx.final_res_batch is not None
assert isinstance(ctx.n_queries, int)
ctx.final_res_batch = self._post_process(
outputs=ctx.final_res_batch, n_queries=ctx.n_queries
)
#######################################
# offline APIs
def get_request_factory_offline(
self, ctx: AnyOfflineInputsContext
) -> tuple[RequestFactory, int]:
assert isinstance(ctx, OfflineScoringInputsContext)
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
pooling_params=ctx.pooling_params
)
scoring_data = ctx.scoring_data
if max_tokens_per_query > 0 or max_tokens_per_doc > 0:
scoring_data = self._truncate_scoring_data(
scoring_data, max_tokens_per_query, max_tokens_per_doc
)
data_1 = score_data_to_prompts(scoring_data.data_1, "query", self.model_config)
data_2 = score_data_to_prompts(
scoring_data.data_2, "document", self.model_config
)
prompts = data_1 + data_2
return super().get_request_factory_offline(
OfflineEncodeInputsContext(
pooling_task=self.pooling_task,
prompts=prompts,
tokenization_kwargs=ctx.tokenization_kwargs,
pooling_params=ctx.pooling_params,
lora_request=ctx.lora_request,
priorities=ctx.priorities,
)
)
def post_process_offline(
self,
ctx: OfflineOutputsContext,
) -> list[PoolingRequestOutput]:
assert ctx.n_queries is not None
return self._post_process(outputs=ctx.outputs, n_queries=ctx.n_queries)
#######################################
# helpers
def _post_process(self, outputs: list[PoolingRequestOutput], n_queries: int):
emb_data_1 = outputs[:n_queries]
emb_data_2 = outputs[n_queries:]
if len(emb_data_1) == 1:
emb_data_1 = emb_data_1 * len(emb_data_2)
final_res_batch: list[PoolingRequestOutput] = []
for emb_1, emb_2 in zip(emb_data_1, emb_data_2):
if error_output := self._pair_error_output(emb_1, emb_2):
final_res_batch.append(error_output)
continue
pair_score = F.cosine_similarity(
emb_1.outputs.data.float(), emb_2.outputs.data.float(), dim=0
)
padding: list[int] = []
if self.pad_token_id is not None:
padding = [self.pad_token_id]
tokens = emb_1.prompt_token_ids + padding + emb_2.prompt_token_ids
final_res_batch.append(
PoolingRequestOutput(
request_id=f"{emb_1.request_id}_{emb_2.request_id}",
outputs=pair_score,
prompt_token_ids=tokens,
num_cached_tokens=emb_1.num_cached_tokens + emb_2.num_cached_tokens,
finished=True,
)
)
return final_res_batch
class LateInteractionIOProcessor(BiEncoderIOProcessor):
name = "late-interaction"
pooling_task: PoolingTask = "token_embed"
def _post_process(self, outputs: list[PoolingRequestOutput], n_queries: int):
# Split into query and document embeddings
emb_data_1 = outputs[:n_queries]
emb_data_2 = outputs[n_queries:]
# Expand queries if 1:N scoring
if len(emb_data_1) == 1:
emb_data_1 = emb_data_1 * len(emb_data_2)
final_res_batch: list[PoolingRequestOutput] = []
padding: list[int] = []
if (pad_token_id := self.pad_token_id) is not None:
padding = [pad_token_id]
# Compute MaxSim scores
for emb_1, emb_2 in zip(emb_data_1, emb_data_2):
if error_output := self._pair_error_output(emb_1, emb_2):
final_res_batch.append(error_output)
continue
# emb_1.outputs.data: [query_len, dim]
# emb_2.outputs.data: [doc_len, dim]
q_emb = emb_1.outputs.data
d_emb = emb_2.outputs.data
maxsim_score = compute_maxsim_score(q_emb, d_emb)
tokens = emb_1.prompt_token_ids + padding + emb_2.prompt_token_ids
final_res_batch.append(
PoolingRequestOutput(
request_id=f"{emb_1.request_id}_{emb_2.request_id}",
outputs=maxsim_score,
prompt_token_ids=tokens,
num_cached_tokens=emb_1.num_cached_tokens + emb_2.num_cached_tokens,
finished=True,
)
)
return final_res_batch
class FlashLateInteractionIOProcessor(LateInteractionIOProcessor):
name = "flash-late-interaction"
def post_process_online(
self,
ctx: ScoringServeContext,
):
assert ctx.query_final_res_batch is not None
assert ctx.final_res_batch is not None
assert isinstance(ctx.n_queries, int)
# Expand queries if 1:N scoring
if len(ctx.query_final_res_batch) == 1:
ctx.query_final_res_batch = ctx.query_final_res_batch * len(
ctx.final_res_batch
)
final_res_batch: list[PoolingRequestOutput] = []
for d1, d2 in zip(ctx.query_final_res_batch, ctx.final_res_batch):
padding: list[int] = []
if (pad_token_id := self.pad_token_id) is not None:
padding = [pad_token_id]
tokens = d1.prompt_token_ids + padding + d2.prompt_token_ids
final_res_batch.append(
PoolingRequestOutput(
request_id=f"{d1.request_id}_{d2.request_id}",
outputs=d2.outputs,
prompt_token_ids=tokens,
num_cached_tokens=d1.num_cached_tokens + d2.num_cached_tokens,
finished=True,
)
)
ctx.final_res_batch = final_res_batch
class CrossEncoderIOProcessor(ScoringIOProcessor):
name = "cross-encoder"
pooling_task: PoolingTask = "classify"
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if is_mistral_tokenizer(self.tokenizer):
raise ValueError("MistralTokenizer not supported for cross-encoding")
from vllm.model_executor.model_loader import get_model_cls
from vllm.model_executor.models.interfaces import supports_score_template
model = get_model_cls(self.model_config)
self.supports_score_template = supports_score_template(model)
self.model = model if self.supports_score_template else None
self.use_sep_token = self.model_config.use_sep_token
if (
getattr(self.model_config.hf_config, "is_original_qwen3_reranker", False)
and self.chat_template is None
):
suggested_template = (
"examples/pooling/score/template/qwen3_vl_reranker.jinja"
if self.model_config.is_multimodal_model
else "examples/pooling/score/template/qwen3_reranker.jinja"
)
logger.warning(
"Serving an original Qwen3 reranker (%s) without a "
"--chat-template. The model was trained with a specific prompt "
"template; running without it may produce inaccurate relevance "
"scores. Consider specifying `--chat-template %s`.",
self.model_config.model,
suggested_template,
)
#######################################
# online APIs
def get_request_factory_online(
self, ctx: PoolingServeContext
) -> Sequence[AnyRenderParam]:
request = ctx.request
scoring_data = self.valid_inputs_online(request)
data_1 = scoring_data.data_1
data_2 = scoring_data.data_2
num_requests = len(data_2)
if len(data_1) == 1:
data_1 = data_1 * num_requests
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
request=request
)
tok_params = request.build_tok_params(self.model_config)
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
requests = [
ScoringRenderParams(
data_1=data_1[i],
data_2=data_2[i],
chat_template=self.chat_template,
max_tokens_per_query=max_tokens_per_query,
max_tokens_per_doc=max_tokens_per_doc,
tok_params=tok_params,
prompt_extras=ctx.prompt_extras,
skip_mm_cache=False,
params=ctx.pooling_params,
lora_requests=seq_lora_requests[i],
priorities=seq_priority[i],
)
for i in range(num_requests)
]
return requests
#######################################
# offline APIs
def get_request_factory_offline(
self, ctx: AnyOfflineInputsContext
) -> tuple[RequestFactory, int]:
assert isinstance(ctx, OfflineScoringInputsContext)
data_1 = ctx.scoring_data.data_1
data_2 = ctx.scoring_data.data_2
num_requests = len(data_2)
if len(data_1) == 1:
data_1 = data_1 * num_requests
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
pooling_params=ctx.pooling_params
)
tok_params = self.renderer.default_cmpl_tok_params.with_kwargs(
**(ctx.tokenization_kwargs or {})
)
prompt_extras = ctx.pooling_params.extra_kwargs
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
def request_factory() -> RequestGenerator:
for i in range(num_requests):
yield ScoringRenderParams(
data_1=data_1[i],
data_2=data_2[i],
chat_template=(
self.chat_template
if ctx.chat_template is None
else ctx.chat_template
),
max_tokens_per_query=max_tokens_per_query,
max_tokens_per_doc=max_tokens_per_doc,
tok_params=tok_params,
prompt_extras=prompt_extras,
skip_mm_cache=False,
params=ctx.pooling_params,
lora_requests=seq_lora_requests[i],
priorities=seq_priority[i],
)
return request_factory, num_requests
#######################################
# helpers
def render(
self,
render_params: EncodeCMPLRenderParams
| EncodeChatRenderParams
| ScoringRenderParams,
) -> PoolingEngineInput:
if "data_1" not in render_params:
raise ValueError(
f"Unsupported render_params type {render_params.__class__.__name__}"
)
render_params = cast(ScoringRenderParams, render_params)
arrival_time = time.time()
tok_params = render_params["tok_params"]
params = render_params["params"]
prompt_extras = render_params["prompt_extras"]
_, engine_prompt = self.get_score_prompt(
data_1=render_params["data_1"],
data_2=render_params["data_2"],
encode_kwargs=tok_params.get_encode_kwargs(),
chat_template=render_params["chat_template"],
max_tokens_per_query=render_params["max_tokens_per_query"],
max_tokens_per_doc=render_params["max_tokens_per_doc"],
chat_template_kwargs=prompt_extras.get("chat_template_kwargs")
if prompt_extras
else None,
)
tok_params.apply_post_tokenization(self.tokenizer, engine_prompt)
if token_type_ids := engine_prompt.pop("token_type_ids", None):
params = params.clone()
compressed = compress_token_type_ids(
_apply_post_tokenization_to_token_type_ids(
self.tokenizer, tok_params, token_type_ids
)
)
params.extra_kwargs = {
**(params.extra_kwargs or {}),
"compressed_token_type_ids": compressed,
}
engine_prompt_extras = (
{
k: v
for k in ("mm_processor_kwargs", "cache_salt")
if (v := prompt_extras.get(k)) is not None
}
if prompt_extras
else None
)
if engine_prompt_extras:
target_prompt = extract_target_prompt(self.model_config, engine_prompt)
target_prompt.update(engine_prompt_extras)
engine_input = self.renderer.process_for_engine(engine_prompt, arrival_time)
return PoolingEngineInput(
prompts=engine_input,
params=params,
lora_requests=render_params["lora_requests"],
priorities=render_params["priorities"],
)
def get_score_prompt(
self,
data_1: ScoreData,
data_2: ScoreData,
encode_kwargs: dict[str, Any],
chat_template: str | None = None,
max_tokens_per_query: int = 0,
max_tokens_per_doc: int = 0,
chat_template_kwargs: dict[str, Any] | None = None,
):
model_config = self.model_config
tokenizer = self.tokenizer
prompt_1, prompt_2, mm_data, mm_uuids = parse_score_data(
data_1,
data_2,
model_config,
)
# Limit the query and document separately before composing them.
if max_tokens_per_query > 0 and isinstance(prompt_1, str):
prompt_1 = truncate_text_to_tokens(
prompt_1, tokenizer, max_tokens_per_query
)
if max_tokens_per_doc > 0 and isinstance(prompt_2, str):
prompt_2 = truncate_text_to_tokens(prompt_2, tokenizer, max_tokens_per_doc)
def default_tokenizer_encode():
local_kwargs = encode_kwargs.copy()
if self.supports_score_template:
assert self.model is not None
full_prompt = self.model.get_score_template(prompt_1, prompt_2)
if full_prompt is None:
raise ValueError("Get empty score template from model")
prompt_inputs = tokenizer(full_prompt, **local_kwargs)
else:
if self.use_sep_token:
# cross_encoder models defaults to using separating token.
if max_tokens_per_doc > 0 and isinstance(prompt_2, str):
query_tokens = tokenizer.encode(
prompt_1, add_special_tokens=False
)
num_special = get_num_special_tokens_for_pair(tokenizer)
doc_limit_max_length = (
len(query_tokens) + max_tokens_per_doc + num_special
)
existing_max_length = local_kwargs.get("max_length")
if existing_max_length is not None:
effective_max_length = min(
doc_limit_max_length, existing_max_length
)
else:
effective_max_length = doc_limit_max_length
local_kwargs["truncation"] = "only_second"
local_kwargs["max_length"] = effective_max_length
prompt_inputs = tokenizer(
text=prompt_1, text_pair=prompt_2, **local_kwargs
)
full_prompt = tokenizer.decode(prompt_inputs["input_ids"])
else:
# `llm as reranker` defaults to not using separating token.
full_prompt = prompt_1 + prompt_2
prompt_inputs = tokenizer(text=full_prompt, **local_kwargs)
return full_prompt, prompt_inputs
# FIXME: For now, we only apply a template when one is explicitly provided.
# We cannot rely on the tokenizer's chat template because many models
# inherit junk templates from their base LLM, which breaks both the models
# and the tests that use them.
if chat_template is None:
full_prompt, prompt_inputs = default_tokenizer_encode()
else:
# FIXME:
# Try applying a score template from the CLI arg or tokenizer_config.json
# If that fails because there is no such template,
# fall back to the default implementation.
try:
_safe_kwargs = chat_template_kwargs or {}
_reserved = {"chat_template", "tools", "tokenize"}
_unexpected = _reserved & _safe_kwargs.keys()
if _unexpected:
raise ValueError(
"chat_template_kwargs contains reserved keys that "
f"conflict with fixed scorer arguments: {_unexpected}"
)
full_prompt = safe_apply_chat_template(
model_config,
tokenizer,
[
{"role": "query", "content": prompt_1},
{"role": "document", "content": prompt_2},
],
chat_template=chat_template,
tools=None,
tokenize=False,
**_safe_kwargs,
)
prompt_inputs = tokenizer(full_prompt, **encode_kwargs)
except ChatTemplateResolutionError:
full_prompt, prompt_inputs = default_tokenizer_encode()
engine_prompt = TokensPrompt(prompt_token_ids=prompt_inputs["input_ids"])
if (token_type_ids := prompt_inputs.get("token_type_ids")) is not None:
engine_prompt["token_type_ids"] = token_type_ids
if self.model is not None:
self.model.post_process_tokens(engine_prompt)
if mm_data is not None:
engine_prompt["multi_modal_data"] = mm_data
if mm_uuids is not None:
engine_prompt["multi_modal_uuids"] = mm_uuids
return full_prompt, engine_prompt
class JinaRankingIOProcessorMixin:
@staticmethod
def format_docs_prompts_func(
query: str,
docs: list[str],
special_tokens: dict[str, str] | None = None,
instruction: str | None = None,
no_thinking: bool = True,
) -> str:
# TODO: Try converting the code below into a chat template.
default_special_tokens = {
"query_embed_token": "<|rerank_token|>",
"doc_embed_token": "<|embed_token|>",
}
if special_tokens is None:
special_tokens = default_special_tokens
def sanitize_input(text: str) -> str:
for token in special_tokens.values():
text = text.replace(token, "")
return text
query = sanitize_input(query)
docs = [sanitize_input(doc) for doc in docs]
prefix = (
"<|im_start|>system\n"
"You are a search relevance expert who can determine a ranking of the passages based on how relevant they are to the query. " # noqa: E501
"If the query is a question, how relevant a passage is depends on how well it answers the question. " # noqa: E501
"If not, try to analyze the intent of the query and assess how well each passage satisfies the intent. " # noqa: E501
"If an instruction is provided, you should follow the instruction when determining the ranking." # noqa: E501
"<|im_end|>\n<|im_start|>user\n"
)
suffix = "<|im_end|>\n<|im_start|>assistant\n"
if no_thinking:
suffix += "<think>\n\n</think>\n\n"
doc_emb_token = special_tokens["doc_embed_token"]
query_emb_token = special_tokens["query_embed_token"]
prompt = (
f"I will provide you with {len(docs)} passages, each indicated by a numerical identifier. " # noqa: E501
f"Rank the passages based on their relevance to query: {query}\n"
)
if instruction:
instruction = sanitize_input(instruction)
prompt += f"<instruct>\n{instruction}\n</instruct>\n"
doc_prompts = [
f'<passage id="{i}">\n{doc}{doc_emb_token}\n</passage>'
for i, doc in enumerate(docs)
]
prompt += "\n".join(doc_prompts) + "\n"
prompt += f"<query>\n{query}{query_emb_token}\n</query>"
return prefix + prompt + suffix
@staticmethod
def ensure_str(data: Sequence[Any]) -> list[str]:
text: list[str] = []
for prompt in data:
if not isinstance(prompt, str):
raise ValueError(
"The JinaForRanking model only supports text as input."
)
text.append(prompt)
return text
class JinaRankingIOProcessor(LateInteractionIOProcessor, JinaRankingIOProcessorMixin):
name = "jina-reranking-scoring"
pooling_task: PoolingTask = "token_embed"
def get_request_factory_online(
self, ctx: PoolingServeContext
) -> Sequence[AnyRenderParam]:
request = ctx.request
ctx.n_queries = 1
prompt_extras = ctx.prompt_extras
scoring_data = self.valid_inputs_online(request)
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
request=request
)
if max_tokens_per_query > 0 or max_tokens_per_doc > 0:
scoring_data = self._truncate_scoring_data(
scoring_data, max_tokens_per_query, max_tokens_per_doc
)
queries = self.ensure_str(scoring_data.data_1)
docs = self.ensure_str(scoring_data.data_2)
chat_template_kwargs = (
prompt_extras.get("chat_template_kwargs") if prompt_extras else None
)
instruction = (
chat_template_kwargs.get("instruction") if chat_template_kwargs else None
)
if len(queries) == 1:
prompts = [
self.format_docs_prompts_func(
query=queries[0], docs=docs, instruction=instruction
)
]
else:
prompts = [
self.format_docs_prompts_func(
query=q, docs=[d], instruction=instruction
)
for q, d in zip(queries, docs)
]
# Forward truncation from the real request: the base factory reads
# these off ctx.request, so omitting them here silently drops
# truncate_prompt_tokens for Jina rerank/score (unlike the embed and
# bi/cross-encoder paths, which read them from the real request).
ctx.request = PoolingCompletionRequest(
task="token_embed",
input=prompts,
truncate_prompt_tokens=request.truncate_prompt_tokens,
truncation_side=request.truncation_side,
)
requests = PoolingIOProcessor.get_request_factory_online(self, ctx)
ctx.request = request
return requests
def get_request_factory_offline(
self, ctx: AnyOfflineInputsContext
) -> tuple[RequestFactory, int]:
assert isinstance(ctx, OfflineScoringInputsContext)
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
pooling_params=ctx.pooling_params
)
scoring_data = ctx.scoring_data
if max_tokens_per_query > 0 or max_tokens_per_doc > 0:
scoring_data = self._truncate_scoring_data(
scoring_data, max_tokens_per_query, max_tokens_per_doc
)
prompt_extras = ctx.pooling_params.extra_kwargs
queries = self.ensure_str(scoring_data.data_1)
docs = self.ensure_str(scoring_data.data_2)
chat_template_kwargs = (
prompt_extras.get("chat_template_kwargs") if prompt_extras else None
)
instruction = (
chat_template_kwargs.get("instruction") if chat_template_kwargs else None
)
if len(queries) == 1:
prompts = [
self.format_docs_prompts_func(
query=queries[0], docs=docs, instruction=instruction
)
]
else:
prompts = [
self.format_docs_prompts_func(
query=q, docs=[d], instruction=instruction
)
for q, d in zip(queries, docs)
]
return PoolingIOProcessor.get_request_factory_offline(
self,
OfflineEncodeInputsContext(
pooling_task=self.pooling_task,
prompts=prompts,
tokenization_kwargs=ctx.tokenization_kwargs,
pooling_params=ctx.pooling_params,
lora_request=ctx.lora_request,
priorities=ctx.priorities,
),
)
def _post_process(self, outputs: list[PoolingRequestOutput], n_queries: int):
final_res_batch: list[PoolingRequestOutput] = []
for i in range(len(outputs)):
if outputs[i].error is not None:
final_res_batch.append(outputs[i])
continue
embeds = outputs[i].outputs.data.float()
# The JinaForRanking model concatenates docs first, then query.
# Let's stay consistent with this novel design.
query_embeds = embeds[-1]
doc_embeds = embeds[:-1]
scores = F.cosine_similarity(query_embeds, doc_embeds)
for score in scores:
final_res_batch.append(
PoolingRequestOutput(
request_id=outputs[i].request_id,
outputs=score,
prompt_token_ids=outputs[i].prompt_token_ids,
num_cached_tokens=outputs[i].num_cached_tokens,
finished=True,
)
)
return final_res_batch
ScoringIOProcessors: dict[str, type[ScoringIOProcessor]] = {
p.name: p
for p in [
BiEncoderIOProcessor,
LateInteractionIOProcessor,
JinaRankingIOProcessor,
FlashLateInteractionIOProcessor,
CrossEncoderIOProcessor,
]
}
@@ -1,187 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from typing import Any, TypeAlias
from pydantic import BaseModel, Field, model_validator
from vllm import PoolingParams
from vllm.config import ModelConfig
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
from vllm.renderers import TokenizeParams
from vllm.tasks import PoolingTask
from vllm.utils import random_uuid
from ..base.protocol import ClassifyRequestMixin, PoolingBasicRequestMixin
from .typing import ScoreContentPartParam, ScoreInput
class ScoringRequestMixin(PoolingBasicRequestMixin, ClassifyRequestMixin):
# --8<-- [start:scoring-common-params]
max_tokens_per_query: int = Field(
default=0,
description=(
"Maximum number of tokens per query. Queries longer than "
"this will be truncated to this length. 0 means no "
"query-level truncation is applied."
),
)
max_tokens_per_doc: int = Field(
default=0,
description=(
"Maximum number of tokens per document. Documents longer than "
"this will be truncated to this length. 0 means no "
"document-level truncation is applied (only truncate_prompt_tokens "
"applies to the combined query+document)."
),
)
instruction: str | None = Field(
default=None,
description=(
"Task instruction prepended to each scored pair via the chat "
"template. Equivalent to passing "
"chat_template_kwargs={'instruction': ...}."
),
)
chat_template_kwargs: dict[str, Any] | None = Field(
default=None,
description=(
"Additional keyword args to pass to the chat template renderer. "
"Will be accessible by the score/rerank chat template."
),
)
# --8<-- [end:scoring-common-params]
@model_validator(mode="after")
def _merge_instruction_into_kwargs(self) -> "ScoringRequestMixin":
"""Fold the top-level `instruction` field into `chat_template_kwargs`.
This allows callers to use either the convenience field or the generic
dict. Explicit keys inside `chat_template_kwargs` take precedence over
the top-level `instruction` field.
"""
if self.instruction is not None:
merged = dict(self.chat_template_kwargs or {})
merged.setdefault("instruction", self.instruction)
self.chat_template_kwargs = merged
return self
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
return self._build_pooling_tok_params(
model_config,
add_special_tokens=True,
max_total_tokens=model_config.max_model_len,
max_output_tokens=0,
)
def to_pooling_params(self, task: PoolingTask = "classify"):
return PoolingParams(
task=task,
use_activation=self.use_activation,
)
class ScoreDataRequest(ScoringRequestMixin):
data_1: ScoreInput | list[ScoreInput]
data_2: ScoreInput | list[ScoreInput]
class ScoreQueriesDocumentsRequest(ScoringRequestMixin):
# --8<-- [start:score-request-params]
queries: ScoreInput | list[ScoreInput]
documents: ScoreInput | list[ScoreInput]
# --8<-- [end:score-request-params]
@property
def data_1(self):
return self.queries
@property
def data_2(self):
return self.documents
class ScoreQueriesItemsRequest(ScoringRequestMixin):
queries: ScoreInput | list[ScoreInput]
items: ScoreInput | list[ScoreInput]
@property
def data_1(self):
return self.queries
@property
def data_2(self):
return self.items
class ScoreTextRequest(ScoringRequestMixin):
text_1: ScoreInput | list[ScoreInput]
text_2: ScoreInput | list[ScoreInput]
@property
def data_1(self):
return self.text_1
@property
def data_2(self):
return self.text_2
ScoreRequest: TypeAlias = (
ScoreQueriesDocumentsRequest
| ScoreQueriesItemsRequest
| ScoreDataRequest
| ScoreTextRequest
)
class RerankRequest(ScoringRequestMixin):
# --8<-- [start:rerank-request-params]
query: ScoreInput
documents: ScoreInput | list[ScoreInput]
top_n: int = Field(default=0, ge=0)
# --8<-- [end:rerank-request-params]
ScoringRequest: TypeAlias = ScoreRequest | RerankRequest
class RerankDocument(BaseModel):
text: str | None = None
multi_modal: list[ScoreContentPartParam] | None = None
class RerankResult(BaseModel):
index: int
document: RerankDocument
relevance_score: float
class RerankUsage(BaseModel):
prompt_tokens: int
total_tokens: int
class RerankResponse(OpenAIBaseModel):
id: str
model: str
usage: RerankUsage
results: list[RerankResult]
class ScoreResponseData(OpenAIBaseModel):
index: int
object: str = "score"
score: float
class ScoreResponse(OpenAIBaseModel):
id: str = Field(default_factory=lambda: f"embd-{random_uuid()}")
object: str = "list"
created: int = Field(default_factory=lambda: int(time.time()))
model: str
data: list[ScoreResponseData]
usage: UsageInfo
ScoringResponse: TypeAlias = RerankResponse | ScoreResponse
-315
View File
@@ -1,315 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from fastapi.responses import JSONResponse, Response
from vllm import PoolingParams
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.serve.engine.protocol import UsageInfo
from vllm.logger import init_logger
from vllm.outputs import PoolingRequestOutput, ScoringRequestOutput
from vllm.tasks import SCORE_TYPE_MAP, SupportedTask
from vllm.utils import random_uuid
from vllm.v1.pool.late_interaction import (
build_late_interaction_doc_params,
build_late_interaction_query_params,
)
from ..base.io_processor import PoolingIOProcessor
from ..base.serving import PoolingServing
from .io_processor import ScoringIOProcessors, ScoringServeContext
from .protocol import (
RerankDocument,
RerankRequest,
RerankResponse,
RerankResult,
RerankUsage,
ScoreRequest,
ScoreResponse,
ScoreResponseData,
)
from .typing import ScoreInput
logger = init_logger(__name__)
class ServingScores(PoolingServing):
request_id_prefix = "score"
def __init__(
self,
engine_client: EngineClient,
*args,
supported_tasks: tuple[SupportedTask, ...],
enable_flash_late_interaction: bool = True,
**kwargs,
):
pooling_task = engine_client.model_config.get_pooling_task(supported_tasks)
score_type = SCORE_TYPE_MAP.get(pooling_task, None) # type: ignore[arg-type]
assert score_type is not None
self.io_processor_name: str = score_type
self.enable_flash_late_interaction = (
self.io_processor_name == "late-interaction"
and enable_flash_late_interaction
)
if self.enable_flash_late_interaction:
self.io_processor_name = "flash-late-interaction"
if engine_client.model_config.architecture == "JinaForRanking":
self.io_processor_name = "jina-reranking-scoring"
self.enable_flash_late_interaction = False
super().__init__(engine_client, *args, **kwargs)
def init_io_processor(self, *args, **kwargs) -> PoolingIOProcessor:
return ScoringIOProcessors[self.io_processor_name](*args, **kwargs)
async def __call__(self, *args, **kwargs) -> Response:
if not self.enable_flash_late_interaction:
return await super().__call__(*args, **kwargs)
return await self.flash_late_interaction(*args, **kwargs)
def _build_response(
self,
ctx: ScoringServeContext,
) -> JSONResponse:
final_res_batch = ctx.final_res_batch
request_id = ctx.request_id
created_time = ctx.created_time
model_name = ctx.model_name
assert ctx.engine_inputs is not None
num_prompt_tokens = sum(
self._extract_prompt_len(engine_input["prompts"])
for engine_input in ctx.engine_inputs
)
if isinstance(ctx.request, ScoreRequest):
return self._request_output_to_score_response(
final_res_batch,
request_id,
created_time,
model_name,
num_prompt_tokens,
)
elif isinstance(ctx.request, RerankRequest):
return self._request_output_to_rerank_response(
final_res_batch,
request_id,
model_name,
ctx.request.documents,
ctx.request.top_n if ctx.request.top_n > 0 else len(final_res_batch),
num_prompt_tokens,
)
else:
raise ValueError(f"Invalid {self.request_id_prefix} request type")
def _request_output_to_score_response(
self,
final_res_batch: list[PoolingRequestOutput],
request_id: str,
created_time: int,
model_name: str,
num_prompt_tokens: int,
) -> JSONResponse:
items: list[ScoreResponseData] = []
for idx, final_res in enumerate(final_res_batch):
classify_res = ScoringRequestOutput.from_base(final_res)
item = ScoreResponseData(
index=idx,
score=classify_res.outputs.score,
)
items.append(item)
usage = UsageInfo(
prompt_tokens=num_prompt_tokens,
total_tokens=num_prompt_tokens,
)
response = ScoreResponse(
id=request_id,
created=created_time,
model=model_name,
data=items,
usage=usage,
)
return JSONResponse(content=response.model_dump())
def _request_output_to_rerank_response(
self,
final_res_batch: list[PoolingRequestOutput],
request_id: str,
model_name: str,
documents: ScoreInput | list[ScoreInput],
top_n: int,
num_prompt_tokens: int,
) -> JSONResponse:
if not isinstance(documents, list):
documents = [documents]
results: list[RerankResult] = []
for idx, final_res in enumerate(final_res_batch):
classify_res = ScoringRequestOutput.from_base(final_res)
document = documents[idx]
if isinstance(document, str):
rerank_document = RerankDocument(text=document)
else:
rerank_document = RerankDocument(
multi_modal=document.get("content", [])
)
result = RerankResult(
index=idx,
document=rerank_document,
relevance_score=classify_res.outputs.score,
)
results.append(result)
# sort by relevance, then return the top n if set
results.sort(key=lambda x: x.relevance_score, reverse=True)
if top_n < len(documents):
results = results[:top_n]
response = RerankResponse(
id=request_id,
model=model_name,
results=results,
usage=RerankUsage(
total_tokens=num_prompt_tokens, prompt_tokens=num_prompt_tokens
),
)
return JSONResponse(content=response.model_dump())
###################################################################################
### Run pooling score MaxSim on worker side (GPU) in the API server process
### Can significantly improve late-interaction scoring performance.
async def flash_late_interaction(self, *args, **kwargs) -> Response:
ctx = await self._init_ctx(self.io_processor, *args, **kwargs)
await self._preprocessing(self.io_processor, ctx)
try:
# stage 1: encode queries and cache token embeddings on workers.
await self._flash_late_interaction_encode_queries(ctx)
# stage 2: encode docs and return scalar scores from workers.
await self._flash_late_interaction_encode_docs(ctx)
except BaseException:
try:
await self._cleanup_flash_late_interaction(ctx)
except Exception:
logger.exception("Failed to clean up late-interaction query cache.")
raise
return await self._postprocessing_async(self.io_processor, ctx)
async def _cleanup_flash_late_interaction(self, ctx: ScoringServeContext) -> None:
query_keys = ctx.late_interaction_query_keys
if not query_keys:
return
doc_keys = ctx.late_interaction_doc_keys or []
try:
# Stop documents before removing the query tensors they reference.
await self.engine_client.abort([*query_keys, *doc_keys])
finally:
await self.engine_client.collective_rpc(
"release_late_interaction_query_cache",
args=(query_keys,),
)
async def _flash_late_interaction_encode_queries(self, ctx: ScoringServeContext):
assert ctx.n_queries is not None
assert ctx.engine_inputs is not None
assert isinstance(ctx.pooling_params, PoolingParams)
n_queries = ctx.n_queries
n_docs = len(ctx.engine_inputs) - n_queries
query_engine_inputs = ctx.engine_inputs[:n_queries]
query_namespace = random_uuid()
query_keys = [
f"late-interaction-{query_namespace}-query-{i}" for i in range(n_queries)
]
ctx.late_interaction_query_keys = query_keys
query_uses = [n_docs if n_queries == 1 else 1] * n_queries
for i in range(n_queries):
pooling_params = ctx.pooling_params.clone()
pooling_params.late_interaction_params = (
build_late_interaction_query_params(
query_key=query_keys[i],
query_uses=query_uses[i],
)
)
query_engine_inputs[i]["params"] = pooling_params
assert n_queries == len(query_engine_inputs) == len(query_keys)
query_ctx = ScoringServeContext(
request=ctx.request,
raw_request=ctx.raw_request,
model_name=ctx.model_name,
request_id=ctx.request_id,
pooling_params=ctx.pooling_params,
prompt_request_ids=query_keys,
engine_inputs=query_engine_inputs,
lora_request=ctx.lora_request,
priorities=ctx.priorities,
prompt_extras=ctx.prompt_extras,
)
await self._prepare_generators(query_ctx)
await self._collect_batch(query_ctx)
ctx.query_final_res_batch = query_ctx.final_res_batch
async def _flash_late_interaction_encode_docs(self, ctx: ScoringServeContext):
assert ctx.n_queries is not None
assert ctx.engine_inputs is not None
assert isinstance(ctx.pooling_params, PoolingParams)
n_queries = ctx.n_queries
n_docs = len(ctx.engine_inputs) - n_queries
doc_engine_inputs = ctx.engine_inputs[n_queries:]
query_keys = ctx.late_interaction_query_keys
if query_keys is None:
raise RuntimeError("Late-interaction query keys were not initialized.")
doc_namespace = random_uuid()
doc_keys = [f"late-interaction-{doc_namespace}-doc-{i}" for i in range(n_docs)]
ctx.late_interaction_doc_keys = doc_keys
for i in range(n_docs):
query_idx = 0 if n_queries == 1 else i
pooling_params = ctx.pooling_params.clone()
pooling_params.late_interaction_params = build_late_interaction_doc_params(
query_key=query_keys[query_idx]
)
doc_engine_inputs[i]["params"] = pooling_params
assert n_docs == len(doc_engine_inputs) == len(doc_keys)
doc_ctx = ScoringServeContext(
request=ctx.request,
raw_request=ctx.raw_request,
model_name=ctx.model_name,
request_id=ctx.request_id,
pooling_params=ctx.pooling_params,
prompt_request_ids=doc_keys,
engine_inputs=doc_engine_inputs,
lora_request=ctx.lora_request,
priorities=ctx.priorities,
prompt_extras=ctx.prompt_extras,
)
await self._prepare_generators(doc_ctx)
await self._collect_batch(doc_ctx)
ctx.final_res_batch = doc_ctx.final_res_batch
@@ -1,45 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import Required, TypeAlias
from typing_extensions import TypedDict
from vllm.renderers.chat_utils import (
ChatCompletionContentPartImageEmbedsParam,
ChatCompletionContentPartImageParam,
ChatCompletionContentPartTextParam,
ChatCompletionContentPartVideoParam,
)
ScoreContentPartParam: TypeAlias = (
ChatCompletionContentPartImageParam
| ChatCompletionContentPartImageEmbedsParam
| ChatCompletionContentPartTextParam
| ChatCompletionContentPartVideoParam
)
class ScoreMultiModalParam(TypedDict, total=False):
"""A specialized parameter type for scoring multimodal content.
The reasons why don't reuse `CustomChatCompletionMessageParam` directly:
1. Score tasks don't need the 'role' field (user/assistant/system) that's required in chat completions
2. Including chat-specific fields would confuse users about their purpose in scoring
3. This is a more focused interface that only exposes what's needed for scoring
""" # noqa: E501
content: Required[list[ScoreContentPartParam]]
"""The multimodal contents"""
# Raw input data with content key in ScoreMultiModalParam.
ScoreInput = str | ScoreMultiModalParam
# Score data without content key.
ScoreData = str | list[ScoreContentPartParam]
@dataclass
class ScoringData:
data_1: list[ScoreData]
data_2: list[ScoreData]
-291
View File
@@ -1,291 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
from typing import cast
import torch
from vllm import PromptType, TextPrompt
from vllm.config import ModelConfig
from vllm.exceptions import VLLMValidationError
from vllm.inputs import MultiModalDataDict, MultiModalUUIDDict
from vllm.renderers.chat_utils import (
BaseMultiModalItemTracker,
ChatCompletionContentPartParam,
ChatCompletionContentPartTextParam,
ConversationMessage,
MultiModalItemTracker,
parse_chat_message_content_parts,
)
from .typing import (
ScoreContentPartParam,
ScoreData,
ScoreInput,
ScoringData,
)
def get_num_special_tokens_for_pair(tokenizer) -> int:
"""Get number of special tokens added for a text pair encoding."""
method = getattr(tokenizer, "num_special_tokens_to_add", None)
if method is not None:
try:
return method(pair=True)
except TypeError:
pass
# Fallback: compute by tokenizing empty strings
empty_encoding = tokenizer("", text_pair="", add_special_tokens=True)
return len(empty_encoding["input_ids"])
def truncate_text_to_tokens(
text: str,
tokenizer,
max_tokens: int,
) -> str:
"""Truncate text to a maximum number of content tokens.
Uses offset_mapping to slice the original text at the exact character
boundary, avoiding lossy encode→decode round-trips that can shift
the token count due to BPE merge boundary changes.
"""
if max_tokens <= 0:
return ""
while text:
encoding = tokenizer(
text,
add_special_tokens=False,
return_offsets_mapping=True,
)
offsets = encoding["offset_mapping"]
if len(encoding["input_ids"]) <= max_tokens:
break
char_end = min(offsets[max_tokens - 1][1], offsets[max_tokens][0])
if char_end >= len(text):
char_end = len(text) - 1
text = text[:char_end]
return text
def compute_maxsim_score(q_emb: torch.Tensor, d_emb: torch.Tensor) -> torch.Tensor:
"""Compute ColBERT MaxSim score.
Args:
q_emb: Query token embeddings [query_len, dim]
d_emb: Document token embeddings [doc_len, dim]
Returns:
MaxSim score (sum over query tokens of max similarity to any doc token)
"""
# compute in float32 for numerical stability
# [query_len, doc_len]
token_scores = torch.matmul(q_emb.float(), d_emb.float().T)
# Max over document tokens, sum over query tokens
return token_scores.amax(dim=-1).sum()
def _validate_mm_score_input(
data: list[ScoreInput],
is_multimodal_model: bool,
architecture: str,
) -> list[ScoreData]:
out: list[ScoreData] = []
for d in data:
if isinstance(d, str):
out.append(d)
else:
if not is_multimodal_model:
raise VLLMValidationError(
f"MultiModalParam is not supported for {architecture}"
)
content = cast(list[ScoreContentPartParam], d.get("content", []))
out.append(content)
return out
def _validate_score_input_lens(
data_1: list[ScoreData],
data_2: list[ScoreData],
):
len_1 = len(data_1)
len_2 = len(data_2)
if len_1 > 1 and len_1 != len_2:
raise VLLMValidationError("Input lengths must be either 1:1, 1:N or N:N")
if len_1 == 0:
raise VLLMValidationError("At least one text element must be given")
if len_2 == 0:
raise VLLMValidationError("At least one text_pair element must be given")
def validate_score_input(
data_1: ScoreInput | list[ScoreInput],
data_2: ScoreInput | list[ScoreInput],
is_multimodal_model: bool,
architecture: str,
) -> ScoringData:
if not isinstance(data_1, list):
data_1 = [data_1]
if not isinstance(data_2, list):
data_2 = [data_2]
score_input_1 = _validate_mm_score_input(data_1, is_multimodal_model, architecture)
score_input_2 = _validate_mm_score_input(data_2, is_multimodal_model, architecture)
_validate_score_input_lens(score_input_1, score_input_2)
return ScoringData(data_1=score_input_1, data_2=score_input_2)
def score_data_to_prompts(
data_list: list[ScoreData],
role: str,
model_config: ModelConfig,
) -> list[PromptType]:
"""Convert a list of ScoreData into PromptType objects.
For plain text inputs, returns the string directly.
For multimodal inputs (list of content parts), parses them into
a :class:`TextPrompt` with attached ``multi_modal_data`` /
``multi_modal_uuids``.
This is used by late-interaction scoring where each query/document
is encoded independently.
"""
prompts: list[PromptType] = []
for data in data_list:
if isinstance(data, str):
prompts.append(data)
else:
text, mm_data, mm_uuids = parse_score_data_single(data, role, model_config)
prompt: TextPrompt = TextPrompt(prompt=text)
if mm_data is not None:
prompt["multi_modal_data"] = mm_data
if mm_uuids is not None:
prompt["multi_modal_uuids"] = mm_uuids
prompts.append(prompt)
return prompts
def _ensure_str(content: list[ConversationMessage]) -> str:
"""Extract a single string prompt from parsed conversation content."""
assert len(content) == 1
prompt = content[0]["content"]
if prompt is not None and isinstance(prompt, str):
return cast(str, prompt)
raise ValueError(f"Only string content is supported, but got {content}.")
def _parse_score_content(
role: str,
data: ScoreData,
mm_tracker: BaseMultiModalItemTracker,
) -> list[ConversationMessage]:
parts: Iterable[ChatCompletionContentPartParam]
if isinstance(data, str):
parts = [ChatCompletionContentPartTextParam(type="text", text=data)]
else:
parts = cast(Iterable[ChatCompletionContentPartParam], data)
mm_parser = mm_tracker.create_parser()
parse_res = parse_chat_message_content_parts(
role=role,
parts=parts,
mm_tracker=mm_tracker,
wrap_dicts=False,
interleave_strings=False,
multimodal_content_part_separator="",
)
if parse_res:
return parse_res
mm_placeholder_storage = mm_parser.mm_placeholder_storage()
if (
len(mm_placeholder_storage) != 1
or len(next(iter(mm_placeholder_storage.values()))) != 1
):
raise ValueError("Only one multi-modal item is supported")
return next(iter(mm_placeholder_storage.values()))[0]
def parse_score_data_single(
data: ScoreData,
role: str,
model_config: ModelConfig,
) -> tuple[str, MultiModalDataDict | None, MultiModalUUIDDict | None]:
"""Parse **one** ScoreData into a text prompt and its own multi-modal
data.
Unlike :func:`parse_score_data`, each call creates an **independent**
:class:`MultiModalItemTracker` so multi-modal items are kept separate.
This is the correct behaviour for late-interaction scoring, where
query and document are encoded independently.
"""
mm_tracker = MultiModalItemTracker(model_config)
content = _parse_score_content(role, data, mm_tracker)
prompt = _ensure_str(content)
mm_items, mm_uuids = mm_tracker.resolve_items()
return prompt, mm_items, mm_uuids
def parse_score_data(
data_1: ScoreData,
data_2: ScoreData,
model_config: ModelConfig,
) -> tuple[str, str, MultiModalDataDict | None, MultiModalUUIDDict | None]:
"""Parse a query-document pair into text prompts and shared multi-modal
data.
Uses a **single** :class:`MultiModalItemTracker` so that multi-modal
items from both inputs are merged into one ``mm_data`` dict. This is
the correct behaviour for cross-encoder scoring, where query and
document are concatenated into a single model prompt.
"""
mm_tracker = MultiModalItemTracker(model_config)
content_1 = _parse_score_content("query", data_1, mm_tracker)
content_2 = _parse_score_content("document", data_2, mm_tracker)
prompt_1 = _ensure_str(content_1)
prompt_2 = _ensure_str(content_2)
mm_items, mm_uuids = mm_tracker.resolve_items()
return prompt_1, prompt_2, mm_items, mm_uuids
def compress_token_type_ids(token_type_ids: list[int]) -> tuple[int, int]:
"""Return the half-open range containing token type 1.
Right padding uses the tokenizer's padding token type, which is commonly
zero. Therefore, a padded query-document pair can contain zeros followed
by ones followed by trailing zeros.
"""
first_one = len(token_type_ids)
first_zero_after_one = len(token_type_ids)
err_msg = (
"Token type ids are expected to be a sequence"
" of zeros followed by ones and optional trailing zeros"
)
for i, type_id in enumerate(token_type_ids):
if type_id not in (0, 1):
raise ValueError(err_msg)
if type_id == 1:
if first_zero_after_one < len(token_type_ids):
raise ValueError(err_msg)
if first_one == len(token_type_ids):
first_one = i
elif first_one < len(token_type_ids) and first_zero_after_one == len(
token_type_ids
):
first_zero_after_one = i
return first_one, first_zero_after_one
-214
View File
@@ -1,214 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from collections.abc import AsyncGenerator, Callable, Generator, Sequence
from dataclasses import dataclass, field
from typing import Any, Generic, NotRequired, TypeAlias, TypedDict, TypeVar
from fastapi import Request
from pydantic import ConfigDict
from vllm import PoolingParams, PoolingRequestOutput, PromptType
from vllm.inputs import DataPrompt, EngineInput
from vllm.lora.request import LoRARequest
from vllm.outputs import LateChunkingMetadata
from vllm.renderers import ChatParams, TokenizeParams
from vllm.renderers.chat_utils import ChatCompletionMessageParam
from vllm.renderers.inputs import DictPrompt
from ...tasks import PoolingTask
from .classify.protocol import (
ClassificationChatRequest,
ClassificationCompletionRequest,
ClassificationResponse,
)
from .embed.protocol import (
CohereEmbedRequest,
EmbeddingBytesResponse,
EmbeddingChatInputRequest,
EmbeddingChatRequest,
EmbeddingCompletionRequest,
EmbeddingRequest,
EmbeddingResponse,
)
from .pooling.protocol import (
IOProcessorRequest,
PoolingBytesResponse,
PoolingChatRequest,
PoolingCompletionRequest,
PoolingResponse,
)
from .scoring.protocol import ScoringRequest, ScoringResponse
from .scoring.typing import ScoreData, ScoringData
PoolingCompletionLikeRequest: TypeAlias = (
EmbeddingCompletionRequest
| ClassificationCompletionRequest
| PoolingCompletionRequest
)
PoolingChatLikeRequest: TypeAlias = (
EmbeddingChatRequest
| EmbeddingChatInputRequest
| ClassificationChatRequest
| PoolingChatRequest
)
AnyPoolingRequest: TypeAlias = (
EmbeddingRequest
| PoolingCompletionLikeRequest
| PoolingChatLikeRequest
| IOProcessorRequest
| ScoringRequest
| CohereEmbedRequest
)
AnyPoolingResponse: TypeAlias = (
ClassificationResponse
| EmbeddingResponse
| EmbeddingBytesResponse
| PoolingResponse
| PoolingBytesResponse
| ScoringResponse
)
PoolingRequestT = TypeVar("PoolingRequestT", bound=AnyPoolingRequest)
@dataclass(kw_only=True)
class ChunkedEmbeddingMetadata:
prompt_index: int
chunk_index: int
@dataclass
class LateChunkingSlot:
"""Per-document source ranges shared by frontend rendering and output contexts."""
metadata: LateChunkingMetadata | None = None
@dataclass(kw_only=True)
class PoolingServeContext(Generic[PoolingRequestT]):
model_config = ConfigDict(arbitrary_types_allowed=True)
request: PoolingRequestT
raw_request: Request | None = None
model_name: str
request_id: str
pooling_params: PoolingParams
lora_request: LoRARequest | None
priorities: int | Sequence[int] | None
prompt_extras: dict[str, Any] | None
created_time: int = field(default_factory=lambda: int(time.time()))
engine_inputs: Sequence["PoolingEngineInput"] | None = None
prompt_request_ids: list[str] | None = None
result_generator: AsyncGenerator[tuple[int, PoolingRequestOutput], None] | None = (
None
)
final_res_batch: list[PoolingRequestOutput] = field(default_factory=list)
## for Long Text Embedding with Chunked Processing
original_engine_inputs: Sequence["PoolingEngineInput"] | None = None
chunked_embedding_metadata: list[ChunkedEmbeddingMetadata] | None = None
## for bi-encoder & late-interaction
n_queries: int | None = None
## for IOProcessorResponse
response: Any | None = None
## for flash-late-interaction
query_final_res_batch: list[PoolingRequestOutput] | None = None
late_interaction_query_keys: list[str] | None = None
late_interaction_doc_keys: list[str] | None = None
@dataclass
class OfflineInputsContext:
pooling_task: PoolingTask
tokenization_kwargs: dict[str, Any] | None
lora_request: Sequence[LoRARequest | None] | None
priorities: int | Sequence[int] | None
late_chunking: list[LateChunkingSlot | None] = field(
default_factory=list, kw_only=True
)
@dataclass
class OfflineEncodeInputsContext(OfflineInputsContext):
prompts: PromptType | Sequence[PromptType]
pooling_params: PoolingParams | Sequence[PoolingParams] | None
@dataclass
class OfflineScoringInputsContext(OfflineInputsContext):
scoring_data: ScoringData
chat_template: str | None
pooling_params: PoolingParams
@dataclass
class OfflinePluginInputsContext(OfflineInputsContext):
prompts: DataPrompt
pooling_params: PoolingParams | Sequence[PoolingParams] | None
AnyOfflineInputsContext: TypeAlias = (
OfflineEncodeInputsContext
| OfflineScoringInputsContext
| OfflinePluginInputsContext
)
@dataclass
class OfflineOutputsContext:
outputs: list[PoolingRequestOutput]
## for bi-encoder & late-interaction
n_queries: int | None = None
late_chunking: list[LateChunkingSlot | None] = field(default_factory=list)
class RenderParams(TypedDict):
tok_params: TokenizeParams
prompt_extras: dict[str, Any] | None
skip_mm_cache: bool
params: PoolingParams
lora_requests: LoRARequest | None
priorities: int
late_chunking: NotRequired[LateChunkingSlot]
class EncodeCMPLRenderParams(RenderParams):
prompts: DictPrompt
class EncodeChatRenderParams(RenderParams):
conversations: list["ChatCompletionMessageParam"]
chat_params: ChatParams
class ScoringRenderParams(RenderParams):
data_1: ScoreData
data_2: ScoreData
chat_template: str | None
max_tokens_per_query: int
max_tokens_per_doc: int
AnyRenderParam: TypeAlias = (
EncodeCMPLRenderParams | EncodeChatRenderParams | ScoringRenderParams
)
RequestGenerator: TypeAlias = Generator[AnyRenderParam]
RequestFactory: TypeAlias = Callable[[], RequestGenerator]
class PoolingEngineInput(TypedDict):
prompts: EngineInput
params: PoolingParams
lora_requests: LoRARequest | None
priorities: int
-250
View File
@@ -1,250 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import importlib.util
import json
import math
from collections.abc import Callable, Sequence
from dataclasses import dataclass
from functools import lru_cache, partial
from typing import Any, Literal, cast
import pybase64
import torch
from fastapi.responses import JSONResponse, StreamingResponse
from vllm.config import ModelConfig
from vllm.entrypoints.serve.engine.protocol import UsageInfo
from vllm.logger import init_logger
from vllm.outputs import PoolingRequestOutput
from vllm.tasks import SupportedTask
from vllm.utils.serial_utils import (
EMBED_DTYPES,
EmbedDType,
Endianness,
binary2tensor,
tensor2binary,
)
logger = init_logger(__name__)
JsonEncodingFormat = Literal["float", "base64"]
BytesEncodingFormat = Literal["bytes", "bytes_only"]
FloatEncodedPoolingOutput = list[float] | list[list[float]]
JsonEncodedPoolingOutput = FloatEncodedPoolingOutput | str
@dataclass
class MetadataItem:
index: int
embed_dtype: EmbedDType
endianness: Endianness
start: int
end: int
shape: tuple[int, ...]
def build_metadata_items(
embed_dtype: EmbedDType,
endianness: Endianness,
shape: tuple[int, ...],
n_request: int,
) -> list[MetadataItem]:
n_bytes = EMBED_DTYPES[embed_dtype].nbytes
size = math.prod(shape)
return [
MetadataItem(
index=i,
embed_dtype=embed_dtype,
endianness=endianness,
start=i * size * n_bytes,
end=(i + 1) * size * n_bytes,
shape=shape,
)
for i in range(n_request)
]
def encode_pooling_output_float(
output: PoolingRequestOutput,
) -> FloatEncodedPoolingOutput:
return output.outputs.data.tolist()
def encode_pooling_output_float_or_ndarray(output: PoolingRequestOutput) -> Any:
"""Return an ndarray when the response renderer can serialize NumPy."""
try:
data = output.outputs.data
if not data.is_contiguous():
data = data.contiguous()
return data.numpy()
except (RuntimeError, TypeError):
return output.outputs.data.tolist()
def encode_pooling_output_base64(
output: PoolingRequestOutput,
embed_dtype: EmbedDType,
endianness: Endianness,
) -> str:
embedding_bytes = tensor2binary(output.outputs.data, embed_dtype, endianness)
return pybase64.b64encode(embedding_bytes).decode("utf-8")
def encode_pooling_bytes(
pooling_outputs: list[PoolingRequestOutput],
embed_dtype: EmbedDType,
endianness: Endianness,
) -> tuple[list[bytes], list[dict[str, Any]]]:
items: list[dict[str, Any]] = []
body: list[bytes] = []
offset = 0
for idx, output in enumerate(pooling_outputs):
binary = tensor2binary(
tensor=output.outputs.data,
embed_dtype=embed_dtype,
endianness=endianness,
)
size = len(binary)
# Dictionary form of MetadataItem
item = dict(
index=idx,
embed_dtype=embed_dtype,
endianness=endianness,
start=offset,
end=offset + size,
shape=output.outputs.data.shape,
)
body.append(binary)
items.append(item)
offset += size
return body, items
def get_pooling_output_encoder(
encoding_format: JsonEncodingFormat,
embed_dtype: EmbedDType,
endianness: Endianness,
) -> Callable[[PoolingRequestOutput], JsonEncodedPoolingOutput]:
return cast(
Callable[[PoolingRequestOutput], JsonEncodedPoolingOutput],
(
encode_pooling_output_float
if encoding_format == "float"
else partial(
encode_pooling_output_base64,
embed_dtype=embed_dtype,
endianness=endianness,
)
),
)
def get_pooling_usage(
pooling_outputs: Sequence[PoolingRequestOutput],
) -> UsageInfo:
num_prompt_tokens = sum(
len(output.prompt_token_ids) if output.prompt_token_ids is not None else 0
for output in pooling_outputs
)
return UsageInfo(
prompt_tokens=num_prompt_tokens,
total_tokens=num_prompt_tokens,
)
def get_pooling_usage_payload(
pooling_outputs: Sequence[PoolingRequestOutput],
) -> dict[str, int]:
usage = get_pooling_usage(pooling_outputs)
return {
"prompt_tokens": usage.prompt_tokens,
"total_tokens": usage.total_tokens,
}
def build_pooling_bytes_streaming_response(
pooling_outputs: list[PoolingRequestOutput],
request_id: str,
created_time: int,
model_name: str,
encoding_format: BytesEncodingFormat,
embed_dtype: EmbedDType,
endianness: Endianness,
) -> StreamingResponse:
content, items = encode_pooling_bytes(
pooling_outputs=pooling_outputs,
embed_dtype=embed_dtype,
endianness=endianness,
)
headers = (
None
if encoding_format == "bytes_only"
else {
"metadata": json.dumps(
{
"id": request_id,
"created": created_time,
"model": model_name,
"data": items,
"usage": get_pooling_usage_payload(pooling_outputs),
}
)
}
)
return StreamingResponse(
content=content,
headers=headers,
media_type="application/octet-stream",
)
def decode_pooling_output(items: list[MetadataItem], body: bytes) -> list[torch.Tensor]:
return [
binary2tensor(
body[item.start : item.end],
item.shape,
item.embed_dtype,
item.endianness,
)
for item in sorted(items, key=lambda x: x.index)
]
@lru_cache(maxsize=1)
def get_json_response_cls() -> type[JSONResponse]:
if importlib.util.find_spec("orjson") is not None:
from fastapi.responses import ORJSONResponse
return ORJSONResponse
logger.warning_once(
"To make v1/embeddings API fast, please install orjson by `pip install orjson`"
)
return JSONResponse
def enable_scoring_api(
supported_tasks: tuple["SupportedTask", ...],
model_config: ModelConfig | None = None,
) -> bool:
if model_config is None:
return False
pooling_task = model_config.get_pooling_task(supported_tasks)
if pooling_task in ("embed", "token_embed"):
return True
if pooling_task == "classify":
num_labels = getattr(model_config.hf_config, "num_labels", 0)
if num_labels != 1:
logger.debug_once("Scoring API is only enabled for num_labels == 1.")
return False
return True
return False
+4 -7
View File
@@ -11,7 +11,6 @@ from vllm.entrypoints.openai.models.serving import (
OpenAIModelRegistry,
OpenAIServingModels,
)
from vllm.entrypoints.pooling.typing import AnyPoolingRequest
from vllm.entrypoints.serve import create_error_response
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.engine.typing import AnyRequest
@@ -48,7 +47,7 @@ class BaseServing:
async def _check_model(
self,
request: AnyRequest | AnyPoolingRequest,
request: AnyRequest,
) -> ErrorResponse | None:
if self.request_logger is not None:
self.request_logger.log_request_body(request)
@@ -141,7 +140,7 @@ class BaseServing:
return random_uuid() if default is None else default
def _get_message_types(self, request: AnyRequest | AnyPoolingRequest) -> set[str]:
def _get_message_types(self, request: AnyRequest) -> set[str]:
"""Retrieve the set of types from message content dicts up
until `_`; we use this to match potential multimodal data
with default per modality loras.
@@ -166,9 +165,7 @@ class BaseServing:
message_types.add(content_dict["type"].split("_")[0])
return message_types
def _get_active_default_mm_loras(
self, request: AnyRequest | AnyPoolingRequest
) -> LoRARequest | None:
def _get_active_default_mm_loras(self, request: AnyRequest) -> LoRARequest | None:
"""Determine if there are any active default multimodal loras."""
# TODO: Currently this is only enabled for chat completions
# to be better aligned with only being enabled for .generate
@@ -193,7 +190,7 @@ class BaseServing:
def _maybe_get_adapters(
self,
request: AnyRequest | AnyPoolingRequest,
request: AnyRequest,
supports_default_mm_loras: bool = False,
) -> LoRARequest | None:
if request.model in self.models.lora_requests:
@@ -12,7 +12,6 @@ from fastapi.responses import JSONResponse, Response
from vllm.config import ModelConfig
from vllm.entrypoints.generate.factories import get_generate_invocation_types
from vllm.entrypoints.pooling.factories import get_pooling_invocation_types
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.engine.serving import BaseServing
from vllm.entrypoints.serve.instrumentator.basic import base
@@ -79,9 +78,7 @@ def _attach_router(
router = APIRouter()
# NOTE: Construct the TypeAdapters only once
INVOCATION_TYPES = get_generate_invocation_types(
supported_tasks, model_config
) + get_pooling_invocation_types(supported_tasks, model_config)
INVOCATION_TYPES = get_generate_invocation_types(supported_tasks, model_config)
INVOCATION_VALIDATORS = [
(pydantic.TypeAdapter(request_type), (get_handler, endpoint))
@@ -7,7 +7,6 @@ from collections.abc import Sequence
import torch
from vllm import envs
from vllm.entrypoints.pooling.typing import AnyPoolingRequest
from vllm.entrypoints.serve.engine.typing import AnyRequest
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
@@ -70,7 +69,7 @@ class RequestLogger:
lora_request,
)
def log_request_body(self, request: AnyRequest | AnyPoolingRequest) -> None:
def log_request_body(self, request: AnyRequest) -> None:
try:
level = logging.getLevelNamesMapping()[envs.VLLM_REQUEST_BODY_LOG_LEVEL]
except KeyError:
-54
View File
@@ -95,7 +95,6 @@ if TYPE_CHECKING:
MAX_JOBS: str | None = None
NVCC_THREADS: str | None = None
VLLM_USE_PRECOMPILED: bool = False
VLLM_USE_PRECOMPILED_RUST: bool = False
VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX: bool = False
VLLM_DOCKER_BUILD_CONTEXT: bool = False
VLLM_BUILD_COMMIT: str = "unknown"
@@ -163,9 +162,6 @@ if TYPE_CHECKING:
VLLM_DISABLE_COMPILE_CACHE: bool = False
VLLM_REPLICATE_EMBED: bool = False
VLLM_USE_LAYERNAME: bool = True
VLLM_USE_RUST_FRONTEND: bool = False
VLLM_USE_RUST_BENCH: bool = False
VLLM_RUST_FRONTEND_PATH: str | None = "auto"
VLLM_SERVER_DEV_MODE: bool = False
VLLM_V1_OUTPUT_PROC_CHUNK_SIZE: int = 128
VLLM_MLA_DISABLE: bool = False
@@ -579,41 +575,6 @@ def get_env_or_set_default(
logger = logging.getLogger(__name__)
def _resolve_rust_cli_path() -> str | None:
"""Resolve the vllm-rs binary path.
Returns None unless VLLM_USE_RUST_FRONTEND or VLLM_USE_RUST_BENCH is enabled.
When enabled, resolves VLLM_RUST_FRONTEND_PATH ("auto" by default)
to the actual binary path.
"""
use_rust = bool(int(os.environ.get("VLLM_USE_RUST_FRONTEND", "0"))) or bool(
int(os.environ.get("VLLM_USE_RUST_BENCH", "0"))
)
raw = os.environ.get("VLLM_RUST_FRONTEND_PATH", "auto")
if not use_rust:
if os.environ.get("VLLM_RUST_FRONTEND_PATH") is not None:
logger.warning(
"VLLM_RUST_FRONTEND_PATH is set without enabling "
"VLLM_USE_RUST_FRONTEND or VLLM_USE_RUST_BENCH. "
"Set one of them to 1 to use the vllm-rs binary."
)
return None
if raw.lower() in ("auto", "1", "true"):
pkg_dir = os.path.dirname(os.path.abspath(__file__))
candidate = os.path.join(pkg_dir, "vllm-rs")
if os.path.isfile(candidate) and os.access(candidate, os.X_OK):
return candidate
raise FileNotFoundError(
"VLLM_RUST_FRONTEND_PATH=auto but the vllm-rs binary was "
f"not found at {candidate}. "
"Build with setuptools-rust or set the path explicitly."
)
return raw
environment_variables: dict[str, Callable[[], Any]] = {
# ================== Installation Time Env Vars ==================
# Target device of vLLM, supporting [cuda (by default),
@@ -668,10 +629,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
os.environ.get("VLLM_USE_PRECOMPILED", "").strip().lower() in ("1", "true")
or bool(os.environ.get("VLLM_PRECOMPILED_WHEEL_LOCATION"))
),
# If set, vllm will use the precompiled Rust frontend binary (vllm-rs).
"VLLM_USE_PRECOMPILED_RUST": lambda: (
os.environ.get("VLLM_USE_PRECOMPILED_RUST", "").strip().lower() in ("1", "true")
),
# If set, skip adding +precompiled suffix to version string
"VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX": lambda: bool(
int(os.environ.get("VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX", "0"))
@@ -1452,17 +1409,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
# If set to "0", disable LayerName opaque type for layer_name
# parameters in custom ops. Defaults to enabled on torch >= 2.11.
"VLLM_USE_LAYERNAME": lambda: bool(int(os.getenv("VLLM_USE_LAYERNAME", "1"))),
# If set, use the Rust frontend binary instead of the Python API server
# process(es).
"VLLM_USE_RUST_FRONTEND": lambda: bool(
int(os.getenv("VLLM_USE_RUST_FRONTEND", "0"))
),
# If set, use the packaged Rust client for `vllm bench serve`.
"VLLM_USE_RUST_BENCH": lambda: bool(int(os.getenv("VLLM_USE_RUST_BENCH", "0"))),
# Path to the vllm-rs binary. Defaults to "auto" which discovers the
# binary installed with the vllm package. Used when VLLM_USE_RUST_FRONTEND=1
# or VLLM_USE_RUST_BENCH=1.
"VLLM_RUST_FRONTEND_PATH": lambda: _resolve_rust_cli_path(),
# If set, vllm will run in development mode, which will enable
# some additional endpoints for developing and debugging,
# e.g. `/reset_prefix_cache`
+2 -1
View File
@@ -68,7 +68,8 @@ class PluggableLayer(nn.Module):
@classmethod
def register(cls, name: str):
def decorator(op_cls):
assert name not in op_registry, f"Duplicate op name: {name}"
if name in op_registry:
return op_registry[name]
op_cls.name = name
op_registry[name] = op_cls
return op_cls
@@ -11,7 +11,6 @@ from vllm.model_executor.layers.attention.cross_attention import CrossAttention
from vllm.model_executor.layers.attention.encoder_only_attention import (
EncoderOnlyAttention,
)
from vllm.model_executor.layers.attention.mla_attention import MLAAttention
from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention
from vllm.model_executor.layers.attention.prefill_prefix_lm_attention import (
PrefillPrefixLMAttention,
@@ -35,7 +34,6 @@ __all__ = [
"ChunkedLocalAttention",
"CrossAttention",
"EncoderOnlyAttention",
"MLAAttention",
"MMEncoderAttention",
"PrefillPrefixLMAttention",
"RSWAAttention",
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -1,25 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.model_executor.layers.hpc.gated_mla import (
hpc_gated_mla_gemm,
hpc_gated_mla_supported,
)
from vllm.model_executor.layers.hpc.hpc_ihc import (
HpcIHCHead,
HpcIHCPost,
HpcIHCPre,
)
from vllm.model_executor.layers.hpc.hpc_module import HpcModule
from vllm.model_executor.layers.hpc.rope_norm import HpcRopeNorm, QkNormPolicy
__all__ = [
"HpcIHCHead",
"HpcIHCPost",
"HpcIHCPre",
"HpcModule",
"HpcRopeNorm",
"QkNormPolicy",
"hpc_gated_mla_gemm",
"hpc_gated_mla_supported",
]
-114
View File
@@ -1,114 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Enablement gate for the HPC fused gated-MLA GEMM.
``hpc.gated_mla_gemm`` fuses the MLA output gating -- the gate projection, the
sigmoid and the elementwise product -- into a single kernel launch, replacing
the eager ``attn_out * sigmoid(hidden_states @ weight.T)`` sequence.
Kernel constraints:
- Requires VLLM_ENABLE_HPC_OPS=1
- Requires the hpc package (.so) built for the current arch
- Only sm100 / sm103 (compute capability 100, 103)
- All three operands must be bfloat16 and contiguous
- The gate width must be a multiple of 256
- Only elementwise gating (headwise has no fused form)
"""
import torch
from vllm import envs
from vllm.logger import init_logger
from vllm.utils.hpc import has_hpc
logger = init_logger(__name__)
# hpc.gated_mla_gemm fuses `attn_out * sigmoid(x @ weight.T)` into one launch.
# It is built for sm100/sm103 and needs all three operands bf16 + contiguous,
# with the gate width a multiple of 256.
_HPC_GATED_GEMM_N_ALIGN = 256
_HPC_GATED_GEMM_CAPABILITIES: frozenset[int] = frozenset({100, 103})
def hpc_gated_mla_supported(
gating_type: str | None, attn_output_gate: torch.nn.Module | None
) -> bool:
"""Gate for running the MLA output gating through hpc.gated_mla_gemm."""
if not envs.VLLM_ENABLE_HPC_OPS:
return False
if attn_output_gate is None:
return False
if not has_hpc():
logger.warning_once(
"HPC gated MLA disabled: 'hpc' package is not installed. "
"Please install the HPC library (.so) to enable the fused kernel."
)
return False
from vllm.platforms import current_platform
if not current_platform.is_cuda():
logger.warning_once("HPC gated MLA disabled: only CUDA is supported.")
return False
capability = current_platform.get_device_capability()
if capability is None or capability.to_int() not in _HPC_GATED_GEMM_CAPABILITIES:
logger.warning_once(
"HPC gated MLA disabled: compute capability %s not in %s.",
capability,
_HPC_GATED_GEMM_CAPABILITIES,
)
return False
# Headwise gating broadcasts one scalar per head over v_head_dim; the
# kernel only implements the elementwise product.
if gating_type != "elementwise":
logger.warning_once(
"HPC gated MLA disabled: gating_type '%s' has no fused form.",
gating_type,
)
return False
weight = getattr(attn_output_gate, "weight", None)
if weight is None or weight.dtype != torch.bfloat16:
logger.warning_once(
"HPC gated MLA disabled: gate weight dtype is %s, the kernel needs "
"bfloat16 (keep the layer out of quantization).",
None if weight is None else weight.dtype,
)
return False
if weight.shape[0] % _HPC_GATED_GEMM_N_ALIGN != 0:
logger.warning_once(
"HPC gated MLA disabled: gate width %d is not a multiple of %d.",
weight.shape[0],
_HPC_GATED_GEMM_N_ALIGN,
)
return False
logger.info_once("HPC gated MLA enabled by set VLLM_ENABLE_HPC_OPS.")
return True
def hpc_gated_mla_gemm(
x: torch.Tensor,
weight: torch.Tensor,
atten_output: torch.Tensor,
) -> torch.Tensor:
"""Fused ``atten_output * sigmoid(x @ weight.T)``.
Args:
x: ``[m, k]`` bfloat16, contiguous.
weight: ``[n, k]`` bfloat16, contiguous and row-major (the kernel
transposes it). ``n`` must be a multiple of 256.
atten_output: ``[m, n]`` bfloat16, contiguous.
Returns:
``[m, n]`` bfloat16.
"""
from hpc.gemm import gated_mla_gemm
return gated_mla_gemm(x, weight, atten_output)
-237
View File
@@ -1,237 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""HPC fused iHC (independent Hyper-Connections) kernels for HY V4.
Replaces the eager HYV4HCPreLayer / HYV4HCPostLayer / HYV4HCHeadLayer bodies
with single-kernel HPC implementations:
pre : rms square sum + w projection + sigmoid gates + hc-dim weighted sum
post : y[n, i, :] = H_post[n, i] * x[n, :] + residual[n, i, :]
head : same as pre but hc_mult projection rows and no H_post output
The eager path issues 20 / 5 / 15 kernels for pre / post / head; each HPC op is
a single kernel.
Constraints:
- Requires VLLM_ENABLE_HPC_OPS=1
- Requires the hpc package (.so) built for the current arch
- Only sm100 / sm103 (compute capability 100, 103)
- Only hc_mult == 4 and hidden_size in {4096, 6144}
- x / residual must be bfloat16; hc_fn weights stay float32 (the checkpoint
keeps them out of fp8 quantization via modules_to_not_convert)
The fused modules own no parameters: their forwards read the weights of the
eager layer they replace directly via the captured owner reference.
NOTE: The reference implementation also offers a cross-layer post+pre fusion
(``HpcIHCPostPre``) that folds one segment's post into the next segment's pre.
It requires reworking the decoder-layer forward scheduling and is not ported
here. TODO: add it once the decoder dataflow is restructured for it.
"""
import torch
from vllm import envs
from vllm.logger import init_logger
from vllm.model_executor.layers.hpc.hpc_module import HpcModule
from vllm.utils.hpc import has_hpc
logger = init_logger(__name__)
# Shapes the HPC kernels are instantiated for. Other shapes launch nothing,
# so gate on this rather than relying on a silent no-op.
_SUPPORTED_HIDDEN_SIZES: frozenset[int] = frozenset({4096, 6144})
_SUPPORTED_HC_MULTS: frozenset[int] = frozenset({4})
_SUPPORTED_CAPABILITIES: frozenset[int] = frozenset({100, 103})
def _ihc_supported(hc_mult: int, hidden_size: int) -> bool:
"""Shared gate for all three iHC ops."""
if not envs.VLLM_ENABLE_HPC_OPS:
return False
if not has_hpc():
logger.warning_once(
"HPC iHC disabled: 'hpc' package is not installed. "
"Please install the HPC library (.so) to enable fused kernels."
)
return False
from vllm.platforms import current_platform
if not current_platform.is_cuda():
logger.warning_once("HPC iHC disabled: only CUDA is supported.")
return False
capability = current_platform.get_device_capability()
if capability is None or capability.to_int() not in _SUPPORTED_CAPABILITIES:
logger.warning_once(
"HPC iHC disabled: compute capability %s not in %s.",
capability,
_SUPPORTED_CAPABILITIES,
)
return False
if hc_mult not in _SUPPORTED_HC_MULTS:
logger.warning_once(
"HPC iHC disabled: hc_mult=%d not in %s.", hc_mult, _SUPPORTED_HC_MULTS
)
return False
if hidden_size not in _SUPPORTED_HIDDEN_SIZES:
logger.warning_once(
"HPC iHC disabled: hidden_size=%d not in %s.",
hidden_size,
_SUPPORTED_HIDDEN_SIZES,
)
return False
if envs.VLLM_BATCH_INVARIANT:
# Batch-invariant mode relies on torch's deterministic reductions; the
# HPC kernel reduces in a different order, so keep the two regimes
# apart.
logger.warning_once(
"HPC iHC disabled: not supported under VLLM_BATCH_INVARIANT."
)
return False
logger.info_once("HPC iHC enabled by set VLLM_ENABLE_HPC_OPS.")
return True
class HpcIHCPre(HpcModule):
"""Fused iHC pre block.
Computes, in one kernel:
x_flat = x.flatten(1)
r = rsqrt(x_flat.square().mean(-1) + rms_norm_eps)
mixes = (x_flat @ w.T) * r
H_pre = sigmoid(mixes[:, :hc] * hc_scale[0] + hc_base[:hc]) + hc_eps
H_post = magnitude * sigmoid(
mixes[:, hc:] * hc_scale[1] + hc_base[hc:]) + hc_eps
y = sum_i H_pre[:, i] * x[:, i, :]
Args:
hc_mult: HC expand ratio.
hidden_size: Model hidden dimension.
magnitude: H_post multiplier (config.hc_magnitude).
hc_eps: Epsilon added to both gates (config.hc_eps).
norm_eps: Epsilon inside the rsqrt (config.rms_norm_eps).
fallback_op: The eager HYV4HCPreLayer to source weights from.
norm_owner: Optional RMSNorm that immediately follows this pre block.
When given, its weight/eps are folded into the kernel so a single
launch covers pre + RMSNorm (the caller then skips the separate
layernorm). None keeps the original pre-only behaviour.
"""
def __init__(
self,
hc_mult: int,
hidden_size: int,
magnitude: float,
hc_eps: float,
norm_eps: float,
fallback_op: torch.nn.Module,
norm_owner: torch.nn.Module | None = None,
) -> None:
super().__init__()
self.hc_mult = hc_mult
self.hidden_size = hidden_size
self.magnitude = magnitude
self.hc_eps = hc_eps
self.norm_eps = norm_eps
# Stash the owner outside nn.Module's attribute machinery: callers pass
# their own `self` here, and registering that as a submodule would make
# the module tree cyclic (state_dict() then hits RecursionError).
object.__setattr__(self, "_fallback_op", fallback_op)
# Same anti-cycle idiom for the fused RMSNorm owner (a sibling module).
object.__setattr__(self, "_norm_owner", norm_owner)
@classmethod
def support(cls, hc_mult: int, hidden_size: int) -> bool:
return _ihc_supported(hc_mult, hidden_size)
def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
import hpc
owner = self._fallback_op
norm_owner = self._norm_owner
return hpc.fuse_ihc_pre(
x,
owner.hc_fn.weight,
owner.hc_scale,
owner.hc_base,
self.norm_eps,
self.hc_eps,
self.magnitude,
norm_owner.weight if norm_owner is not None else None,
norm_owner.variance_epsilon if norm_owner is not None else 0.0,
)
class HpcIHCPost(HpcModule):
"""Fused iHC post block: H_post gating plus multi-channel residual add.
Unlike mHC there is no comb matrix, so each output channel only needs its
own residual channel and the whole thing is one fused multiply-add per
element.
"""
def __init__(self, hc_mult: int, hidden_size: int) -> None:
super().__init__()
self.hc_mult = hc_mult
self.hidden_size = hidden_size
@classmethod
def support(cls, hc_mult: int, hidden_size: int) -> bool:
return _ihc_supported(hc_mult, hidden_size)
def forward(
self, x: torch.Tensor, residual: torch.Tensor, H_post: torch.Tensor
) -> torch.Tensor:
import hpc
return hpc.fuse_ihc_post(x, residual, H_post)
class HpcIHCHead(HpcModule):
"""Fused iHC head block: merge the hc channels back into one hidden state.
Same structure as HpcIHCPre but the projection emits only hc_mult gate
logits and there is no H_post output. Called once per forward, after the
last decoder layer.
"""
def __init__(
self,
hc_mult: int,
hidden_size: int,
hc_eps: float,
norm_eps: float,
fallback_op: torch.nn.Module,
) -> None:
super().__init__()
self.hc_mult = hc_mult
self.hidden_size = hidden_size
self.hc_eps = hc_eps
self.norm_eps = norm_eps
# See HpcIHCPre: keep the owner out of the module tree.
object.__setattr__(self, "_fallback_op", fallback_op)
@classmethod
def support(cls, hc_mult: int, hidden_size: int) -> bool:
return _ihc_supported(hc_mult, hidden_size)
def forward(self, x: torch.Tensor) -> torch.Tensor:
import hpc
owner = self._fallback_op
return hpc.fuse_ihc_head(
x,
owner.hc_head_fn.weight,
owner.hc_head_scale,
owner.hc_head_base,
self.norm_eps,
self.hc_eps,
)

Some files were not shown because too many files have changed in this diff Show More