[Mini-vLLM] SM86-only cut: models, backends, Rust, tests
- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries - Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels - Remove inc/quark quant loaders; keep compressed-tensors and Marlin - Drop the Rust frontend, pooling entrypoints, and idle test suites - Rebuild kernels for sm_80;sm_86 and strip the extensions - Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
+6
-4
@@ -15,10 +15,12 @@ from vllm import _custom_ops as ops
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
rocm_aiter_sparse_attn_indexer,
|
||||
rocm_aiter_sparse_attn_indexer_fake,
|
||||
)
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
rocm_aiter_sparse_attn_indexer,
|
||||
rocm_aiter_sparse_attn_indexer_fake,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from aiter import ActivationType
|
||||
|
||||
@@ -1,88 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.benchmarks.datasets.datasets import (
|
||||
DEFAULT_NUM_PROMPTS,
|
||||
AIMODataset,
|
||||
ASRDataset,
|
||||
BenchmarkDataset,
|
||||
BFCLDataset,
|
||||
BlazeditDataset,
|
||||
BurstGPTDataset,
|
||||
ConversationDataset,
|
||||
CustomAudioDataset,
|
||||
CustomDataset,
|
||||
CustomImageDataset,
|
||||
HuggingFaceDataset,
|
||||
InstructCoderDataset,
|
||||
MLPerfDataset,
|
||||
MMStarDataset,
|
||||
MMVUDataset,
|
||||
MTBenchDataset,
|
||||
MultiModalConversationDataset,
|
||||
NextEditPredictionDataset,
|
||||
PrefixRepetitionRandomDataset,
|
||||
RandomDataset,
|
||||
RandomDatasetForReranking,
|
||||
RandomMultiModalDataset,
|
||||
SampleRequest,
|
||||
ShareGPTDataset,
|
||||
SpecBench,
|
||||
VisionArenaDataset,
|
||||
add_dataset_parser,
|
||||
add_random_dataset_base_args,
|
||||
add_random_multimodal_dataset_args,
|
||||
gen_prompt_decode_to_target_len,
|
||||
get_samples,
|
||||
is_valid_sequence,
|
||||
lora_path_on_disk,
|
||||
lora_tokenizer_cache,
|
||||
process_audio,
|
||||
process_image,
|
||||
process_video,
|
||||
zeta_prompt,
|
||||
)
|
||||
from vllm.benchmarks.datasets.utils import RangeRatio
|
||||
|
||||
__all__ = [
|
||||
"DEFAULT_NUM_PROMPTS",
|
||||
"AIMODataset",
|
||||
"ASRDataset",
|
||||
"BenchmarkDataset",
|
||||
"BFCLDataset",
|
||||
"BlazeditDataset",
|
||||
"BurstGPTDataset",
|
||||
"ConversationDataset",
|
||||
"CustomDataset",
|
||||
"CustomAudioDataset",
|
||||
"CustomImageDataset",
|
||||
"HuggingFaceDataset",
|
||||
"InstructCoderDataset",
|
||||
"MLPerfDataset",
|
||||
"MMStarDataset",
|
||||
"MMVUDataset",
|
||||
"MTBenchDataset",
|
||||
"MultiModalConversationDataset",
|
||||
"NextEditPredictionDataset",
|
||||
"PrefixRepetitionRandomDataset",
|
||||
"RandomDataset",
|
||||
"RandomDatasetForReranking",
|
||||
"RandomMultiModalDataset",
|
||||
"SampleRequest",
|
||||
"ShareGPTDataset",
|
||||
"SpecBench",
|
||||
"VisionArenaDataset",
|
||||
"add_dataset_parser",
|
||||
"add_random_dataset_base_args",
|
||||
"add_random_multimodal_dataset_args",
|
||||
"gen_prompt_decode_to_target_len",
|
||||
"get_samples",
|
||||
"is_valid_sequence",
|
||||
"lora_path_on_disk",
|
||||
"lora_tokenizer_cache",
|
||||
"process_audio",
|
||||
"process_image",
|
||||
"process_video",
|
||||
"RangeRatio",
|
||||
"zeta_prompt",
|
||||
]
|
||||
@@ -1,207 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Convert a plain-text file (local path or URL) into a JSONL dataset
|
||||
compatible with ``CustomDataset`` (``--dataset-name custom``), by
|
||||
randomly slicing the tokenized text into prompts.
|
||||
|
||||
Each line of the output JSONL contains a ``prompt`` (decoded from a random
|
||||
slice of the tokenized source text) and an ``output_tokens`` count.
|
||||
|
||||
Usage
|
||||
-----
|
||||
::
|
||||
|
||||
python -m vllm.benchmarks.datasets.create_txt_slices_dataset \\
|
||||
--input corpus.txt \\
|
||||
--output dataset.jsonl \\
|
||||
--tokenizer gpt2 \\
|
||||
--num-prompts 1000 \\
|
||||
--input-len 1024 \\
|
||||
--output-len 128
|
||||
|
||||
The resulting JSONL file can then be used with the serving benchmark::
|
||||
|
||||
python -m vllm.benchmarks.serve \\
|
||||
--dataset-name custom \\
|
||||
--dataset-path dataset.jsonl \\
|
||||
...
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import random
|
||||
import urllib.request
|
||||
|
||||
import numpy as np
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from vllm.benchmarks.datasets.utils import RangeRatio, get_sampling_params
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def load_text(path: str) -> str:
|
||||
"""Load text from a local file or URL."""
|
||||
if path.startswith(("http://", "https://")):
|
||||
with urllib.request.urlopen(path) as response:
|
||||
return response.read().decode("utf-8")
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
def create_txt_slices_jsonl(
|
||||
*,
|
||||
input_path: str,
|
||||
output_path: str,
|
||||
tokenizer_name: str,
|
||||
num_prompts: int,
|
||||
input_len: int,
|
||||
output_len: int,
|
||||
range_ratio: RangeRatio = 0.0,
|
||||
seed: int = 0,
|
||||
trust_remote_code: bool = False,
|
||||
) -> None:
|
||||
"""Read *input_path*, slice it into prompts, and write JSONL to
|
||||
*output_path*."""
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
tokenizer_name, trust_remote_code=trust_remote_code
|
||||
)
|
||||
|
||||
text = load_text(input_path)
|
||||
if not text:
|
||||
raise ValueError("The text file is empty and cannot be sampled from.")
|
||||
|
||||
token_ids = tokenizer(text, add_special_tokens=False).input_ids
|
||||
if not token_ids:
|
||||
raise ValueError("Tokenizing the text produced zero tokens; cannot sample.")
|
||||
|
||||
rng_np = np.random.default_rng(seed)
|
||||
rng_py = random.Random(seed)
|
||||
|
||||
input_lens, output_lens, _ = get_sampling_params(
|
||||
rng_np,
|
||||
num_prompts,
|
||||
range_ratio,
|
||||
input_len,
|
||||
output_len,
|
||||
tokenizer,
|
||||
)
|
||||
|
||||
num_available_tokens = len(token_ids)
|
||||
|
||||
records: list[dict[str, object]] = []
|
||||
for i in range(num_prompts):
|
||||
req_input_len = int(input_lens[i])
|
||||
req_output_len = int(output_lens[i])
|
||||
|
||||
# Randomly select a start position and slice with cycling
|
||||
start_pos = rng_py.randint(0, num_available_tokens - 1)
|
||||
prompt_token_ids = [
|
||||
token_ids[(start_pos + j) % num_available_tokens]
|
||||
for j in range(req_input_len)
|
||||
]
|
||||
prompt = tokenizer.decode(prompt_token_ids, skip_special_tokens=False)
|
||||
|
||||
records.append({"prompt": prompt, "output_tokens": req_output_len})
|
||||
|
||||
with open(output_path, "w", encoding="utf-8") as f:
|
||||
for record in records:
|
||||
f.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||||
|
||||
logger.info(
|
||||
"Wrote %d prompts to %s",
|
||||
len(records),
|
||||
output_path,
|
||||
)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> None:
|
||||
parser = FlexibleArgumentParser(
|
||||
description="Convert a plain-text file into a JSONL dataset "
|
||||
"for CustomDataset (--dataset-name custom).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--input",
|
||||
required=True,
|
||||
help="Path or URL to the source text file.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
required=True,
|
||||
help="Path for the output JSONL file.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--tokenizer",
|
||||
required=True,
|
||||
help="HuggingFace tokenizer name or path.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num-prompts",
|
||||
type=int,
|
||||
default=1000,
|
||||
help="Number of prompt samples to generate (default: 1000).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--input-len",
|
||||
type=int,
|
||||
default=1024,
|
||||
help="Target number of input tokens per prompt (default: 1024).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-len",
|
||||
type=int,
|
||||
default=128,
|
||||
help="Target number of output tokens per prompt (default: 128).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--range-ratio",
|
||||
type=str,
|
||||
default="0.0",
|
||||
help="Range ratio for input/output length sampling (default: 0.0). "
|
||||
"A single float applies to both ISL and OSL. "
|
||||
'A JSON dict like \'{"input": 0.3, "output": 0.5}\' sets them '
|
||||
"independently. Values must be in [0, 1).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--seed",
|
||||
type=int,
|
||||
default=0,
|
||||
help="Random seed for reproducibility (default: 0).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--trust-remote-code",
|
||||
action="store_true",
|
||||
help="Trust remote code from HuggingFace.",
|
||||
)
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
# Parse --range-ratio: try float first, then JSON dict.
|
||||
range_ratio: RangeRatio
|
||||
try:
|
||||
range_ratio = float(args.range_ratio)
|
||||
except ValueError:
|
||||
import json as _json
|
||||
|
||||
range_ratio = _json.loads(args.range_ratio)
|
||||
|
||||
create_txt_slices_jsonl(
|
||||
input_path=args.input,
|
||||
output_path=args.output,
|
||||
tokenizer_name=args.tokenizer,
|
||||
num_prompts=args.num_prompts,
|
||||
input_len=args.input_len,
|
||||
output_len=args.output_len,
|
||||
range_ratio=range_ratio,
|
||||
seed=args.seed,
|
||||
trust_remote_code=args.trust_remote_code,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,99 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared utilities for benchmark dataset sampling."""
|
||||
|
||||
import logging
|
||||
import math
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Type alias: a single float applies to both ISL and OSL; a dict allows
|
||||
# specifying them independently via ``{"input": …, "output": …}``.
|
||||
RangeRatio = float | dict[str, float]
|
||||
|
||||
|
||||
def _resolve_range_ratios(
|
||||
range_ratio: RangeRatio,
|
||||
) -> tuple[float, float]:
|
||||
"""Return ``(input_range_ratio, output_range_ratio)`` from *range_ratio*.
|
||||
|
||||
*range_ratio* is either a single float (used for both input and output)
|
||||
or a dict with ``"input"`` and ``"output"`` keys.
|
||||
"""
|
||||
if isinstance(range_ratio, dict):
|
||||
try:
|
||||
return float(range_ratio["input"]), float(range_ratio["output"])
|
||||
except KeyError as exc:
|
||||
raise ValueError(
|
||||
"When range_ratio is a dict it must contain 'input' and "
|
||||
f"'output' keys, got: {sorted(range_ratio)}"
|
||||
) from exc
|
||||
ratio = float(range_ratio)
|
||||
return ratio, ratio
|
||||
|
||||
|
||||
def get_sampling_params(
|
||||
rng: np.random.Generator,
|
||||
num_requests: int,
|
||||
range_ratio: RangeRatio,
|
||||
input_len: int,
|
||||
output_len: int,
|
||||
tokenizer: TokenizerLike,
|
||||
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
|
||||
"""Sample per-request input/output token lengths and vocab offsets.
|
||||
|
||||
Lengths are drawn uniformly from integer ranges around the configured
|
||||
means, controlled by *range_ratio*. It may be a single ``float``
|
||||
(applied to both input and output) or a ``dict`` with ``"input"`` and
|
||||
``"output"`` keys for independent control.
|
||||
|
||||
Tokenizer special tokens are subtracted from ``input_len`` before
|
||||
computing the sampling interval.
|
||||
|
||||
Returns:
|
||||
(input_lens, output_lens, offsets) – three 1-D ``np.ndarray`` of
|
||||
shape ``(num_requests,)``.
|
||||
|
||||
"""
|
||||
input_range_ratio, output_range_ratio = _resolve_range_ratios(range_ratio)
|
||||
|
||||
if not (0.0 <= input_range_ratio < 1.0):
|
||||
raise ValueError("input_range_ratio must be in [0, 1).")
|
||||
if not (0.0 <= output_range_ratio < 1.0):
|
||||
raise ValueError("output_range_ratio must be in [0, 1).")
|
||||
num_special_tokens = int(tokenizer.num_special_tokens_to_add())
|
||||
real_input_len = max(0, int(input_len) - num_special_tokens)
|
||||
input_low = math.floor(real_input_len * (1 - input_range_ratio))
|
||||
input_high = math.ceil(real_input_len * (1 + input_range_ratio))
|
||||
output_low = math.floor(output_len * (1 - output_range_ratio))
|
||||
output_high = math.ceil(output_len * (1 + output_range_ratio))
|
||||
# Ensure the lower bound for output length is at least 1 to
|
||||
# prevent sampling 0 tokens.
|
||||
output_low = max(output_low, 1)
|
||||
output_high = max(output_high, 1)
|
||||
|
||||
if input_low > input_high:
|
||||
raise ValueError(
|
||||
f"Invalid input sampling interval: low={input_low} > high={input_high}"
|
||||
)
|
||||
if output_low > output_high:
|
||||
raise ValueError(
|
||||
f"Invalid output sampling interval: low={output_low} > high={output_high}"
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Sampling input_len from [%s, %s] and output_len from [%s, %s]",
|
||||
input_low,
|
||||
input_high,
|
||||
output_low,
|
||||
output_high,
|
||||
)
|
||||
|
||||
input_lens = rng.integers(input_low, input_high + 1, size=num_requests)
|
||||
output_lens = rng.integers(output_low, output_high + 1, size=num_requests)
|
||||
offsets = rng.integers(0, tokenizer.vocab_size, size=num_requests)
|
||||
return input_lens, output_lens, offsets
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,148 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Build a `SchedulerOutput` for an exact batch, without the scheduler."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.benchmarks.forward.shapes import BenchCase
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.core.sched.output import NewRequestData, SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
SlidingWindowSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class BenchBatch:
|
||||
scheduler_output: SchedulerOutput
|
||||
# req_id -> token IDs of the query tokens, in order.
|
||||
query_token_ids: dict[str, np.ndarray]
|
||||
# Per KV cache group: allocated (begin, end) block ranges.
|
||||
group_block_ranges: list[list[tuple[int, int]]]
|
||||
num_blocks: int
|
||||
# req_id -> the runner's expected (scheduled, computed, is_decode).
|
||||
expected: dict[str, tuple[int, int, bool]]
|
||||
|
||||
@property
|
||||
def num_tokens(self) -> int:
|
||||
return self.scheduler_output.total_num_scheduled_tokens
|
||||
|
||||
@property
|
||||
def req_ids(self) -> list[str]:
|
||||
return list(self.scheduler_output.num_scheduled_tokens)
|
||||
|
||||
def finish(self) -> SchedulerOutput:
|
||||
"""A step that only removes this batch's requests."""
|
||||
output = SchedulerOutput.make_empty()
|
||||
output.finished_req_ids = set(self.req_ids)
|
||||
return output
|
||||
|
||||
|
||||
def kv_block_size(spec: KVCacheSpec, dcp_size: int) -> int:
|
||||
"""Tokens per block as the scheduler counts them: DCP spreads each
|
||||
block's tokens over the DCP ranks."""
|
||||
return spec.block_size * (dcp_size if spec.dcp_sharded else 1)
|
||||
|
||||
|
||||
def _num_blocks(
|
||||
spec: KVCacheSpec, computed: int, kv_len: int, dcp_size: int
|
||||
) -> tuple[int, int]:
|
||||
"""Return (leading null blocks, live blocks) for one request."""
|
||||
block_size = kv_block_size(spec, dcp_size)
|
||||
if isinstance(spec, UniformTypeKVCacheSpecs):
|
||||
spec = spec.first_spec
|
||||
if isinstance(spec, (CircularBufferSpec, MambaSpec)):
|
||||
return 0, 1
|
||||
total = -(-kv_len // block_size)
|
||||
skipped = 0
|
||||
if isinstance(spec, SlidingWindowSpec):
|
||||
# Blocks wholly before the window are freed to the null block.
|
||||
skipped = max(0, computed - spec.sliding_window + 1) // block_size
|
||||
return skipped, total - skipped
|
||||
|
||||
|
||||
def build_batch(
|
||||
case: BenchCase,
|
||||
kv_cache_groups,
|
||||
num_kv_blocks: int,
|
||||
vocab_size: int,
|
||||
rng: np.random.Generator,
|
||||
dcp_size: int = 1,
|
||||
) -> BenchBatch:
|
||||
"""Allocate disjoint blocks and build `NewRequestData` for every request.
|
||||
|
||||
Groups share one block pool, so block IDs are unique across groups. Block
|
||||
0 is the null block.
|
||||
|
||||
With prefix P = K - Q (KV length K, query length Q):
|
||||
|
||||
| Kind | prompt tokens | num_computed_tokens | spec tokens |
|
||||
| ------------------ | ------------- | ------------------- | ----------- |
|
||||
| Prefill | K | P | - |
|
||||
| Decode | P | P | - |
|
||||
| Verify (D drafts) | P | P | D |
|
||||
|
||||
Unless chunked, a prefill's prefix is a prefix hit, which replays its last
|
||||
R = min(window, P) tokens on models with SWA bounded replay (window 0
|
||||
otherwise), like the scheduler: it schedules R + Q tokens from
|
||||
replay_start = num_computed_tokens = P - R.
|
||||
"""
|
||||
replay_window = max(
|
||||
(group.kv_cache_spec.prefix_replay_tokens for group in kv_cache_groups),
|
||||
default=0,
|
||||
)
|
||||
expected: dict[str, tuple[int, int, bool]] = {}
|
||||
cursor = 1
|
||||
new_reqs = []
|
||||
query_token_ids = {}
|
||||
group_block_ranges: list[list[tuple[int, int]]] = [[] for _ in kv_cache_groups]
|
||||
output = SchedulerOutput.make_empty()
|
||||
for i, req in enumerate(case.requests):
|
||||
req_id = f"bench-{i}"
|
||||
replay = min(replay_window, req.prefix_len) if req.prefix_hit else 0
|
||||
computed = req.prefix_len - replay
|
||||
block_ids = []
|
||||
for group, ranges in zip(kv_cache_groups, group_block_ranges):
|
||||
skipped, live = _num_blocks(
|
||||
group.kv_cache_spec, computed, req.kv_len, dcp_size
|
||||
)
|
||||
block_ids.append([0] * skipped + list(range(cursor, cursor + live)))
|
||||
ranges.append((cursor, cursor + live))
|
||||
cursor += live
|
||||
tokens = rng.integers(0, vocab_size, req.kv_len, dtype=np.int64)
|
||||
prompt = tokens[: req.prefix_len] if req.is_decode else tokens
|
||||
query_token_ids[req_id] = tokens[computed:]
|
||||
expected[req_id] = (req.query_len + replay, computed, req.is_decode)
|
||||
new_reqs.append(
|
||||
NewRequestData(
|
||||
req_id=req_id,
|
||||
prompt_token_ids=prompt.tolist(),
|
||||
prefill_token_ids=prompt.tolist(),
|
||||
mm_features=[],
|
||||
sampling_params=SamplingParams(max_tokens=16, temperature=0.0),
|
||||
pooling_params=None,
|
||||
block_ids=tuple(block_ids),
|
||||
num_computed_tokens=computed,
|
||||
lora_request=None,
|
||||
replay_start=computed if replay else 0,
|
||||
)
|
||||
)
|
||||
output.num_scheduled_tokens[req_id] = req.query_len + replay
|
||||
if req.num_draft_tokens:
|
||||
output.scheduled_spec_decode_tokens[req_id] = tokens[
|
||||
req.prefix_len + 1 :
|
||||
].tolist()
|
||||
if cursor > num_kv_blocks:
|
||||
raise ValueError(
|
||||
f"Case {case.name!r} needs {cursor} KV blocks, but only "
|
||||
f"{num_kv_blocks} exist. Increase the KV cache size."
|
||||
)
|
||||
output.scheduled_new_reqs = new_reqs
|
||||
output.total_num_scheduled_tokens = sum(output.num_scheduled_tokens.values())
|
||||
return BenchBatch(output, query_token_ids, group_block_ranges, cursor, expected)
|
||||
@@ -1,266 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Capture the forward of one exact batch as a one-shot CUDA graph and time it.
|
||||
|
||||
The graph is captured for a single batch and only replays that batch, so the
|
||||
host-side decisions baked in at capture (attention plans, kernel choices by
|
||||
query length, chunk splits) are correct by construction. This deliberately
|
||||
bypasses vLLM's CUDA graph manager: no capture sizes, padding or dummy
|
||||
metadata. Each replay first restores the KV blocks the step writes, so state the
|
||||
forward mutates in place (e.g. compressor state) is the same on every replay.
|
||||
"""
|
||||
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.benchmarks.forward.batch import BenchBatch
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.distributed.parallel_state import get_world_group, graph_capture
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
|
||||
|
||||
def _first_tensor(output: Any) -> torch.Tensor:
|
||||
if isinstance(output, tuple):
|
||||
output = output[0]
|
||||
assert isinstance(output, torch.Tensor), type(output)
|
||||
return output
|
||||
|
||||
|
||||
def _compare(actual: torch.Tensor, expected: torch.Tensor) -> dict[str, float]:
|
||||
delta = actual.float() - expected.float()
|
||||
scale = expected.float().square().mean().sqrt().clamp(min=1e-12)
|
||||
return {
|
||||
"max_abs": delta.abs().max().item(),
|
||||
"relative_rms": (delta.square().mean().sqrt() / scale).item(),
|
||||
}
|
||||
|
||||
|
||||
class ForwardBench:
|
||||
"""Runs one batch through the runner's `execute_model`, timing its forward.
|
||||
|
||||
`execute_model` prepares the inputs, attention metadata and forward context
|
||||
as in serving; the bench takes over the eager model call. Two steps: the
|
||||
first ("warmup") runs the model eagerly, which triggers JIT, autotuning and
|
||||
lazy init. The second ("capture") captures, validates and times the graph.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
runner: Any,
|
||||
batch: BenchBatch,
|
||||
iters: int,
|
||||
samples: int,
|
||||
state_views: list[torch.Tensor],
|
||||
name: str,
|
||||
profile_prefix: str | None = None,
|
||||
):
|
||||
self.runner = runner
|
||||
self.batch = batch
|
||||
self.device = runner.device
|
||||
self.iters = iters
|
||||
self.samples = samples
|
||||
self.state_views = state_views
|
||||
self.name = name
|
||||
self.profile_prefix = profile_prefix
|
||||
self.phase = "warmup"
|
||||
self.result: dict[str, Any] = {}
|
||||
|
||||
def run(self) -> dict[str, Any]:
|
||||
runner, model = self.runner, self.runner.model
|
||||
# The runner's eager branch calls `model(**inputs)`.
|
||||
assert runner.compilation_config.cudagraph_mode == CUDAGraphMode.NONE
|
||||
assert runner.ubatch_runner is None
|
||||
prepare_inputs = runner.prepare_inputs
|
||||
self._model_forward = model.forward
|
||||
|
||||
def prepare_and_check(*args, **kwargs) -> InputBatch:
|
||||
self._input_batch = prepare_inputs(*args, **kwargs)
|
||||
self._check_batch(self._input_batch)
|
||||
return self._input_batch
|
||||
|
||||
# Instance attributes shadow the methods for these two steps only.
|
||||
runner.prepare_inputs = prepare_and_check
|
||||
model.forward = self._forward
|
||||
try:
|
||||
for self.phase in ("warmup", "capture"):
|
||||
runner.execute_model(self.batch.scheduler_output)
|
||||
finally:
|
||||
del runner.prepare_inputs, model.forward
|
||||
# No sample_tokens follows.
|
||||
runner.execute_model_state = None
|
||||
return self.result
|
||||
|
||||
def _forward(self, **inputs) -> Any:
|
||||
self._write_input_ids(self._input_batch, inputs["input_ids"])
|
||||
# Models may update the forward context in place (e.g. sequence
|
||||
# parallelism shards is_padding); every call must start from the
|
||||
# context the runner built.
|
||||
context = get_forward_context()
|
||||
saved = dict(vars(context))
|
||||
|
||||
def run_model() -> Any:
|
||||
for name, value in saved.items():
|
||||
setattr(context, name, value)
|
||||
return self._model_forward(**inputs)
|
||||
|
||||
self._run_model = run_model
|
||||
if self.phase == "warmup":
|
||||
return run_model()
|
||||
return self._capture_and_time()
|
||||
|
||||
def _check_batch(self, input_batch: InputBatch) -> None:
|
||||
"""Fail unless the runner built exactly the requested batch."""
|
||||
expected = self.batch.expected
|
||||
assert sorted(input_batch.req_ids) == sorted(expected), input_batch.req_ids
|
||||
for i, req_id in enumerate(input_batch.req_ids):
|
||||
actual = (
|
||||
int(input_batch.num_scheduled_tokens[i]),
|
||||
int(input_batch.num_computed_tokens_np[i]),
|
||||
not bool(input_batch.is_prefilling_np[i]),
|
||||
)
|
||||
assert actual == expected[req_id], (
|
||||
f"{req_id}: got (scheduled, computed, decode) = {actual}, "
|
||||
f"wanted {expected[req_id]}"
|
||||
)
|
||||
assert input_batch.num_tokens == self.batch.num_tokens
|
||||
self.result["num_tokens_after_padding"] = input_batch.num_tokens_after_padding
|
||||
self.result["batch_order"] = input_batch.req_ids
|
||||
|
||||
def _write_input_ids(
|
||||
self, input_batch: InputBatch, input_ids: torch.Tensor | None
|
||||
) -> None:
|
||||
"""Decode/verify inputs otherwise come from sampler state."""
|
||||
if input_ids is None:
|
||||
return
|
||||
ids = np.concatenate(
|
||||
[self.batch.query_token_ids[req_id] for req_id in input_batch.req_ids]
|
||||
)
|
||||
input_ids[: len(ids)].copy_(torch.from_numpy(ids))
|
||||
|
||||
def _capture_and_time(self) -> Any:
|
||||
world = get_world_group()
|
||||
# The forward mutates state it reads (e.g. compressor state), so
|
||||
# every run starts by restoring the KV blocks the step writes.
|
||||
snapshots = [(view, view.clone()) for view in self.state_views]
|
||||
|
||||
def restore() -> None:
|
||||
for view, saved in snapshots:
|
||||
view.copy_(saved)
|
||||
|
||||
# A rendezvous and the timing events live inside the graph, after the
|
||||
# restore: neither rank skew, the restore nor host launch gaps reach
|
||||
# the timed interval.
|
||||
flag = torch.zeros(1, device=self.device)
|
||||
# TODO(woosuk): Support ROCm. The bench hangs there, likely on these
|
||||
# in-graph events.
|
||||
self._events = (
|
||||
torch.cuda.Event(enable_timing=True, external=True),
|
||||
torch.cuda.Event(enable_timing=True, external=True),
|
||||
)
|
||||
if world.world_size > 1:
|
||||
# Initialize the communicator outside capture.
|
||||
torch.distributed.all_reduce(flag, group=world.device_group)
|
||||
with graph_capture(self.device) as capture_context:
|
||||
stream = capture_context.stream
|
||||
# Some kernels set up per-stream state on first use, outside capture.
|
||||
eager = _first_tensor(self._run_model()).clone()
|
||||
restore()
|
||||
reference = _first_tensor(self._run_model()).clone()
|
||||
stream.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=stream):
|
||||
restore()
|
||||
if world.world_size > 1:
|
||||
torch.distributed.all_reduce(flag, group=world.device_group)
|
||||
self._events[0].record()
|
||||
output = self._run_model()
|
||||
self._events[1].record()
|
||||
hidden = _first_tensor(output)
|
||||
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
first = hidden.clone()
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
self.result.update(
|
||||
# Same step from the same restored state: should match exactly.
|
||||
eager_repeat=_compare(reference, eager),
|
||||
graph_vs_eager=_compare(first, reference),
|
||||
graph_deterministic=bool(torch.equal(first, hidden)),
|
||||
finite=bool(hidden.isfinite().all()),
|
||||
state_reset_bytes=sum(
|
||||
v.numel() * v.element_size() for v in self.state_views
|
||||
),
|
||||
)
|
||||
self.result["latency_us"] = self._time(graph)
|
||||
from vllm.benchmarks.forward.profile import profile_graph
|
||||
|
||||
# Each replay restores the state views, then (multi-rank) rendezvouses,
|
||||
# before the timed forward.
|
||||
skip = len(self.state_views) + (world.world_size > 1)
|
||||
summary = profile_graph(graph, self.name, self.profile_prefix, skip)
|
||||
if summary is not None:
|
||||
self.result["profile"] = {
|
||||
"summary": f"{self.profile_prefix}.kernels.json",
|
||||
"span_us": summary["span_us"],
|
||||
"busy_us": summary["busy_us"],
|
||||
"activities": summary["activities"],
|
||||
"top_kernels": summary["kernels"][:10],
|
||||
}
|
||||
del graph
|
||||
return output
|
||||
|
||||
def _time(self, graph: torch.cuda.CUDAGraph) -> dict[str, Any]:
|
||||
"""Per-replay forward time between in-graph events, MAX across ranks."""
|
||||
world = get_world_group()
|
||||
start, end = self._events
|
||||
self._warmup(graph)
|
||||
times = []
|
||||
for _ in range(self.iters * self.samples):
|
||||
graph.replay()
|
||||
end.synchronize()
|
||||
times.append(start.elapsed_time(end) * 1000)
|
||||
arr = torch.tensor(times, device=self.device)
|
||||
if world.world_size > 1:
|
||||
torch.distributed.all_reduce(
|
||||
arr, op=torch.distributed.ReduceOp.MAX, group=world.device_group
|
||||
)
|
||||
arr = arr.cpu().numpy()
|
||||
sample_medians = np.median(arr.reshape(self.samples, -1), axis=1)
|
||||
median = float(np.median(arr))
|
||||
return {
|
||||
"timer": "graph_events" + ("_rank_max" if world.world_size > 1 else ""),
|
||||
"median": median,
|
||||
# Stability: spread of per-sample medians relative to the median.
|
||||
"sample_spread_pct": float(
|
||||
100 * (sample_medians.max() - sample_medians.min()) / median
|
||||
),
|
||||
"mean": float(arr.mean()),
|
||||
"std": float(arr.std()),
|
||||
"min": float(arr.min()),
|
||||
"p90": float(np.percentile(arr, 90)),
|
||||
"count": len(arr),
|
||||
"samples_us": [round(float(t), 2) for t in arr],
|
||||
}
|
||||
|
||||
def _warmup(self, graph: torch.cuda.CUDAGraph, seconds: float = 1.0) -> None:
|
||||
"""Replay until clocks and caches reach a steady state."""
|
||||
world = get_world_group()
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
begin = time.perf_counter()
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
per_replay = max(time.perf_counter() - begin, 1e-6)
|
||||
# Same count on every rank: the graph may contain collectives.
|
||||
count = torch.tensor([max(self.iters, int(seconds / per_replay))])
|
||||
torch.distributed.all_reduce(
|
||||
count, op=torch.distributed.ReduceOp.MAX, group=world.cpu_group
|
||||
)
|
||||
for _ in range(int(count.item())):
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
@@ -1,70 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""KV blocks of a benchmark case: initialization and the blocks a step writes.
|
||||
|
||||
Only the case's blocks are touched, group by group: groups can alias one
|
||||
allocation with different layouts, and the batch builder keeps their block IDs
|
||||
disjoint. Zero KV times within a few percent of random KV, even for sparse
|
||||
attention (DeepSeek-V4.1, ~100K context), so zeros are the only init for now.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterator
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.benchmarks.forward.batch import BenchBatch, kv_block_size
|
||||
|
||||
|
||||
def _tensors(kv_cache) -> Iterator[torch.Tensor]:
|
||||
if isinstance(kv_cache, torch.Tensor):
|
||||
if kv_cache.numel():
|
||||
yield kv_cache
|
||||
elif isinstance(kv_cache, (list, tuple)):
|
||||
for item in kv_cache:
|
||||
yield from _tensors(item)
|
||||
|
||||
|
||||
def _block_dim(tensor: torch.Tensor, num_blocks: int) -> tuple[int, int]:
|
||||
"""Return (block dim, kernel blocks per allocator block)."""
|
||||
for dim in range(min(2, tensor.dim())):
|
||||
if tensor.shape[dim] >= num_blocks and tensor.shape[dim] % num_blocks == 0:
|
||||
return dim, tensor.shape[dim] // num_blocks
|
||||
raise ValueError(f"No block dim of size {num_blocks} in {tuple(tensor.shape)}")
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def zero_kv_blocks(
|
||||
static_forward_context: dict, kv_cache_groups, batch: BenchBatch, num_blocks: int
|
||||
) -> None:
|
||||
for group, ranges in zip(kv_cache_groups, batch.group_block_ranges):
|
||||
for name in group.layer_names:
|
||||
layer = static_forward_context[name]
|
||||
for tensor in _tensors(getattr(layer, "kv_cache", None)):
|
||||
dim, factor = _block_dim(tensor, num_blocks)
|
||||
for begin, end in ranges:
|
||||
tensor.narrow(dim, begin * factor, (end - begin) * factor).zero_()
|
||||
|
||||
|
||||
def touched_kv_views(
|
||||
static_forward_context: dict,
|
||||
kv_cache_groups,
|
||||
batch: BenchBatch,
|
||||
num_blocks: int,
|
||||
dcp_size: int,
|
||||
) -> list[torch.Tensor]:
|
||||
"""Views of the KV blocks a step may write: each request's last
|
||||
ceil(scheduled / block_size) + 2 blocks, in every group."""
|
||||
scheduled_tokens = [batch.expected[req_id][0] for req_id in batch.req_ids]
|
||||
views = []
|
||||
for group, ranges in zip(kv_cache_groups, batch.group_block_ranges):
|
||||
block_size = kv_block_size(group.kv_cache_spec, dcp_size)
|
||||
for name in group.layer_names:
|
||||
layer = static_forward_context[name]
|
||||
for tensor in _tensors(getattr(layer, "kv_cache", None)):
|
||||
dim, factor = _block_dim(tensor, num_blocks)
|
||||
for (begin, end), tokens in zip(ranges, scheduled_tokens):
|
||||
first = max(begin, end - (-(-tokens // block_size) + 2))
|
||||
views.append(
|
||||
tensor.narrow(dim, first * factor, (end - first) * factor)
|
||||
)
|
||||
return views
|
||||
@@ -1,256 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Benchmark the model forward of exact batches as one-shot CUDA graphs.
|
||||
|
||||
Each case is a list of requests `(query_len, kv_len, num_draft_tokens)`
|
||||
per DP rank. The bench builds the batch without the scheduler, runs it through
|
||||
the real `execute_model` (input prep, attention metadata, DP sync), and times
|
||||
the target-model forward captured as a CUDA graph. Sampling, the speculator
|
||||
and CPU step overhead aren't measured.
|
||||
|
||||
Single GPU: `vllm bench forward --model M --case decode=32*1@8192`.
|
||||
Multi-GPU: `torchrun --nproc-per-node=N -m vllm.entrypoints.cli.main bench
|
||||
forward ...`, one process per GPU.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
from dataclasses import asdict
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.benchmarks.forward.shapes import (
|
||||
BenchCase,
|
||||
jitter_kv_lens,
|
||||
load_cases,
|
||||
parse_case,
|
||||
)
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def add_cli_args(parser: FlexibleArgumentParser) -> None:
|
||||
parser.add_argument(
|
||||
"--case",
|
||||
action="append",
|
||||
default=[],
|
||||
help="`NAME=REQS`, where REQS is a comma list of `[N*]Q[@K][c][:D]` "
|
||||
"(query length Q, KV length K including the query (default Q), D draft "
|
||||
"tokens, N requests). A prefill's prefix is a prefix-cache hit unless c "
|
||||
"marks a chunked prefill. E.g. `decode=32*1@8192`, "
|
||||
"`verify=16*4@4096:3`, `prefill=8192`, `hit=2048@10240`, "
|
||||
"`mix=8192,32*1@8192`. Repeatable.",
|
||||
)
|
||||
parser.add_argument("--shapes", help="JSON or YAML file with cases.")
|
||||
parser.add_argument(
|
||||
"--kv-len-jitter",
|
||||
type=float,
|
||||
default=0.5,
|
||||
help="Draw each request's prefix (KV length minus query length) "
|
||||
"uniformly within +-this fraction of its value, like the varied lengths "
|
||||
"of a real batch, capped at the max model length. Seeded by --seed only, "
|
||||
"so every DP rank gets the same lengths. 0 keeps the exact lengths.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--kv-init",
|
||||
choices=["zeros"],
|
||||
default="zeros",
|
||||
help="Initial KV cache contents of each case's blocks.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--routing",
|
||||
choices=["auto", "real", "balanced"],
|
||||
default="auto",
|
||||
help="MoE routing. balanced: fixed routing with equal load per EP rank. "
|
||||
"auto: balanced with dummy weights, else real.",
|
||||
)
|
||||
parser.add_argument("--iters", type=int, default=20, help="Replays per sample.")
|
||||
parser.add_argument("--samples", type=int, default=5)
|
||||
parser.add_argument("--output-json", help="Write results to this file.")
|
||||
parser.add_argument(
|
||||
"--profile-dir",
|
||||
help="After timing, trace a few replays of each case and write "
|
||||
"`<case>.rank<r>.trace.json.gz` (Perfetto) and `<case>.rank<r>.kernels.json` "
|
||||
"(GPU time per kernel per forward) here. Summarize or compare with "
|
||||
"`python -m vllm.benchmarks.forward.profile A.kernels.json [B.kernels.json]`.",
|
||||
)
|
||||
EngineArgs.add_cli_args(parser)
|
||||
parser.set_defaults(load_format="dummy", enable_prefix_caching=False)
|
||||
|
||||
|
||||
def _setup_env() -> None:
|
||||
os.environ["VLLM_USE_V2_MODEL_RUNNER"] = "1"
|
||||
os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] = "0"
|
||||
# Each case's eager pass compiles exactly the shapes it needs.
|
||||
os.environ.setdefault("VLLM_DEEP_GEMM_WARMUP", "skip")
|
||||
if "RANK" in os.environ:
|
||||
return # Launched by torchrun.
|
||||
with socket.socket() as s:
|
||||
s.bind(("127.0.0.1", 0))
|
||||
port = s.getsockname()[1]
|
||||
os.environ.update(
|
||||
RANK="0",
|
||||
LOCAL_RANK="0",
|
||||
WORLD_SIZE="1",
|
||||
MASTER_ADDR="127.0.0.1",
|
||||
MASTER_PORT=str(port),
|
||||
)
|
||||
|
||||
|
||||
def _make_engine_args(args: argparse.Namespace, cases: list[BenchCase]):
|
||||
from vllm.config.compilation import CompilationMode, CUDAGraphMode
|
||||
|
||||
engine_args = EngineArgs.from_cli_args(args)
|
||||
engine_args.distributed_executor_backend = "external_launcher"
|
||||
engine_args.enable_prefix_caching = False
|
||||
engine_args.compilation_config.mode = CompilationMode.NONE
|
||||
engine_args.compilation_config.cudagraph_mode = CUDAGraphMode.NONE
|
||||
if engine_args.max_num_seqs is None:
|
||||
engine_args.max_num_seqs = max(len(c.requests) for c in cases)
|
||||
if engine_args.max_num_batched_tokens is None:
|
||||
# Prefix hits may replay up to a window more per request.
|
||||
engine_args.max_num_batched_tokens = max(
|
||||
8192,
|
||||
*(
|
||||
c.num_tokens + 1024 * sum(r.prefix_hit for r in c.requests)
|
||||
for c in cases
|
||||
),
|
||||
)
|
||||
return engine_args
|
||||
|
||||
|
||||
def run_case(runner, case: BenchCase, args: argparse.Namespace) -> dict[str, Any]:
|
||||
required_drafts = max((r.num_draft_tokens for r in case.requests), default=0)
|
||||
if required_drafts > runner.num_speculative_steps:
|
||||
raise ValueError(
|
||||
f"Case {case.name!r} needs {required_drafts} draft tokens per request, "
|
||||
f"but the runner supports {runner.num_speculative_steps}. "
|
||||
"Set --speculative-config with a sufficient num_speculative_tokens."
|
||||
)
|
||||
|
||||
from vllm.benchmarks.forward.batch import build_batch
|
||||
from vllm.benchmarks.forward.capture import ForwardBench
|
||||
from vllm.benchmarks.forward.kv_init import touched_kv_views, zero_kv_blocks
|
||||
from vllm.benchmarks.forward.routing import install_routing
|
||||
|
||||
context = runner.vllm_config.compilation_config.static_forward_context
|
||||
kv_config = runner.kv_cache_config
|
||||
dcp_size = runner.parallel_config.decode_context_parallel_size
|
||||
rng = np.random.default_rng([args.seed, runner.dp_rank])
|
||||
batch = build_batch(
|
||||
case,
|
||||
kv_config.kv_cache_groups,
|
||||
kv_config.num_blocks,
|
||||
runner.model_config.get_vocab_size(),
|
||||
rng,
|
||||
dcp_size,
|
||||
)
|
||||
zero_kv_blocks(context, kv_config.kv_cache_groups, batch, kv_config.num_blocks)
|
||||
profile_prefix = None
|
||||
if args.profile_dir:
|
||||
from vllm.distributed.parallel_state import get_world_group
|
||||
|
||||
os.makedirs(args.profile_dir, exist_ok=True)
|
||||
rank = get_world_group().rank
|
||||
profile_prefix = os.path.join(args.profile_dir, f"{case.name}.rank{rank}")
|
||||
bench = ForwardBench(
|
||||
runner,
|
||||
batch,
|
||||
args.iters,
|
||||
args.samples,
|
||||
touched_kv_views(
|
||||
context, kv_config.kv_cache_groups, batch, kv_config.num_blocks, dcp_size
|
||||
),
|
||||
case.name,
|
||||
profile_prefix,
|
||||
)
|
||||
routing, balanced = install_routing(
|
||||
runner, args.routing, args.load_format == "dummy", batch.num_tokens, args.seed
|
||||
)
|
||||
try:
|
||||
result = bench.run()
|
||||
if balanced is not None:
|
||||
balanced.check_applied()
|
||||
finally:
|
||||
runner.execute_model(batch.finish())
|
||||
if balanced is not None:
|
||||
balanced.remove()
|
||||
return {
|
||||
"num_tokens": batch.num_tokens,
|
||||
"kv_blocks": batch.num_blocks,
|
||||
"routing": routing,
|
||||
**result,
|
||||
}
|
||||
|
||||
|
||||
def main(args: argparse.Namespace) -> None:
|
||||
cases = [parse_case(c) for c in args.case]
|
||||
if args.shapes:
|
||||
cases += load_cases(args.shapes)
|
||||
if not cases:
|
||||
raise ValueError("Pass at least one --case or --shapes.")
|
||||
if not 0 <= args.kv_len_jitter < 1:
|
||||
raise ValueError("--kv-len-jitter must be in [0, 1).")
|
||||
_setup_env()
|
||||
|
||||
from vllm import LLM
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.distributed.parallel_state import get_world_group
|
||||
|
||||
llm = LLM.from_engine_args(_make_engine_args(args, cases))
|
||||
runner = llm.llm_engine.model_executor.driver_worker.worker.model_runner
|
||||
# Same draw on every rank: shapes are balanced across DP ranks.
|
||||
max_kv_len = runner.model_config.max_model_len
|
||||
cases = [
|
||||
jitter_kv_lens(c, args.kv_len_jitter, max_kv_len, args.seed) for c in cases
|
||||
]
|
||||
world = get_world_group()
|
||||
results = []
|
||||
with torch.inference_mode(), set_current_vllm_config(runner.vllm_config):
|
||||
for case in cases:
|
||||
result = run_case(runner, case, args)
|
||||
gathered: list[Any] = [None] * world.world_size
|
||||
torch.distributed.all_gather_object(gathered, result, group=world.cpu_group)
|
||||
if world.rank != 0:
|
||||
continue
|
||||
rank0 = gathered[0]
|
||||
latency = rank0["latency_us"] # Each replay is the max over ranks.
|
||||
results.append(
|
||||
{
|
||||
"case": case.name,
|
||||
"requests": [asdict(r) for r in case.requests],
|
||||
"median_us": latency["median"],
|
||||
"ranks": gathered,
|
||||
}
|
||||
)
|
||||
print(
|
||||
f"[forward] {case.name}: {len(case.requests)} reqs, "
|
||||
f"{rank0['num_tokens']} tokens, median {latency['median']:.1f} us, "
|
||||
f"sample spread {latency['sample_spread_pct']:.2f}%, graph vs "
|
||||
f"eager rel rms {rank0['graph_vs_eager']['relative_rms']:.1e}",
|
||||
flush=True,
|
||||
)
|
||||
if profile := rank0.get("profile"):
|
||||
from vllm.benchmarks.forward.profile import format_summary
|
||||
|
||||
print(
|
||||
f"[forward] {case.name} rank 0 kernels ({profile['summary']}):\n"
|
||||
+ format_summary({**profile, "kernels": profile["top_kernels"]}),
|
||||
flush=True,
|
||||
)
|
||||
if latency["sample_spread_pct"] > 0.5:
|
||||
logger.warning(
|
||||
"%s: unstable timing (sample medians spread %.2f%%).",
|
||||
case.name,
|
||||
latency["sample_spread_pct"],
|
||||
)
|
||||
if world.rank == 0 and args.output_json:
|
||||
with open(args.output_json, "w") as f:
|
||||
json.dump(results, f, indent=2)
|
||||
@@ -1,236 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""GPU kernel profiles of `vllm bench forward` cases.
|
||||
|
||||
With `--profile-dir`, each case's graph is replayed a few more times under
|
||||
`torch.profiler` after timing. Each rank writes a Perfetto trace and a kernel
|
||||
summary (`<case>.rank<r>.kernels.json`): per-kernel time per forward, sorted.
|
||||
|
||||
Summarize one file or compare two (e.g. before/after a change):
|
||||
`python -m vllm.benchmarks.forward.profile A.kernels.json [B.kernels.json]`.
|
||||
|
||||
The same replays run between `cudaProfilerStart`/`Stop` in an NVTX range per
|
||||
case (with or without `--profile-dir`), so Nsight Systems can record just them:
|
||||
`nsys profile --capture-range=cudaProfilerApi --capture-range-end=repeat
|
||||
--cuda-graph-trace=node vllm bench forward ...`.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import json
|
||||
import statistics
|
||||
from collections import defaultdict
|
||||
from typing import Any
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
|
||||
_GPU_CATEGORIES = ("kernel", "gpu_memcpy", "gpu_memset")
|
||||
# Kernels named after a generic launcher are identified by their first
|
||||
# template argument instead.
|
||||
_GENERIC = re.compile(r"(^|::)(device_kernel|Kernel|kernel|KernelWrapper)$")
|
||||
# CuTe DSL kernel names embed per-shape layouts after these markers.
|
||||
_CUTE_DSL_SUFFIX = re.compile(r"_object_at_|_tensorptr")
|
||||
|
||||
|
||||
def _match_back(text: str, end: int, open_ch: str, close_ch: str) -> int:
|
||||
"""Index of the bracket opening the one closing at `end`, skipping
|
||||
brackets of the other kind nested inside parentheses."""
|
||||
depth, parens = 0, 0
|
||||
for i in range(end, -1, -1):
|
||||
ch = text[i]
|
||||
if open_ch == "<" and ch in "()":
|
||||
parens += 1 if ch == ")" else -1
|
||||
continue
|
||||
if parens:
|
||||
continue
|
||||
if ch == close_ch:
|
||||
depth += 1
|
||||
elif ch == open_ch:
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return i
|
||||
return -1
|
||||
|
||||
|
||||
def short_name(name: str) -> str:
|
||||
"""`void ns::foo<T, 4>(args)` -> `ns::foo`: return type, template and
|
||||
function arguments dropped. Generic launchers (e.g.
|
||||
`cutlass::device_kernel<Op<...>>`) keep their first template argument."""
|
||||
if name.startswith("kernel_cutlass_"):
|
||||
return _CUTE_DSL_SUFFIX.split(name, maxsplit=1)[0]
|
||||
head = name.strip()
|
||||
if head.endswith(")"):
|
||||
head = head[: max(_match_back(head, len(head) - 1, "(", ")"), 0)].rstrip()
|
||||
template = ""
|
||||
if head.endswith(">"):
|
||||
start = _match_back(head, len(head) - 1, "<", ">")
|
||||
if start > 0:
|
||||
head, template = head[:start], head[start + 1 : -1]
|
||||
base = head.split()[-1] if head.split() else name
|
||||
if _GENERIC.search(base) and template:
|
||||
arg = re.split(r"[<>(,]", template, maxsplit=1)[0].strip()
|
||||
if arg:
|
||||
return f"{base}<{arg.removeprefix('void ')}>"
|
||||
return base
|
||||
|
||||
|
||||
def summarize(events: list[dict], replays: int, skip: int) -> dict[str, Any]:
|
||||
"""Per-forward kernel times from the GPU activities of `replays` replays.
|
||||
|
||||
Each replay first runs `skip` activities that aren't part of the timed
|
||||
forward (the KV state restore and the rank rendezvous); they are reported
|
||||
separately as `pre_forward_us`.
|
||||
"""
|
||||
acts = sorted(
|
||||
(e for e in events if e.get("cat") in _GPU_CATEGORIES and "dur" in e),
|
||||
key=lambda e: e["ts"],
|
||||
)
|
||||
if not acts:
|
||||
raise ValueError(
|
||||
"No GPU activity recorded. Another profiler (e.g. nsys) may hold "
|
||||
"CUPTI; don't combine --profile-dir with it."
|
||||
)
|
||||
if len(acts) % replays:
|
||||
raise ValueError(
|
||||
f"{len(acts)} GPU activities don't split into {replays} replays"
|
||||
)
|
||||
per_replay = len(acts) // replays
|
||||
if per_replay <= skip:
|
||||
raise ValueError(f"{per_replay} activities per replay, {skip} to skip")
|
||||
kernels: dict[str, dict[str, Any]] = defaultdict(
|
||||
lambda: {"total_us": 0.0, "calls": 0, "example": ""}
|
||||
)
|
||||
spans, busy, pre = [], [], []
|
||||
for r in range(replays):
|
||||
chunk = acts[r * per_replay : (r + 1) * per_replay]
|
||||
pre.append(sum(e["dur"] for e in chunk[:skip]))
|
||||
forward = chunk[skip:]
|
||||
spans.append(
|
||||
max(e["ts"] + e["dur"] for e in forward) - min(e["ts"] for e in forward)
|
||||
)
|
||||
busy.append(sum(e["dur"] for e in forward))
|
||||
for e in forward:
|
||||
name = short_name(e["name"]) if e["cat"] == "kernel" else e["name"]
|
||||
k = kernels[name]
|
||||
k["total_us"] += e["dur"]
|
||||
k["calls"] += 1
|
||||
k["example"] = k["example"] or e["name"][:300]
|
||||
total = sum(k["total_us"] for k in kernels.values())
|
||||
rows = [
|
||||
{
|
||||
"name": name,
|
||||
"us": round(k["total_us"] / replays, 2),
|
||||
"pct": round(100 * k["total_us"] / total, 2),
|
||||
"calls": k["calls"] // replays,
|
||||
"example": k["example"],
|
||||
}
|
||||
for name, k in kernels.items()
|
||||
]
|
||||
rows.sort(key=lambda row: -row["us"])
|
||||
return {
|
||||
"replays": replays,
|
||||
# First to last forward activity; busy can exceed it when streams overlap.
|
||||
"span_us": round(statistics.median(spans), 2),
|
||||
"busy_us": round(statistics.median(busy), 2),
|
||||
"pre_forward_us": round(statistics.median(pre), 2),
|
||||
"activities": per_replay - skip,
|
||||
"kernels": rows,
|
||||
}
|
||||
|
||||
|
||||
def profile_graph(
|
||||
graph: torch.cuda.CUDAGraph,
|
||||
name: str,
|
||||
prefix: str | None,
|
||||
skip: int,
|
||||
replays: int = 3,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Replay `replays` times between `cudaProfilerStart`/`Stop`, in an NVTX
|
||||
range named after the case, for `nsys --capture-range=cudaProfilerApi`.
|
||||
|
||||
With `prefix`, also trace them with `torch.profiler` and write
|
||||
`prefix.trace.json.gz` and `prefix.kernels.json`. Every rank must call
|
||||
this (replays may contain collectives).
|
||||
"""
|
||||
|
||||
def run() -> None:
|
||||
torch.cuda.profiler.start()
|
||||
with torch.cuda.nvtx.range(f"forward/{name}"):
|
||||
for _ in range(replays):
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
torch.cuda.profiler.stop()
|
||||
|
||||
if prefix is None:
|
||||
run()
|
||||
return None
|
||||
activities = [torch.profiler.ProfilerActivity.CUDA]
|
||||
with torch.profiler.profile(activities=activities) as prof:
|
||||
run()
|
||||
trace = f"{prefix}.trace.json.gz"
|
||||
prof.export_chrome_trace(trace)
|
||||
with gzip.open(trace, "rt") as f:
|
||||
events = json.load(f)["traceEvents"]
|
||||
summary = {"trace": trace, **summarize(events, replays, skip)}
|
||||
with open(f"{prefix}.kernels.json", "w") as f:
|
||||
json.dump(summary, f, indent=1)
|
||||
return summary
|
||||
|
||||
|
||||
def format_summary(summary: dict[str, Any], top: int = 20) -> str:
|
||||
lines = [
|
||||
f"span {summary['span_us']:.1f} us, busy {summary['busy_us']:.1f} us, "
|
||||
f"{summary['activities']} activities/forward",
|
||||
f"{'us':>10} {'%':>6} {'calls':>6} kernel",
|
||||
]
|
||||
for row in summary["kernels"][:top]:
|
||||
lines.append(
|
||||
f"{row['us']:10.1f} {row['pct']:6.2f} {row['calls']:6d} {row['name']}"
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def format_diff(a: dict[str, Any], b: dict[str, Any], top: int = 20) -> str:
|
||||
rows_a = {row["name"]: row for row in a["kernels"]}
|
||||
rows_b = {row["name"]: row for row in b["kernels"]}
|
||||
deltas = []
|
||||
for name in rows_a.keys() | rows_b.keys():
|
||||
ra, rb = rows_a.get(name), rows_b.get(name)
|
||||
us_a, us_b = ra["us"] if ra else 0.0, rb["us"] if rb else 0.0
|
||||
calls_a, calls_b = ra["calls"] if ra else 0, rb["calls"] if rb else 0
|
||||
deltas.append((us_b - us_a, us_a, us_b, calls_a, calls_b, name))
|
||||
deltas.sort(key=lambda d: -abs(d[0]))
|
||||
lines = [
|
||||
f"span {a['span_us']:.1f} -> {b['span_us']:.1f} us "
|
||||
f"({b['span_us'] - a['span_us']:+.1f}), busy {a['busy_us']:.1f} -> "
|
||||
f"{b['busy_us']:.1f} us ({b['busy_us'] - a['busy_us']:+.1f})",
|
||||
f"{'delta us':>10} {'A us':>10} {'B us':>10} {'A n':>5} {'B n':>5} kernel",
|
||||
]
|
||||
for delta, us_a, us_b, calls_a, calls_b, name in deltas[:top]:
|
||||
lines.append(
|
||||
f"{delta:+10.1f} {us_a:10.1f} {us_b:10.1f} {calls_a:5d} {calls_b:5d} "
|
||||
f"{name}"
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument("summaries", nargs="+", help="One or two *.kernels.json.")
|
||||
parser.add_argument("--top", type=int, default=20)
|
||||
args = parser.parse_args()
|
||||
if len(args.summaries) > 2:
|
||||
parser.error("Pass one summary, or two to compare.")
|
||||
loaded = []
|
||||
for path in args.summaries:
|
||||
with open(path) as f:
|
||||
loaded.append(json.load(f))
|
||||
if len(loaded) == 1:
|
||||
print(format_summary(loaded[0], args.top))
|
||||
else:
|
||||
print(format_diff(loaded[0], loaded[1], args.top))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,227 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Balanced MoE routing for benchmarks with dummy weights.
|
||||
|
||||
Dummy gate weights give degenerate routing, which skews MoE and all-to-all
|
||||
cost. The routing still runs; its result is replaced by the fixture at
|
||||
whichever boundary the layer exposes:
|
||||
|
||||
- Gate output: fixed logits whose top-k is the fixture. Covers every backend
|
||||
that routes from logits, including monolithic kernels that run top-k
|
||||
internally (e.g. FlashInfer TRTLLM).
|
||||
- Experts input: models that compute top-k themselves and pass
|
||||
`topk_ids` to their experts (e.g. MegaMoE with a fused gate) get the
|
||||
fixture's IDs. The routing weights are kept; they don't affect cost.
|
||||
|
||||
Hash-routed layers ignore logits and keep their tables unless their experts
|
||||
take `topk_ids`.
|
||||
"""
|
||||
|
||||
import inspect
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Fixture logits: chosen experts score far above the rest under any monotonic
|
||||
# scoring function, so top-k picks exactly them.
|
||||
_CHOSEN_LOGIT = 4.0
|
||||
_OTHER_LOGIT = -4.0
|
||||
|
||||
|
||||
def balanced_topk_ids(
|
||||
num_tokens: int, num_experts: int, top_k: int, ep_size: int, rng
|
||||
) -> np.ndarray:
|
||||
"""Expert IDs [num_tokens, top_k] giving every EP rank equal load.
|
||||
|
||||
Each token's slots go round-robin over EP ranks, to distinct experts.
|
||||
"""
|
||||
assert num_experts % ep_size == 0
|
||||
local = num_experts // ep_size
|
||||
slots = np.arange(num_tokens)[:, None] * top_k + np.arange(top_k)
|
||||
ranks = slots % ep_size
|
||||
# Slots of one token that land on the same rank take consecutive experts.
|
||||
repeat = np.zeros_like(ranks)
|
||||
for j in range(1, top_k):
|
||||
repeat[:, j] = (ranks[:, :j] == ranks[:, j : j + 1]).sum(axis=1)
|
||||
assert repeat.max() < local
|
||||
base = rng.integers(local, size=(num_tokens, ep_size))
|
||||
offset = np.take_along_axis(base, ranks, axis=1)
|
||||
return ranks * local + (offset + repeat) % local
|
||||
|
||||
|
||||
def _moe_layers(model: torch.nn.Module) -> list[tuple[torch.nn.Module, GateLinear]]:
|
||||
"""(MoE module, gate) per layer, in model order."""
|
||||
layers: dict[GateLinear, torch.nn.Module] = {}
|
||||
for module in model.modules():
|
||||
gate = getattr(module, "gate", None)
|
||||
if not isinstance(gate, GateLinear) or gate in layers:
|
||||
continue
|
||||
if _top_k(module) is not None:
|
||||
layers[gate] = module
|
||||
return [(module, gate) for gate, module in layers.items()]
|
||||
|
||||
|
||||
def _top_k(module: torch.nn.Module) -> int | None:
|
||||
"""The MoE layer's top-k, from the module or its experts submodules."""
|
||||
for sub in module.modules():
|
||||
for name in ("top_k", "n_activated_experts"):
|
||||
top_k = getattr(sub, name, None)
|
||||
if isinstance(top_k, int):
|
||||
return top_k
|
||||
return None
|
||||
|
||||
|
||||
class BalancedRouting:
|
||||
"""Installs balanced routing for one case; `remove()` restores the model."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: torch.nn.Module,
|
||||
num_tokens: int,
|
||||
dp_rank: int,
|
||||
dp_size: int,
|
||||
ep_size: int,
|
||||
seed: int,
|
||||
device: torch.device,
|
||||
):
|
||||
layers = _moe_layers(model)
|
||||
if not layers:
|
||||
raise ValueError("No MoE gates found; use --routing real.")
|
||||
self._hooks = []
|
||||
self._biases: list[tuple[torch.Tensor, torch.Tensor]] = []
|
||||
self._fired = [0] * len(layers)
|
||||
self._layers = layers
|
||||
# Per-rank rows, with headroom for padding; every rank generates the
|
||||
# same global fixture and takes its slice.
|
||||
rows = -(-num_tokens // 256) * 256 + 256
|
||||
for layer, (module, gate) in enumerate(layers):
|
||||
if (getattr(module, "num_expert_group", None) or 1) > 1:
|
||||
raise NotImplementedError("Balanced routing with grouped top-k.")
|
||||
num_experts = gate.weight.shape[0]
|
||||
top_k = _top_k(module)
|
||||
assert top_k is not None
|
||||
rng = np.random.default_rng([seed, layer])
|
||||
ids = balanced_topk_ids(rows * dp_size, num_experts, top_k, ep_size, rng)[
|
||||
dp_rank * rows : (dp_rank + 1) * rows
|
||||
]
|
||||
logits = torch.full((rows, num_experts), _OTHER_LOGIT, dtype=torch.float32)
|
||||
logits.scatter_(1, torch.from_numpy(ids), _CHOSEN_LOGIT)
|
||||
# Distinct values: no ties for any top-k implementation.
|
||||
logits += torch.from_numpy(rng.uniform(0, 0.5, logits.shape)).float()
|
||||
self._hooks.append(
|
||||
gate.register_forward_hook(
|
||||
self._substitute(logits.to(device), self._fired, layer)
|
||||
)
|
||||
)
|
||||
bias = getattr(gate, "e_score_correction_bias", None)
|
||||
if bias is None:
|
||||
bias = getattr(module, "e_score_correction_bias", None)
|
||||
if isinstance(bias, torch.Tensor):
|
||||
self._biases.append((bias, bias.detach().clone()))
|
||||
bias.data.zero_()
|
||||
experts = getattr(module, "experts", None)
|
||||
if isinstance(experts, torch.nn.Module):
|
||||
params = list(inspect.signature(experts.forward).parameters)
|
||||
if "topk_ids" in params:
|
||||
self._hooks.append(
|
||||
experts.register_forward_pre_hook(
|
||||
self._substitute_ids(
|
||||
torch.from_numpy(ids).to(device),
|
||||
params.index("topk_ids"),
|
||||
self._fired,
|
||||
layer,
|
||||
),
|
||||
with_kwargs=True,
|
||||
)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _substitute(fixture: torch.Tensor, fired: list[int], layer: int):
|
||||
cast = [fixture]
|
||||
|
||||
def hook(module, inputs, output):
|
||||
fired[layer] += 1
|
||||
logits = output[0] if isinstance(output, tuple) else output
|
||||
if cast[0].dtype != logits.dtype:
|
||||
cast[0] = fixture.to(logits.dtype) # Once, at warmup.
|
||||
replaced = cast[0][: logits.shape[0]]
|
||||
return (replaced, *output[1:]) if isinstance(output, tuple) else replaced
|
||||
|
||||
return hook
|
||||
|
||||
@staticmethod
|
||||
def _substitute_ids(fixture: torch.Tensor, index: int, fired, layer: int):
|
||||
cast = [fixture]
|
||||
|
||||
def hook(module, args, kwargs):
|
||||
fired[layer] += 1
|
||||
ids = kwargs["topk_ids"] if "topk_ids" in kwargs else args[index]
|
||||
if cast[0].dtype != ids.dtype:
|
||||
cast[0] = fixture.to(ids.dtype) # Once, at warmup.
|
||||
replaced = cast[0][: ids.shape[0]]
|
||||
if "topk_ids" in kwargs:
|
||||
return args, {**kwargs, "topk_ids": replaced}
|
||||
return (*args[:index], replaced, *args[index + 1 :]), kwargs
|
||||
|
||||
return hook
|
||||
|
||||
def check_applied(self) -> None:
|
||||
"""Fail if a layer bypassed both the gate and the experts hooks."""
|
||||
missed = [
|
||||
type(module).__name__
|
||||
for (module, _), fired in zip(self._layers, self._fired)
|
||||
if not fired
|
||||
]
|
||||
if missed:
|
||||
raise RuntimeError(
|
||||
f"Balanced routing didn't reach {len(missed)} MoE layers "
|
||||
f"({missed[0]}): their gate is fused away. Use --routing real."
|
||||
)
|
||||
|
||||
def remove(self) -> None:
|
||||
for hook in self._hooks:
|
||||
hook.remove()
|
||||
for bias, saved in self._biases:
|
||||
bias.data.copy_(saved)
|
||||
|
||||
|
||||
def install_routing(
|
||||
runner, requested: str, dummy_weights: bool, num_tokens: int, seed: int
|
||||
) -> tuple[str, BalancedRouting | None]:
|
||||
"""Resolve `--routing` and install balanced routing if selected.
|
||||
|
||||
`auto` is balanced with dummy weights and real otherwise; it falls back to
|
||||
real routing when the model doesn't support the fixture.
|
||||
"""
|
||||
mode = requested
|
||||
if mode == "auto":
|
||||
mode = "balanced" if dummy_weights else "real"
|
||||
if mode == "real":
|
||||
return mode, None
|
||||
parallel = runner.parallel_config
|
||||
ep_size = (
|
||||
parallel.data_parallel_size * parallel.tensor_parallel_size
|
||||
if parallel.enable_expert_parallel
|
||||
else 1
|
||||
)
|
||||
try:
|
||||
routing = BalancedRouting(
|
||||
runner.model,
|
||||
num_tokens,
|
||||
runner.dp_rank,
|
||||
runner.dp_size,
|
||||
ep_size,
|
||||
seed,
|
||||
runner.device,
|
||||
)
|
||||
except (ValueError, NotImplementedError) as e:
|
||||
if requested == "balanced":
|
||||
raise
|
||||
logger.warning("Using real routing: %s", e)
|
||||
return "real", None
|
||||
return mode, routing
|
||||
@@ -1,137 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Exact batch shapes for `vllm bench forward`."""
|
||||
|
||||
import json
|
||||
import zlib
|
||||
from dataclasses import dataclass, replace
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import regex as re
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BenchRequest:
|
||||
query_len: int
|
||||
# KV length after the step: the prefix already in the KV cache plus the
|
||||
# query tokens.
|
||||
kv_len: int
|
||||
# Spec-decode verify: query_len == 1 + num_draft_tokens.
|
||||
num_draft_tokens: int = 0
|
||||
# The prefix was computed by this request's earlier chunks. Otherwise a
|
||||
# prefill's prefix is a prefix-cache hit, of which models with SWA bounded
|
||||
# replay (DeepSeek-V4.1) recompute the last window.
|
||||
chunked: bool = False
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if (
|
||||
self.query_len < 1
|
||||
or self.kv_len < self.query_len
|
||||
or self.num_draft_tokens < 0
|
||||
):
|
||||
raise ValueError(f"Invalid request: {self}")
|
||||
if self.num_draft_tokens and not self.is_decode:
|
||||
raise ValueError(
|
||||
f"Verify needs query_len == 1 + num_draft_tokens and a prefix: {self}"
|
||||
)
|
||||
if self.chunked and (self.is_decode or not self.prefix_len):
|
||||
raise ValueError(f"A chunk needs a prefill with a prefix: {self}")
|
||||
|
||||
@property
|
||||
def prefix_len(self) -> int:
|
||||
return self.kv_len - self.query_len
|
||||
|
||||
@property
|
||||
def is_decode(self) -> bool:
|
||||
"""Decode or verify; otherwise a prefill."""
|
||||
return self.prefix_len > 0 and self.query_len == 1 + self.num_draft_tokens
|
||||
|
||||
@property
|
||||
def prefix_hit(self) -> bool:
|
||||
return self.prefix_len > 0 and not self.is_decode and not self.chunked
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BenchCase:
|
||||
name: str
|
||||
requests: tuple[BenchRequest, ...]
|
||||
|
||||
@property
|
||||
def num_tokens(self) -> int:
|
||||
return sum(r.query_len for r in self.requests)
|
||||
|
||||
|
||||
def jitter_kv_lens(
|
||||
case: BenchCase, jitter: float, max_kv_len: int, seed: int
|
||||
) -> BenchCase:
|
||||
"""Draw each request's prefix uniformly within +-jitter of its value.
|
||||
|
||||
Query lengths are kept, so a fresh prefill stays one and a non-empty
|
||||
prefix stays non-empty. KV lengths are capped at `max_kv_len`. The draw
|
||||
depends only on the seed and the case's name, not on the other cases.
|
||||
"""
|
||||
rng = np.random.default_rng([seed, zlib.crc32(case.name.encode())])
|
||||
requests = []
|
||||
for req in case.requests:
|
||||
if req.kv_len > max_kv_len:
|
||||
raise ValueError(f"{case.name}: KV length {req.kv_len} > {max_kv_len}")
|
||||
prefix = req.prefix_len
|
||||
if prefix and jitter:
|
||||
prefix = max(1, round(prefix * rng.uniform(1 - jitter, 1 + jitter)))
|
||||
kv_len = min(req.query_len + prefix, max_kv_len)
|
||||
requests.append(replace(req, kv_len=kv_len))
|
||||
return replace(case, requests=tuple(requests))
|
||||
|
||||
|
||||
_REQ_RE = re.compile(r"^(?:(\d+)\*)?(\d+)(?:@(\d+))?(c)?(?::(\d+))?$")
|
||||
|
||||
|
||||
def parse_case(text: str) -> BenchCase:
|
||||
"""Parse `NAME=REQS`, where REQS is a comma list of `[N*]Q[@K][c][:D]`.
|
||||
|
||||
Q is the query length, K the KV length including the query (default Q, a
|
||||
fresh prefill), D the number of draft tokens (verify) and N the number of
|
||||
such requests. A prefill's prefix is a prefix-cache hit unless `c` marks
|
||||
it as a chunked prefill. For example, `decode=32*1@8192`,
|
||||
`verify=16*4@4096:3`, `prefill=8192`, `hit=8192@40960`,
|
||||
`chunk=8192@40960c` or `mix=8192,32*1@8192`.
|
||||
"""
|
||||
name, sep, body = text.partition("=")
|
||||
if not sep:
|
||||
name, body = text, text
|
||||
requests: list[BenchRequest] = []
|
||||
for item in body.split(","):
|
||||
match = _REQ_RE.match(item.strip())
|
||||
if match is None:
|
||||
raise ValueError(f"Invalid request {item!r} in case {text!r}")
|
||||
count, query, kv, chunked, drafts = match.groups()
|
||||
request = BenchRequest(
|
||||
int(query), int(kv or query), int(drafts or 0), chunked=bool(chunked)
|
||||
)
|
||||
requests.extend([request] * int(count or 1))
|
||||
return BenchCase(name, tuple(requests))
|
||||
|
||||
|
||||
def load_cases(path: str) -> list[BenchCase]:
|
||||
"""Load cases from a JSON or YAML file.
|
||||
|
||||
Format: `{"cases": [{"name": ..., "requests": [{"query_len": ...,
|
||||
"kv_len": ..., "num_draft_tokens": ..., "chunked": ..., "count": ...}]}]}`.
|
||||
"""
|
||||
text = Path(path).read_text()
|
||||
if path.endswith((".yaml", ".yml")):
|
||||
import yaml
|
||||
|
||||
data = yaml.safe_load(text)
|
||||
else:
|
||||
data = json.loads(text)
|
||||
cases = []
|
||||
for case in data["cases"]:
|
||||
requests: list[BenchRequest] = []
|
||||
for req in case["requests"]:
|
||||
req = dict(req)
|
||||
count = req.pop("count", 1)
|
||||
requests.extend([BenchRequest(**req)] * count)
|
||||
cases.append(BenchCase(case["name"], tuple(requests)))
|
||||
return cases
|
||||
@@ -1,180 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Benchmark the latency of processing a single batch of requests."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
from tqdm import tqdm
|
||||
|
||||
from vllm.benchmarks.lib.utils import convert_to_pytorch_benchmark_format, write_to_json
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.inputs import TextPrompt, TokensPrompt
|
||||
from vllm.sampling_params import BeamSearchParams
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
|
||||
def save_to_pytorch_benchmark_format(
|
||||
args: argparse.Namespace, results: dict[str, Any]
|
||||
) -> None:
|
||||
pt_records = convert_to_pytorch_benchmark_format(
|
||||
args=args,
|
||||
metrics={"latency": results["latencies"]},
|
||||
extra_info={k: results[k] for k in ["avg_latency", "percentiles"]},
|
||||
)
|
||||
if pt_records:
|
||||
pt_file = f"{os.path.splitext(args.output_json)[0]}.pytorch.json"
|
||||
write_to_json(pt_file, pt_records)
|
||||
|
||||
|
||||
def add_cli_args(parser: FlexibleArgumentParser):
|
||||
parser.add_argument("--input-len", type=int, default=32)
|
||||
parser.add_argument("--output-len", type=int, default=128)
|
||||
parser.add_argument("--batch-size", type=int, default=8)
|
||||
parser.add_argument(
|
||||
"--n",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Number of generated sequences per prompt.",
|
||||
)
|
||||
parser.add_argument("--use-beam-search", action="store_true")
|
||||
parser.add_argument(
|
||||
"--num-iters-warmup",
|
||||
type=int,
|
||||
default=10,
|
||||
help="Number of iterations to run for warmup.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num-iters", type=int, default=30, help="Number of iterations to run."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--profile",
|
||||
action="store_true",
|
||||
help="profile the generation process of a single batch",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-json",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to save the latency results in JSON format.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--disable-detokenize",
|
||||
action="store_true",
|
||||
help=(
|
||||
"Do not detokenize responses (i.e. do not include "
|
||||
"detokenization time in the latency measurement)"
|
||||
),
|
||||
)
|
||||
|
||||
parser = EngineArgs.add_cli_args(parser)
|
||||
# V1 enables prefix caching by default which skews the latency
|
||||
# numbers. We need to disable prefix caching by default.
|
||||
parser.set_defaults(enable_prefix_caching=False)
|
||||
|
||||
|
||||
def main(args: argparse.Namespace):
|
||||
engine_args = EngineArgs.from_cli_args(args)
|
||||
|
||||
# Lazy import to avoid importing LLM when the bench command is not selected.
|
||||
from vllm import LLM, SamplingParams
|
||||
|
||||
# NOTE(woosuk): If the request cannot be processed in a single batch,
|
||||
# the engine will automatically process the request in multiple batches.
|
||||
llm = LLM.from_engine_args(engine_args)
|
||||
assert llm.llm_engine.model_config.max_model_len >= (
|
||||
args.input_len + args.output_len
|
||||
), (
|
||||
"Please ensure that max_model_len is greater than"
|
||||
" the sum of input_len and output_len."
|
||||
)
|
||||
|
||||
sampling_params = SamplingParams(
|
||||
n=args.n,
|
||||
temperature=1.0,
|
||||
top_p=1.0,
|
||||
ignore_eos=True,
|
||||
max_tokens=args.output_len,
|
||||
detokenize=not args.disable_detokenize,
|
||||
)
|
||||
dummy_prompt_token_ids = np.random.randint(
|
||||
10000, size=(args.batch_size, args.input_len)
|
||||
)
|
||||
dummy_prompts: list[TokensPrompt | TextPrompt] = [
|
||||
TokensPrompt(prompt_token_ids=batch)
|
||||
for batch in dummy_prompt_token_ids.tolist()
|
||||
]
|
||||
|
||||
def llm_generate():
|
||||
if not args.use_beam_search:
|
||||
llm.generate(dummy_prompts, sampling_params=sampling_params, use_tqdm=False)
|
||||
else:
|
||||
llm.beam_search(
|
||||
dummy_prompts,
|
||||
BeamSearchParams(
|
||||
beam_width=args.n,
|
||||
max_tokens=args.output_len,
|
||||
ignore_eos=True,
|
||||
watermarking=False,
|
||||
),
|
||||
)
|
||||
|
||||
def run_to_completion(do_profile: bool = False):
|
||||
if do_profile:
|
||||
llm.start_profile()
|
||||
llm_generate()
|
||||
llm.stop_profile()
|
||||
else:
|
||||
start_time = time.perf_counter()
|
||||
llm_generate()
|
||||
end_time = time.perf_counter()
|
||||
latency = end_time - start_time
|
||||
return latency
|
||||
|
||||
print("Warming up...")
|
||||
for _ in tqdm(range(args.num_iters_warmup), desc="Warmup iterations"):
|
||||
run_to_completion(do_profile=False)
|
||||
|
||||
if args.profile:
|
||||
profiler_config = engine_args.profiler_config
|
||||
if profiler_config.profiler == "torch":
|
||||
print(
|
||||
"Profiling with torch profiler (results will be saved to"
|
||||
f" {profiler_config.torch_profiler_dir})..."
|
||||
)
|
||||
elif profiler_config.profiler == "cuda":
|
||||
print("Profiling with cuda profiler ...")
|
||||
elif profiler_config.profiler == "proton":
|
||||
print(
|
||||
"Profiling with Proton (results will be saved under"
|
||||
f" {profiler_config.proton_profiler_dir})..."
|
||||
)
|
||||
run_to_completion(do_profile=True)
|
||||
return
|
||||
|
||||
# Benchmark.
|
||||
latencies = []
|
||||
for _ in tqdm(range(args.num_iters), desc="Bench iterations"):
|
||||
latencies.append(run_to_completion(do_profile=False))
|
||||
latencies = np.array(latencies)
|
||||
percentages = [10, 25, 50, 75, 90, 99]
|
||||
percentiles = np.percentile(latencies, percentages)
|
||||
print(f"Avg latency: {np.mean(latencies)} seconds")
|
||||
for percentage, percentile in zip(percentages, percentiles):
|
||||
print(f"{percentage}% percentile latency: {percentile} seconds")
|
||||
|
||||
# Output JSON results if specified
|
||||
if args.output_json:
|
||||
results = {
|
||||
"model_id": args.model,
|
||||
"avg_latency": np.mean(latencies),
|
||||
"latencies": latencies.tolist(),
|
||||
"percentiles": dict(zip(percentages, percentiles.tolist())),
|
||||
}
|
||||
with open(args.output_json, "w") as f:
|
||||
json.dump(results, f, indent=4)
|
||||
save_to_pytorch_benchmark_format(args, results)
|
||||
@@ -1,3 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Benchmark library utilities."""
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,80 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Utilities for checking endpoint readiness."""
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
|
||||
import aiohttp
|
||||
from tqdm.asyncio import tqdm
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from .endpoint_request_func import RequestFunc, RequestFuncInput, RequestFuncOutput
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
async def wait_for_endpoint(
|
||||
request_func: RequestFunc,
|
||||
test_input: RequestFuncInput,
|
||||
session: aiohttp.ClientSession,
|
||||
timeout_seconds: int = 600,
|
||||
retry_interval: int = 5,
|
||||
) -> RequestFuncOutput:
|
||||
"""Wait for an endpoint to become available before starting benchmarks.
|
||||
|
||||
Args:
|
||||
request_func: The async request function to call
|
||||
test_input: The RequestFuncInput to test with
|
||||
session: The aiohttp session used to issue the probe requests
|
||||
timeout_seconds: Maximum time to wait in seconds (default: 10 minutes)
|
||||
retry_interval: Time between retries in seconds (default: 5 seconds)
|
||||
|
||||
Returns:
|
||||
RequestFuncOutput: The successful response
|
||||
|
||||
Raises:
|
||||
ValueError: If the endpoint doesn't become available within the timeout
|
||||
|
||||
"""
|
||||
deadline = time.perf_counter() + timeout_seconds
|
||||
output = RequestFuncOutput(success=False)
|
||||
print(f"Waiting for endpoint to become up in {timeout_seconds} seconds")
|
||||
|
||||
with tqdm(
|
||||
total=timeout_seconds,
|
||||
bar_format="{desc} |{bar}| {elapsed} elapsed, {remaining} remaining",
|
||||
unit="s",
|
||||
) as pbar:
|
||||
while True:
|
||||
# update progress bar
|
||||
remaining = deadline - time.perf_counter()
|
||||
elapsed = timeout_seconds - remaining
|
||||
update_amount = min(elapsed - pbar.n, timeout_seconds - pbar.n)
|
||||
pbar.update(update_amount)
|
||||
pbar.refresh()
|
||||
if remaining <= 0:
|
||||
pbar.close()
|
||||
break
|
||||
|
||||
# ping the endpoint using request_func
|
||||
try:
|
||||
output = await request_func(
|
||||
request_func_input=test_input, session=session
|
||||
)
|
||||
if output.success:
|
||||
pbar.close()
|
||||
return output
|
||||
else:
|
||||
err_last_line = str(output.error).rstrip().rsplit("\n", 1)[-1]
|
||||
logger.warning("Endpoint is not ready. Error='%s'", err_last_line)
|
||||
except aiohttp.ClientConnectorError:
|
||||
pass
|
||||
|
||||
# retry after a delay
|
||||
sleep_duration = min(retry_interval, remaining)
|
||||
if sleep_duration > 0:
|
||||
await asyncio.sleep(sleep_duration)
|
||||
|
||||
return output
|
||||
@@ -1,139 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
from typing import Any
|
||||
|
||||
|
||||
def redact_sensitive_namespace(
|
||||
args: argparse.Namespace, fields: tuple[str, ...]
|
||||
) -> argparse.Namespace:
|
||||
"""Return a copy of CLI arguments with selected values redacted."""
|
||||
redacted_args = argparse.Namespace(**vars(args))
|
||||
for field in fields:
|
||||
if getattr(redacted_args, field, None) is not None:
|
||||
setattr(redacted_args, field, "***")
|
||||
return redacted_args
|
||||
|
||||
|
||||
def extract_field(
|
||||
args: argparse.Namespace, extra_info: dict[str, Any], field_name: str
|
||||
) -> str:
|
||||
if field_name in extra_info:
|
||||
return extra_info[field_name]
|
||||
|
||||
v: Any = args
|
||||
# For example, args.compilation_config.mode
|
||||
for nested_field in field_name.split("."):
|
||||
if not hasattr(v, nested_field):
|
||||
return ""
|
||||
v = getattr(v, nested_field)
|
||||
return v
|
||||
|
||||
|
||||
def use_compile(args: argparse.Namespace, extra_info: dict[str, Any]) -> bool:
|
||||
"""Check if the benchmark is run with torch.compile."""
|
||||
return not (
|
||||
extract_field(args, extra_info, "compilation_config.mode") == "0"
|
||||
or "eager" in getattr(args, "output_json", "")
|
||||
or "eager" in getattr(args, "result_filename", "")
|
||||
)
|
||||
|
||||
|
||||
def convert_to_pytorch_benchmark_format(
|
||||
args: argparse.Namespace, metrics: dict[str, list], extra_info: dict[str, Any]
|
||||
) -> list:
|
||||
"""Save the benchmark results in the format used by PyTorch OSS benchmark with
|
||||
on metric per record
|
||||
https://github.com/pytorch/pytorch/wiki/How-to-integrate-with-PyTorch-OSS-benchmark-database
|
||||
"""
|
||||
records: list[Any] = []
|
||||
if not os.environ.get("SAVE_TO_PYTORCH_BENCHMARK_FORMAT", False):
|
||||
return records
|
||||
|
||||
for name, benchmark_values in metrics.items():
|
||||
if not isinstance(benchmark_values, list):
|
||||
raise TypeError(
|
||||
f"benchmark_values for metric '{name}' must be a list, "
|
||||
f"but got {type(benchmark_values).__name__}"
|
||||
)
|
||||
|
||||
record = {
|
||||
"benchmark": {
|
||||
"name": "vLLM benchmark",
|
||||
"extra_info": {
|
||||
"args": vars(args),
|
||||
"compilation_config.mode": extract_field(
|
||||
args, extra_info, "compilation_config.mode"
|
||||
),
|
||||
"optimization_level": extract_field(
|
||||
args, extra_info, "optimization_level"
|
||||
),
|
||||
# A boolean field used by vLLM benchmark HUD dashboard
|
||||
"use_compile": use_compile(args, extra_info),
|
||||
},
|
||||
},
|
||||
"model": {
|
||||
"name": args.model,
|
||||
},
|
||||
"metric": {
|
||||
"name": name,
|
||||
"benchmark_values": benchmark_values,
|
||||
"extra_info": extra_info,
|
||||
},
|
||||
}
|
||||
|
||||
tp = record["benchmark"]["extra_info"]["args"].get("tensor_parallel_size")
|
||||
# Save tensor_parallel_size parameter if it's part of the metadata
|
||||
if not tp and "tensor_parallel_size" in extra_info:
|
||||
record["benchmark"]["extra_info"]["args"]["tensor_parallel_size"] = (
|
||||
extra_info["tensor_parallel_size"]
|
||||
)
|
||||
|
||||
records.append(record)
|
||||
|
||||
return records
|
||||
|
||||
|
||||
class InfEncoder(json.JSONEncoder):
|
||||
def clear_inf(self, o: Any):
|
||||
if isinstance(o, dict):
|
||||
return {
|
||||
str(k)
|
||||
if not isinstance(k, (str, int, float, bool, type(None)))
|
||||
else k: self.clear_inf(v)
|
||||
for k, v in o.items()
|
||||
}
|
||||
elif isinstance(o, list):
|
||||
return [self.clear_inf(v) for v in o]
|
||||
elif isinstance(o, float) and math.isinf(o):
|
||||
return "inf"
|
||||
return o
|
||||
|
||||
def iterencode(self, o: Any, *args, **kwargs) -> Any:
|
||||
return super().iterencode(self.clear_inf(o), *args, **kwargs)
|
||||
|
||||
|
||||
def write_to_json(filename: str, records: list) -> None:
|
||||
with open(filename, "w") as f:
|
||||
json.dump(
|
||||
records,
|
||||
f,
|
||||
cls=InfEncoder,
|
||||
default=lambda o: f"<{type(o).__name__} is not JSON serializable>",
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def default_vllm_config():
|
||||
"""Set a default VllmConfig for cases that directly test CustomOps or pathways
|
||||
that use get_current_vllm_config() outside of a full engine context.
|
||||
"""
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
yield
|
||||
@@ -1,533 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
r"""Benchmark multimodal processor latency.
|
||||
|
||||
This benchmark measures the latency of the mm processor module
|
||||
using multimodal prompts from datasets.
|
||||
MM processor stats are automatically enabled.
|
||||
|
||||
Run:
|
||||
vllm bench mm-processor \
|
||||
--model <your_model> \
|
||||
--dataset-name random-mm \
|
||||
--num-prompts 10 \
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Any, Literal
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.benchmarks.datasets import (
|
||||
MultiModalConversationDataset,
|
||||
VisionArenaDataset,
|
||||
)
|
||||
from vllm.benchmarks.throughput import get_requests
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
from vllm.utils.gc_utils import freeze_gc_heap
|
||||
from vllm.utils.import_utils import PlaceholderModule
|
||||
|
||||
try:
|
||||
import pandas as pd
|
||||
except ImportError:
|
||||
pd = PlaceholderModule("pandas")
|
||||
|
||||
if TYPE_CHECKING: # Avoid having to mock during docs build
|
||||
from vllm.v1.engine.llm_engine import LLMEngine
|
||||
else:
|
||||
LLMEngine = object
|
||||
|
||||
|
||||
def get_timing_stats_from_engine(llm_engine: LLMEngine) -> dict[str, dict[str, float]]:
|
||||
"""Get all multimodal timing stats from the LLM engine.
|
||||
|
||||
Collects both preprocessing stats (HF processor, hashing, cache lookup,
|
||||
prompt update) and encoder forward pass timing, merged by request_id.
|
||||
|
||||
Args:
|
||||
llm_engine: The LLM engine (has input_processor and workers).
|
||||
|
||||
Returns:
|
||||
Dictionary mapping request_id to merged stats dict containing
|
||||
both preprocessing and encoder timing metrics.
|
||||
|
||||
Example:
|
||||
{
|
||||
'request-123': {
|
||||
'get_mm_hashes_secs': 0.02,
|
||||
'get_cache_missing_items_secs': 0.01,
|
||||
'apply_hf_processor_secs': 0.45,
|
||||
'merge_mm_kwargs_secs': 0.01,
|
||||
'apply_prompt_updates_secs': 0.03,
|
||||
'preprocessor_total_secs': 0.51,
|
||||
'encoder_forward_secs': 0.23,
|
||||
'num_encoder_calls': 1
|
||||
}
|
||||
}
|
||||
|
||||
"""
|
||||
observability_config = llm_engine.vllm_config.observability_config
|
||||
if not observability_config or not observability_config.enable_mm_processor_stats:
|
||||
return {}
|
||||
|
||||
renderer = llm_engine.renderer
|
||||
mm_processor_stats = renderer._mm_timing_registry.stat()
|
||||
|
||||
encoder_stats = dict[str, dict[str, float]]()
|
||||
for worker_stats in llm_engine.collective_rpc("get_encoder_timing_stats"):
|
||||
if not worker_stats:
|
||||
continue
|
||||
|
||||
for request_id, stats_dict in worker_stats.items():
|
||||
if request_id not in encoder_stats:
|
||||
encoder_stats[request_id] = dict(stats_dict)
|
||||
else:
|
||||
# Aggregate timing metrics across workers
|
||||
current_time = encoder_stats[request_id].get(
|
||||
"encoder_forward_secs", 0.0
|
||||
)
|
||||
new_time = stats_dict.get("encoder_forward_secs", 0.0)
|
||||
encoder_stats[request_id]["encoder_forward_secs"] = max(
|
||||
current_time, new_time
|
||||
)
|
||||
|
||||
current_calls = encoder_stats[request_id].get("num_encoder_calls", 0)
|
||||
new_calls = stats_dict.get("num_encoder_calls", 0)
|
||||
encoder_stats[request_id]["num_encoder_calls"] = max(
|
||||
current_calls, new_calls
|
||||
)
|
||||
|
||||
merged_stats = dict[str, dict[str, float]]()
|
||||
|
||||
for request_id, prep_dict in mm_processor_stats.items():
|
||||
merged_stats[request_id] = dict(prep_dict)
|
||||
|
||||
for request_id, enc_dict in encoder_stats.items():
|
||||
if request_id in merged_stats:
|
||||
merged_stats[request_id].update(enc_dict)
|
||||
continue
|
||||
|
||||
# In V1 engine, the request_id in encoder_stats has a suffix
|
||||
# appended to the original request_id (which is used in
|
||||
# preprocessing_stats).
|
||||
# We try to strip the suffix to find the matching request.
|
||||
possible_original_id = request_id.rpartition("-")[0]
|
||||
if possible_original_id and possible_original_id in merged_stats:
|
||||
merged_stats[possible_original_id].update(enc_dict)
|
||||
else:
|
||||
merged_stats[request_id] = dict(enc_dict)
|
||||
|
||||
return merged_stats
|
||||
|
||||
|
||||
def collect_mm_processor_stats(llm_engine: LLMEngine) -> dict[str, list[float]]:
|
||||
"""Collect multimodal processor timing stats.
|
||||
Returns a dictionary mapping stage names to lists of timing values (in seconds).
|
||||
"""
|
||||
all_stats = get_timing_stats_from_engine(llm_engine)
|
||||
|
||||
stats_by_stage = defaultdict[str, list[float]](list)
|
||||
|
||||
for stats_dict in all_stats.values():
|
||||
for stat_key, stat_val in stats_dict.items():
|
||||
stats_by_stage[stat_key].append(stat_val)
|
||||
|
||||
return stats_by_stage
|
||||
|
||||
|
||||
def calculate_mm_processor_metrics(
|
||||
stats_by_stage: dict[str, list[float]],
|
||||
selected_percentiles: list[float],
|
||||
*,
|
||||
unit: Literal["us", "ms", "s"] = "ms",
|
||||
) -> dict[str, dict[str, float]]:
|
||||
"""Calculate aggregate metrics from stats by stage."""
|
||||
unit2mult = {"us": 1000000, "ms": 1000, "s": 1}
|
||||
unit_mult = unit2mult[unit]
|
||||
|
||||
metrics = {}
|
||||
|
||||
for stage, times in stats_by_stage.items():
|
||||
stage_name = stage.replace("_secs", "_" + unit)
|
||||
|
||||
if not times:
|
||||
metrics[stage_name] = {
|
||||
"mean": 0.0,
|
||||
"median": 0.0,
|
||||
"std": 0.0,
|
||||
**{f"p{p}": 0.0 for p in selected_percentiles},
|
||||
}
|
||||
continue
|
||||
|
||||
is_count_metric = stage == "num_encoder_calls"
|
||||
values = times if is_count_metric else [t * unit_mult for t in times]
|
||||
|
||||
metrics[stage_name] = {
|
||||
"mean": float(np.mean(values)),
|
||||
"median": float(np.median(values)),
|
||||
"std": float(np.std(values)),
|
||||
**{f"p{p}": float(np.percentile(values, p)) for p in selected_percentiles},
|
||||
}
|
||||
|
||||
return metrics
|
||||
|
||||
|
||||
def validate_args(args):
|
||||
"""Validate command-line arguments for mm_processor benchmark."""
|
||||
if not getattr(args, "tokenizer", None):
|
||||
args.tokenizer = args.model
|
||||
if not hasattr(args, "dataset_path"):
|
||||
args.dataset_path = None
|
||||
if not hasattr(args, "lora_path"):
|
||||
args.lora_path = None
|
||||
if not hasattr(args, "max_loras"):
|
||||
args.max_loras = None
|
||||
|
||||
if args.dataset_name == "hf" and not args.dataset_path:
|
||||
raise ValueError(
|
||||
"--dataset-path is required when using --dataset-name hf. "
|
||||
"For multimodal benchmarking, specify a dataset like "
|
||||
"'lmarena-ai/VisionArena-Chat'."
|
||||
)
|
||||
if args.dataset_name == "hf":
|
||||
supported_mm_datasets = (
|
||||
VisionArenaDataset.SUPPORTED_DATASET_PATHS.keys()
|
||||
| MultiModalConversationDataset.SUPPORTED_DATASET_PATHS
|
||||
)
|
||||
if args.dataset_path not in supported_mm_datasets:
|
||||
raise ValueError(
|
||||
f"{args.dataset_path} is not a supported multimodal dataset. "
|
||||
f"Supported multimodal datasets are: {sorted(supported_mm_datasets)}"
|
||||
)
|
||||
|
||||
|
||||
def benchmark_multimodal_processor(
|
||||
args: argparse.Namespace,
|
||||
) -> dict[str, Any]:
|
||||
"""Run the multimodal processor benchmark."""
|
||||
from vllm import LLM, SamplingParams
|
||||
|
||||
validate_args(args)
|
||||
|
||||
if args.seed is None:
|
||||
args.seed = 0
|
||||
|
||||
engine_args = EngineArgs.from_cli_args(args)
|
||||
llm = LLM.from_engine_args(engine_args)
|
||||
|
||||
tokenizer = llm.get_tokenizer()
|
||||
requests = get_requests(args, tokenizer)
|
||||
|
||||
assert all(
|
||||
llm.llm_engine.model_config.max_model_len
|
||||
>= (request.prompt_len + request.expected_output_len)
|
||||
for request in requests
|
||||
), (
|
||||
"Please ensure that max_model_len is greater than the sum of "
|
||||
"prompt_len and expected_output_len for all requests."
|
||||
)
|
||||
|
||||
prompts = [request.prompt for request in requests]
|
||||
expected_output_lens = [request.expected_output_len for request in requests]
|
||||
|
||||
sampling_params = [
|
||||
SamplingParams(
|
||||
n=1,
|
||||
temperature=0.0,
|
||||
max_tokens=output_len,
|
||||
detokenize=True,
|
||||
watermarking=False,
|
||||
)
|
||||
for output_len in expected_output_lens
|
||||
]
|
||||
|
||||
selected_percentiles = [
|
||||
float(p) for p in getattr(args, "metric_percentiles", "99").split(",")
|
||||
]
|
||||
|
||||
freeze_gc_heap()
|
||||
|
||||
num_warmups = getattr(args, "num_warmups", 0)
|
||||
if num_warmups > 0:
|
||||
print(f"Processing {num_warmups} warmup requests...")
|
||||
# Create a temporary args object for warmup requests
|
||||
warmup_args = argparse.Namespace(**vars(args))
|
||||
warmup_args.num_prompts = num_warmups
|
||||
warmup_args.seed += 1
|
||||
warmup_requests = get_requests(warmup_args, tokenizer)
|
||||
warmup_prompts = [req.prompt for req in warmup_requests]
|
||||
warmup_output_lens = [req.expected_output_len for req in warmup_requests]
|
||||
warmup_sampling_params = [
|
||||
SamplingParams(max_tokens=output_len, watermarking=False)
|
||||
for output_len in warmup_output_lens
|
||||
]
|
||||
llm.chat(
|
||||
warmup_prompts,
|
||||
warmup_sampling_params,
|
||||
use_tqdm=not getattr(args, "disable_tqdm", False),
|
||||
)
|
||||
|
||||
# Clear stats from warmup requests
|
||||
collect_mm_processor_stats(llm.llm_engine)
|
||||
|
||||
print(f"Processing {len(prompts)} requests...")
|
||||
start_time = time.perf_counter()
|
||||
|
||||
outputs = llm.chat(
|
||||
prompts, sampling_params, use_tqdm=not getattr(args, "disable_tqdm", False)
|
||||
)
|
||||
|
||||
end_time = time.perf_counter()
|
||||
total_time = end_time - start_time
|
||||
|
||||
mm_stats_by_stage = collect_mm_processor_stats(llm.llm_engine)
|
||||
|
||||
if not any(mm_stats_by_stage.values()):
|
||||
print(
|
||||
"\n⚠️ Warning: No MM processor stats found in registry.\n"
|
||||
" This may indicate that:\n"
|
||||
" - No multimodal requests were processed\n"
|
||||
" - Stats were already retrieved (registry is cleared after retrieval)\n"
|
||||
)
|
||||
|
||||
mm_processor_metrics = calculate_mm_processor_metrics(
|
||||
mm_stats_by_stage, selected_percentiles
|
||||
)
|
||||
|
||||
completed = len([o for o in outputs if o.finished])
|
||||
failed = len(outputs) - completed
|
||||
|
||||
e2el_times = []
|
||||
for output in outputs:
|
||||
if not output.finished or output.metrics is None:
|
||||
continue
|
||||
metrics = output.metrics
|
||||
# Calculate E2E latency as: TTFT + (last_token_ts - first_token_ts)
|
||||
if (
|
||||
getattr(metrics, "first_token_latency", None) is not None
|
||||
and getattr(metrics, "last_token_ts", None) is not None
|
||||
and getattr(metrics, "first_token_ts", None) is not None
|
||||
):
|
||||
ttft = metrics.first_token_latency
|
||||
# Decode time is the duration between the first and last token generation
|
||||
decode_time = max(0.0, metrics.last_token_ts - metrics.first_token_ts)
|
||||
e2el_times.append((ttft + decode_time) * 1000)
|
||||
|
||||
if not e2el_times and completed > 0:
|
||||
print(
|
||||
"\n⚠️ Warning: Detailed end-to-end latency metrics not available.\n"
|
||||
" Falling back to average request latency "
|
||||
"(total_time / num_completed_requests).\n"
|
||||
)
|
||||
avg_time_per_request = total_time / completed
|
||||
e2el_times = [avg_time_per_request * 1000] * completed
|
||||
|
||||
if e2el_times:
|
||||
mean_e2el_ms = float(np.mean(e2el_times))
|
||||
median_e2el_ms = float(np.median(e2el_times))
|
||||
std_e2el_ms = float(np.std(e2el_times))
|
||||
percentiles_e2el_ms = [
|
||||
(p, float(np.percentile(e2el_times, p))) for p in selected_percentiles
|
||||
]
|
||||
else:
|
||||
mean_e2el_ms = 0.0
|
||||
median_e2el_ms = 0.0
|
||||
std_e2el_ms = 0.0
|
||||
percentiles_e2el_ms = [(p, 0.0) for p in selected_percentiles]
|
||||
|
||||
encoder_summary = {}
|
||||
if (
|
||||
"num_encoder_calls" in mm_stats_by_stage
|
||||
and mm_stats_by_stage["num_encoder_calls"]
|
||||
):
|
||||
encoder_calls = mm_stats_by_stage["num_encoder_calls"]
|
||||
encoder_summary = {
|
||||
"total_encoder_calls": int(sum(encoder_calls)),
|
||||
"num_requests_with_encoder_calls": len(encoder_calls),
|
||||
}
|
||||
|
||||
benchmark_result = {
|
||||
"completed": completed,
|
||||
"failed": failed,
|
||||
"mean_e2el_ms": mean_e2el_ms,
|
||||
"median_e2el_ms": median_e2el_ms,
|
||||
"std_e2el_ms": std_e2el_ms,
|
||||
"percentiles_e2el_ms": percentiles_e2el_ms,
|
||||
"mm_processor_stats": mm_processor_metrics,
|
||||
"encoder_summary": encoder_summary,
|
||||
}
|
||||
|
||||
return benchmark_result
|
||||
|
||||
|
||||
def add_cli_args(parser: FlexibleArgumentParser) -> None:
|
||||
"""Add CLI arguments for the multimodal processor benchmark."""
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
|
||||
EngineArgs.add_cli_args(parser)
|
||||
|
||||
parser.set_defaults(enable_mm_processor_stats=True)
|
||||
|
||||
parser.add_argument(
|
||||
"--dataset-name",
|
||||
type=str,
|
||||
default="random-mm",
|
||||
choices=["random-mm", "hf"],
|
||||
help="Name of the dataset to benchmark on. Defaults to 'random-mm'.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num-prompts",
|
||||
type=int,
|
||||
default=10,
|
||||
help="Number of prompts to process.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num-warmups",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Number of warmup prompts to process.",
|
||||
)
|
||||
|
||||
from vllm.benchmarks.datasets import (
|
||||
add_random_dataset_base_args,
|
||||
add_random_multimodal_dataset_args,
|
||||
)
|
||||
|
||||
add_random_dataset_base_args(parser)
|
||||
add_random_multimodal_dataset_args(parser)
|
||||
|
||||
# HuggingFace dataset arguments
|
||||
parser.add_argument(
|
||||
"--dataset-path",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to the dataset file or HuggingFace dataset name "
|
||||
"(e.g., 'yale-nlp/MMVU', 'lmarena-ai/VisionArena-Chat').",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--hf-subset",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Subset of the HuggingFace dataset (optional).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--hf-split",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Split of the HuggingFace dataset (e.g., 'train', 'test', 'validation').",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-len",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Output length for each request. "
|
||||
"Overrides the default output lengths from the dataset.",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--output-json",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to save the benchmark results in JSON format.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--metric-percentiles",
|
||||
type=str,
|
||||
default="99",
|
||||
help="Comma-separated list of percentiles to calculate (e.g., '50,90,99').",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--disable-tqdm",
|
||||
action="store_true",
|
||||
help="Disable tqdm progress bar.",
|
||||
)
|
||||
|
||||
|
||||
def main(args: argparse.Namespace) -> None:
|
||||
"""Main entry point for the multimodal processor benchmark."""
|
||||
print("Starting multimodal processor benchmark...")
|
||||
result = benchmark_multimodal_processor(args)
|
||||
|
||||
print("\n" + "=" * 80)
|
||||
print("Multimodal Processor Benchmark Results")
|
||||
print("=" * 80)
|
||||
|
||||
if "mm_processor_stats" in result:
|
||||
print("\nMM Processor Metrics:")
|
||||
selected_percentiles = [
|
||||
float(p) for p in getattr(args, "metric_percentiles", "99").split(",")
|
||||
]
|
||||
mm_data = []
|
||||
for stage, metrics in result["mm_processor_stats"].items():
|
||||
row = {
|
||||
"Stage": stage,
|
||||
"Mean": f"{metrics['mean']:.2f}",
|
||||
"Median": f"{metrics['median']:.2f}",
|
||||
"Std": f"{metrics['std']:.2f}",
|
||||
}
|
||||
for p in selected_percentiles:
|
||||
row[f"P{p}"] = f"{metrics.get(f'p{p}', 0.0):.2f}"
|
||||
mm_data.append(row)
|
||||
|
||||
mm_df = pd.DataFrame(mm_data)
|
||||
print(mm_df.to_string(index=False))
|
||||
|
||||
if "encoder_summary" in result and result["encoder_summary"]:
|
||||
total_calls = result["encoder_summary"]["total_encoder_calls"]
|
||||
num_requests = result["encoder_summary"]["num_requests_with_encoder_calls"]
|
||||
print(
|
||||
f"\nSummary: {total_calls} total encoder calls "
|
||||
f"across {num_requests} requests."
|
||||
)
|
||||
|
||||
if "mean_e2el_ms" in result:
|
||||
print("\nEnd-to-End Latency (ms):")
|
||||
selected_percentiles = [
|
||||
float(p) for p in getattr(args, "metric_percentiles", "99").split(",")
|
||||
]
|
||||
|
||||
e2el_data = [
|
||||
{"Metric": "Mean", "Value (ms)": f"{result['mean_e2el_ms']:.2f}"},
|
||||
{"Metric": "Median", "Value (ms)": f"{result['median_e2el_ms']:.2f}"},
|
||||
{"Metric": "Std", "Value (ms)": f"{result['std_e2el_ms']:.2f}"},
|
||||
]
|
||||
|
||||
for p in selected_percentiles:
|
||||
percentile_value = next(
|
||||
(val for pct, val in result["percentiles_e2el_ms"] if pct == p),
|
||||
0.0,
|
||||
)
|
||||
e2el_data.append(
|
||||
{
|
||||
"Metric": f"P{p}",
|
||||
"Value (ms)": f"{percentile_value:.2f}",
|
||||
}
|
||||
)
|
||||
|
||||
e2el_df = pd.DataFrame(e2el_data)
|
||||
print(e2el_df.to_string(index=False))
|
||||
|
||||
if args.output_json:
|
||||
result["config"] = {
|
||||
"model": args.model,
|
||||
"num_prompts": args.num_prompts,
|
||||
"input_len": getattr(args, "random_input_len", None),
|
||||
"output_len": getattr(args, "random_output_len", None),
|
||||
}
|
||||
result["timestamp"] = datetime.now().isoformat()
|
||||
|
||||
with open(args.output_json, "w") as f:
|
||||
json.dump(result, f, indent=2)
|
||||
print(f"\nResults saved to {args.output_json}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = FlexibleArgumentParser(description="Benchmark mm processor latency")
|
||||
add_cli_args(parser)
|
||||
args = parser.parse_args()
|
||||
main(args)
|
||||
@@ -1,315 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Generate plots for benchmark results."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from vllm.utils.import_utils import PlaceholderModule
|
||||
|
||||
try:
|
||||
import plotly.express as px
|
||||
import plotly.io as pio
|
||||
except ImportError:
|
||||
_plotly = PlaceholderModule("plotly")
|
||||
px = _plotly.placeholder_attr("express")
|
||||
pio = _plotly.placeholder_attr("io")
|
||||
|
||||
try:
|
||||
import matplotlib.pyplot as plt
|
||||
except ImportError:
|
||||
_matplotlib = PlaceholderModule("matplotlib")
|
||||
plt = _matplotlib.placeholder_attr("pyplot")
|
||||
|
||||
|
||||
def generate_timeline_plot(
|
||||
results: list[dict[str, Any]],
|
||||
output_path: Path,
|
||||
colors: list[str] | None = None,
|
||||
itl_thresholds: list[float] | None = None,
|
||||
labels: list[str] | None = None,
|
||||
) -> None:
|
||||
"""Generate an HTML timeline plot from benchmark results.
|
||||
|
||||
Args:
|
||||
results: List of per-request result dictionaries containing:
|
||||
- start_time: Request start time (seconds)
|
||||
- ttft: Time to first token (seconds)
|
||||
- itl: List of inter-token latencies (seconds)
|
||||
- latency: Total request latency (seconds)
|
||||
- prompt_len: Number of prompt tokens
|
||||
- output_tokens: Number of output tokens
|
||||
output_path: Path where the HTML file will be saved
|
||||
colors: List of colors for ITL categories (default: green, orange, red, black)
|
||||
itl_thresholds: ITL thresholds in seconds (default: [1.0, 4.0, 6.0])
|
||||
labels: Labels for ITL categories (default based on thresholds)
|
||||
|
||||
"""
|
||||
# Set defaults
|
||||
if colors is None:
|
||||
colors = ["#109618", "#FF7F0E", "#D62728"]
|
||||
if itl_thresholds is None:
|
||||
itl_thresholds = [0.025, 0.050]
|
||||
if labels is None:
|
||||
labels = [
|
||||
f"ITL < {itl_thresholds[0] * 1000:.0f}ms",
|
||||
f"{itl_thresholds[0] * 1000:.0f}ms ≤ ITL < {itl_thresholds[1] * 1000:.0f}ms", # noqa
|
||||
f"ITL ≥ {itl_thresholds[1] * 1000:.0f}ms",
|
||||
]
|
||||
|
||||
labels_colors = {"TTFT": "#636EFA", **dict(zip(labels, colors))}
|
||||
labels_order = ["TTFT"] + labels
|
||||
|
||||
timeline_data = construct_timeline_data(results, itl_thresholds, labels)
|
||||
|
||||
if not timeline_data:
|
||||
print("No timeline data to plot")
|
||||
return
|
||||
|
||||
# Create the plot
|
||||
fig = px.timeline(
|
||||
timeline_data,
|
||||
x_start="start",
|
||||
x_end="end",
|
||||
y="request_id",
|
||||
color="type",
|
||||
color_discrete_map=labels_colors,
|
||||
category_orders={"type": labels_order},
|
||||
hover_data=[
|
||||
"prompt_tokens",
|
||||
"output_tokens",
|
||||
"req_start_time",
|
||||
"req_finish_time",
|
||||
"segment_start",
|
||||
"segment_end",
|
||||
"duration",
|
||||
],
|
||||
)
|
||||
|
||||
# Customize hover template to show only time without date
|
||||
fig.update_traces(
|
||||
hovertemplate="<b>%{y}</b><br>"
|
||||
"Type: %{fullData.name}<br>"
|
||||
"Start: %{customdata[4]}<br>"
|
||||
"End: %{customdata[5]}<br>"
|
||||
"Duration: %{customdata[6]}<br>"
|
||||
"Prompt Tokens: %{customdata[0]}<br>"
|
||||
"Output Tokens: %{customdata[1]}<br>"
|
||||
"Request Start Time: %{customdata[2]}<br>"
|
||||
"Request End Time: %{customdata[3]}<br>"
|
||||
"<extra></extra>"
|
||||
)
|
||||
|
||||
fig.update_yaxes(autorange="reversed")
|
||||
fig.update_layout(
|
||||
xaxis_title="Time",
|
||||
yaxis_title="Request ID",
|
||||
showlegend=True,
|
||||
)
|
||||
|
||||
# Save to HTML
|
||||
pio.write_html(fig, str(output_path))
|
||||
print(f"Timeline plot saved to: {output_path}")
|
||||
|
||||
|
||||
def construct_timeline_data(
|
||||
requests_data: list[dict[str, Any]],
|
||||
itl_thresholds: list[float],
|
||||
labels: list[str],
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Construct timeline data from request results.
|
||||
|
||||
Args:
|
||||
requests_data: List of per-request result dictionaries
|
||||
itl_thresholds: ITL thresholds in seconds
|
||||
labels: Labels for ITL categories
|
||||
|
||||
Returns:
|
||||
List of timeline segments for plotting
|
||||
|
||||
"""
|
||||
|
||||
def tostr(sec_time: float) -> str:
|
||||
"""Convert seconds to HH:MM:SS.mmm format."""
|
||||
h = int(sec_time // 3600)
|
||||
assert h < 100, "time seems to last more than 100 hours"
|
||||
m = int((sec_time % 3600) // 60)
|
||||
s = sec_time % 60
|
||||
return f"{h:02d}:{m:02d}:{s:06.3f}"
|
||||
|
||||
def itl_type(itl: float) -> str:
|
||||
"""Categorize ITL based on thresholds."""
|
||||
if itl < itl_thresholds[0]:
|
||||
return labels[0]
|
||||
elif itl < itl_thresholds[1]:
|
||||
return labels[1]
|
||||
else:
|
||||
return labels[2]
|
||||
|
||||
# Find the earliest start time to use as t0
|
||||
t0 = None
|
||||
for request in requests_data:
|
||||
start_time = request.get("start_time")
|
||||
if start_time is not None and (t0 is None or start_time < t0):
|
||||
t0 = start_time
|
||||
|
||||
if t0 is None:
|
||||
return []
|
||||
|
||||
timeline_data = []
|
||||
|
||||
for i, request in enumerate(requests_data):
|
||||
start_time = request.get("start_time")
|
||||
ttft = request.get("ttft")
|
||||
itl = request.get("itl", [])
|
||||
latency = request.get("latency")
|
||||
prompt_len = request.get("prompt_len", 0)
|
||||
output_tokens = request.get("output_tokens", 0)
|
||||
|
||||
# Skip requests without required data
|
||||
if start_time is None or ttft is None or latency is None:
|
||||
continue
|
||||
|
||||
# Normalize start time
|
||||
start_time = start_time - t0
|
||||
start_time_str = tostr(start_time)
|
||||
|
||||
# TTFT segment
|
||||
ttft_end = start_time + ttft
|
||||
ttft_end_str = tostr(ttft_end)
|
||||
|
||||
timeline_data.append(
|
||||
{
|
||||
"request_id": f"Req {i}",
|
||||
"start": start_time_str,
|
||||
"end": ttft_end_str,
|
||||
"type": "TTFT",
|
||||
"prompt_tokens": prompt_len,
|
||||
"output_tokens": output_tokens,
|
||||
"req_start_time": tostr(start_time),
|
||||
"req_finish_time": tostr(start_time + latency),
|
||||
"segment_start": start_time_str,
|
||||
"segment_end": ttft_end_str,
|
||||
"duration": f"{ttft:.3f}s",
|
||||
}
|
||||
)
|
||||
|
||||
# ITL segments
|
||||
prev_time = ttft_end
|
||||
prev_time_str = ttft_end_str
|
||||
|
||||
for itl_value in itl:
|
||||
itl_end = prev_time + itl_value
|
||||
itl_end_str = tostr(itl_end)
|
||||
|
||||
timeline_data.append(
|
||||
{
|
||||
"request_id": f"Req {i}",
|
||||
"start": prev_time_str,
|
||||
"end": itl_end_str,
|
||||
"type": itl_type(itl_value),
|
||||
"prompt_tokens": prompt_len,
|
||||
"output_tokens": output_tokens,
|
||||
"req_start_time": tostr(start_time),
|
||||
"req_finish_time": tostr(start_time + latency),
|
||||
"segment_start": prev_time_str,
|
||||
"segment_end": itl_end_str,
|
||||
"duration": f"{itl_value:.3f}s",
|
||||
}
|
||||
)
|
||||
|
||||
prev_time = itl_end
|
||||
prev_time_str = itl_end_str
|
||||
|
||||
return timeline_data
|
||||
|
||||
|
||||
def generate_dataset_stats_plot(
|
||||
results: list[dict[str, Any]],
|
||||
output_path: Path,
|
||||
) -> None:
|
||||
"""Generate a matplotlib figure with dataset statistics.
|
||||
|
||||
Creates a figure with 4 subplots:
|
||||
- Top-left: Prompt tokens distribution (histogram)
|
||||
- Top-right: Output tokens distribution (histogram)
|
||||
- Bottom-left: Prompt+output tokens distribution (histogram)
|
||||
- Bottom-right: Stacked bar chart (request_id vs tokens)
|
||||
|
||||
Args:
|
||||
results: List of per-request result dictionaries containing:
|
||||
- prompt_len: Number of prompt tokens
|
||||
- output_tokens: Number of output tokens
|
||||
output_path: Path where the figure will be saved
|
||||
|
||||
"""
|
||||
# Extract data
|
||||
prompt_tokens = []
|
||||
output_tokens = []
|
||||
total_tokens = []
|
||||
|
||||
for request in results:
|
||||
prompt_len = request.get("prompt_len", 0)
|
||||
output_len = request.get("output_tokens", 0)
|
||||
|
||||
prompt_tokens.append(prompt_len)
|
||||
output_tokens.append(output_len)
|
||||
total_tokens.append(prompt_len + output_len)
|
||||
|
||||
if not prompt_tokens:
|
||||
print("No data available for dataset statistics plot")
|
||||
return
|
||||
|
||||
# Create figure with 4 subplots
|
||||
fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(14, 10))
|
||||
|
||||
# Top-left: Prompt tokens distribution
|
||||
ax1.hist(prompt_tokens, bins=30, color="steelblue", edgecolor="black", alpha=0.7)
|
||||
ax1.set_xlabel("Prompt Tokens")
|
||||
ax1.set_ylabel("Frequency")
|
||||
ax1.set_title("Prompt Tokens Distribution")
|
||||
ax1.grid(True, alpha=0.3)
|
||||
|
||||
# Top-right: Output tokens distribution
|
||||
ax2.hist(output_tokens, bins=30, color="coral", edgecolor="black", alpha=0.7)
|
||||
ax2.set_xlabel("Output Tokens")
|
||||
ax2.set_ylabel("Frequency")
|
||||
ax2.set_title("Output Tokens Distribution")
|
||||
ax2.grid(True, alpha=0.3)
|
||||
|
||||
# Bottom-left: Prompt+output tokens distribution
|
||||
ax3.hist(
|
||||
total_tokens, bins=30, color="mediumseagreen", edgecolor="black", alpha=0.7
|
||||
)
|
||||
ax3.set_xlabel("Total Tokens (Prompt + Output)")
|
||||
ax3.set_ylabel("Frequency")
|
||||
ax3.set_title("Total Tokens Distribution")
|
||||
ax3.grid(True, alpha=0.3)
|
||||
|
||||
# Bottom-right: Stacked bar chart
|
||||
request_ids = list(range(len(prompt_tokens)))
|
||||
ax4.bar(
|
||||
request_ids, prompt_tokens, label="Prompt Tokens", color="steelblue", alpha=0.7
|
||||
)
|
||||
ax4.bar(
|
||||
request_ids,
|
||||
output_tokens,
|
||||
bottom=prompt_tokens,
|
||||
label="Output Tokens",
|
||||
color="coral",
|
||||
alpha=0.7,
|
||||
)
|
||||
ax4.set_xlabel("Request ID")
|
||||
ax4.set_ylabel("Tokens")
|
||||
ax4.set_title("Tokens per Request (Stacked)")
|
||||
ax4.legend()
|
||||
ax4.grid(True, alpha=0.3, axis="y")
|
||||
|
||||
# Adjust layout to prevent overlap
|
||||
plt.tight_layout()
|
||||
|
||||
# Save figure
|
||||
plt.savefig(str(output_path), dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
|
||||
print(f"Dataset statistics plot saved to: {output_path}")
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,302 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Benchmark the cold and warm startup time of vLLM models.
|
||||
|
||||
This script measures total startup time (including model loading, compilation,
|
||||
and cache operations) for both cold and warm scenarios:
|
||||
- Cold startup: Fresh start with no caches (temporary cache directories)
|
||||
- Warm startup: Using cached compilation and model info
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import multiprocessing
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
import numpy as np
|
||||
from tqdm import tqdm
|
||||
|
||||
from vllm.benchmarks.lib.utils import (
|
||||
convert_to_pytorch_benchmark_format,
|
||||
write_to_json,
|
||||
)
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
PERCENTAGES = [10, 25, 50, 75, 90, 99]
|
||||
|
||||
|
||||
class MetricDesc(NamedTuple):
|
||||
"""Descriptor for a metric to collect from each iteration."""
|
||||
|
||||
iter_key: str # key in the iteration result dict
|
||||
suffix: str # result key suffix, e.g. "startup", "compilation"
|
||||
display_name: str
|
||||
|
||||
|
||||
class MetricStats(NamedTuple):
|
||||
"""Aggregated statistics for a single benchmark metric."""
|
||||
|
||||
key: str # e.g. "cold_startup", "warm_encoder_compilation"
|
||||
display_name: str
|
||||
values: list[float]
|
||||
avg: float
|
||||
percentiles: dict[int, float]
|
||||
|
||||
|
||||
_BASE_METRICS = [
|
||||
MetricDesc("total_startup_time", "startup", "Startup time"),
|
||||
MetricDesc("compilation_time", "compilation", "Compilation time"),
|
||||
]
|
||||
_ENCODER_METRIC = MetricDesc(
|
||||
"encoder_compilation_time",
|
||||
"encoder_compilation",
|
||||
"Encoder compilation time",
|
||||
)
|
||||
|
||||
|
||||
def _compute_metric(
|
||||
phase: str,
|
||||
desc: MetricDesc,
|
||||
iterations: list[dict[str, float]],
|
||||
) -> MetricStats:
|
||||
values = [m[desc.iter_key] for m in iterations]
|
||||
arr = np.array(values)
|
||||
return MetricStats(
|
||||
key=f"{phase}_{desc.suffix}",
|
||||
display_name=desc.display_name,
|
||||
values=values,
|
||||
avg=float(np.mean(arr)),
|
||||
percentiles=dict(zip(PERCENTAGES, np.percentile(arr, PERCENTAGES).tolist())),
|
||||
)
|
||||
|
||||
|
||||
def _collect_phase_metrics(
|
||||
phase: str,
|
||||
iterations: list[dict[str, float]],
|
||||
has_encoder: bool,
|
||||
) -> list[MetricStats]:
|
||||
metrics = [_compute_metric(phase, desc, iterations) for desc in _BASE_METRICS]
|
||||
if has_encoder:
|
||||
metrics.append(_compute_metric(phase, _ENCODER_METRIC, iterations))
|
||||
return metrics
|
||||
|
||||
|
||||
def _print_phase(phase_name: str, metrics: list[MetricStats]) -> None:
|
||||
print(f"\n{phase_name}:")
|
||||
for m in metrics:
|
||||
print(f"Avg {m.display_name.lower()}: {m.avg:.2f} seconds")
|
||||
for m in metrics:
|
||||
print(f"{m.display_name} percentiles:")
|
||||
for pct, val in m.percentiles.items():
|
||||
print(f" {pct}%: {val:.2f} seconds")
|
||||
|
||||
|
||||
def _metric_to_json(m: MetricStats) -> dict[str, Any]:
|
||||
return {
|
||||
f"avg_{m.key}_time": m.avg,
|
||||
f"{m.key}_times": m.values,
|
||||
f"{m.key}_percentiles": m.percentiles,
|
||||
}
|
||||
|
||||
|
||||
@contextmanager
|
||||
def cold_startup():
|
||||
"""Context manager to measure cold startup time:
|
||||
1. Uses a temporary directory for vLLM cache to avoid any pollution
|
||||
between cold startup iterations.
|
||||
2. Uses inductor's fresh_cache to clear torch.compile caches.
|
||||
"""
|
||||
from torch._inductor.utils import fresh_cache
|
||||
|
||||
# Use temporary directory for caching to avoid any pollution between cold startups
|
||||
original_cache_root = os.environ.get("VLLM_CACHE_ROOT")
|
||||
temp_cache_dir = tempfile.mkdtemp(prefix="vllm_startup_bench_cold_")
|
||||
try:
|
||||
os.environ["VLLM_CACHE_ROOT"] = temp_cache_dir
|
||||
with fresh_cache():
|
||||
yield
|
||||
finally:
|
||||
# Clean up temporary cache directory
|
||||
shutil.rmtree(temp_cache_dir, ignore_errors=True)
|
||||
if original_cache_root:
|
||||
os.environ["VLLM_CACHE_ROOT"] = original_cache_root
|
||||
else:
|
||||
os.environ.pop("VLLM_CACHE_ROOT", None)
|
||||
|
||||
|
||||
def run_startup_in_subprocess(engine_args, result_queue):
|
||||
"""Run LLM startup in a subprocess and return timing metrics via a queue.
|
||||
This ensures complete isolation between iterations.
|
||||
"""
|
||||
try:
|
||||
# Import inside the subprocess to avoid issues with forking
|
||||
from vllm import LLM
|
||||
|
||||
# Measure total startup time
|
||||
start_time = time.perf_counter()
|
||||
|
||||
llm = LLM.from_engine_args(engine_args)
|
||||
|
||||
total_startup_time = time.perf_counter() - start_time
|
||||
|
||||
# Extract compilation time if available
|
||||
compilation_time = 0.0
|
||||
encoder_compilation_time = 0.0
|
||||
if hasattr(llm.llm_engine, "vllm_config"):
|
||||
vllm_config = llm.llm_engine.vllm_config
|
||||
if (
|
||||
hasattr(vllm_config, "compilation_config")
|
||||
and vllm_config.compilation_config is not None
|
||||
):
|
||||
compilation_time = vllm_config.compilation_config.compilation_time
|
||||
encoder_compilation_time = (
|
||||
vllm_config.compilation_config.encoder_compilation_time
|
||||
)
|
||||
|
||||
result_queue.put(
|
||||
{
|
||||
"total_startup_time": total_startup_time,
|
||||
"compilation_time": compilation_time,
|
||||
"encoder_compilation_time": encoder_compilation_time,
|
||||
}
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
result_queue.put(None)
|
||||
result_queue.put(str(e))
|
||||
|
||||
|
||||
def save_to_pytorch_benchmark_format(
|
||||
args: argparse.Namespace, metrics: list[MetricStats]
|
||||
) -> None:
|
||||
base_name = os.path.splitext(args.output_json)[0]
|
||||
for m in metrics:
|
||||
records = convert_to_pytorch_benchmark_format(
|
||||
args=args,
|
||||
metrics={f"avg_{m.key}_time": [m.avg]},
|
||||
extra_info={
|
||||
f"{m.key}_times": m.values,
|
||||
f"{m.key}_percentiles": m.percentiles,
|
||||
},
|
||||
)
|
||||
if records:
|
||||
write_to_json(f"{base_name}.{m.key}.pytorch.json", records)
|
||||
|
||||
|
||||
def add_cli_args(parser: FlexibleArgumentParser):
|
||||
parser.add_argument(
|
||||
"--num-iters-cold",
|
||||
type=int,
|
||||
default=3,
|
||||
help="Number of cold startup iterations.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num-iters-warmup",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Number of warmup iterations before benchmarking warm startups.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num-iters-warm",
|
||||
type=int,
|
||||
default=3,
|
||||
help="Number of warm startup iterations.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-json",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to save the startup time results in JSON format.",
|
||||
)
|
||||
|
||||
parser = EngineArgs.add_cli_args(parser)
|
||||
return parser
|
||||
|
||||
|
||||
def main(args: argparse.Namespace):
|
||||
# Set multiprocessing start method to 'spawn' for clean process isolation
|
||||
# This ensures each subprocess starts fresh without inheriting state
|
||||
multiprocessing.set_start_method("spawn", force=True)
|
||||
|
||||
engine_args = EngineArgs.from_cli_args(args)
|
||||
|
||||
def create_llm_and_measure_startup():
|
||||
"""Create LLM instance in a subprocess and measure startup time.
|
||||
Returns timing metrics, using subprocess for complete isolation.
|
||||
"""
|
||||
# Create a queue for inter-process communication
|
||||
result_queue: multiprocessing.Queue[Any] = multiprocessing.Queue()
|
||||
process = multiprocessing.Process(
|
||||
target=run_startup_in_subprocess,
|
||||
args=(
|
||||
engine_args,
|
||||
result_queue,
|
||||
),
|
||||
)
|
||||
process.start()
|
||||
process.join()
|
||||
|
||||
if not result_queue.empty():
|
||||
result = result_queue.get()
|
||||
if result is None:
|
||||
if not result_queue.empty():
|
||||
error_msg = result_queue.get()
|
||||
raise RuntimeError(f"Subprocess failed: {error_msg}")
|
||||
else:
|
||||
raise RuntimeError("Subprocess failed with unknown error")
|
||||
return result
|
||||
else:
|
||||
raise RuntimeError("Subprocess did not return a result")
|
||||
|
||||
os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] = "0"
|
||||
print("Setting VLLM_ENABLE_V1_MULTIPROCESSING=0 to collect startup metrics.\n")
|
||||
|
||||
# Collect cold startup iterations
|
||||
print("Measuring cold startup time...\n")
|
||||
cold_iterations = []
|
||||
for i in tqdm(range(args.num_iters_cold), desc="Cold startup iterations"):
|
||||
with cold_startup():
|
||||
cold_iterations.append(create_llm_and_measure_startup())
|
||||
|
||||
# Warmup for warm startup
|
||||
print("\nWarming up for warm startup measurement...\n")
|
||||
for _ in tqdm(range(args.num_iters_warmup), desc="Warmup iterations"):
|
||||
create_llm_and_measure_startup()
|
||||
|
||||
# Collect warm startup iterations
|
||||
print("\nMeasuring warm startup time...\n")
|
||||
warm_iterations = []
|
||||
for i in tqdm(range(args.num_iters_warm), desc="Warm startup iterations"):
|
||||
warm_iterations.append(create_llm_and_measure_startup())
|
||||
|
||||
# Determine if encoder compilation occurred in any iteration
|
||||
has_encoder = any(
|
||||
m["encoder_compilation_time"] > 0 for m in cold_iterations + warm_iterations
|
||||
)
|
||||
|
||||
cold_metrics = _collect_phase_metrics("cold", cold_iterations, has_encoder)
|
||||
warm_metrics = _collect_phase_metrics("warm", warm_iterations, has_encoder)
|
||||
all_metrics = cold_metrics + warm_metrics
|
||||
|
||||
# Print results
|
||||
print("\n" + "=" * 60)
|
||||
print("STARTUP TIME BENCHMARK RESULTS")
|
||||
print("=" * 60)
|
||||
_print_phase("COLD STARTUP", cold_metrics)
|
||||
_print_phase("WARM STARTUP", warm_metrics)
|
||||
print("=" * 60)
|
||||
|
||||
# Output JSON results if specified
|
||||
if args.output_json:
|
||||
results: dict[str, Any] = {}
|
||||
for m in all_metrics:
|
||||
results.update(_metric_to_json(m))
|
||||
with open(args.output_json, "w") as f:
|
||||
json.dump(results, f, indent=4)
|
||||
save_to_pytorch_benchmark_format(args, all_metrics)
|
||||
@@ -1,45 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
|
||||
from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
from .plot import SweepPlotArgs
|
||||
from .plot import main as plot_main
|
||||
from .plot_pareto import SweepPlotParetoArgs
|
||||
from .plot_pareto import main as plot_pareto_main
|
||||
from .serve import SweepServeArgs
|
||||
from .serve import main as serve_main
|
||||
from .serve_workload import SweepServeWorkloadArgs
|
||||
from .serve_workload import main as serve_workload_main
|
||||
from .startup import SweepStartupArgs
|
||||
from .startup import main as startup_main
|
||||
|
||||
SUBCOMMANDS = (
|
||||
(SweepServeArgs, serve_main),
|
||||
(SweepServeWorkloadArgs, serve_workload_main),
|
||||
(SweepStartupArgs, startup_main),
|
||||
(SweepPlotArgs, plot_main),
|
||||
(SweepPlotParetoArgs, plot_pareto_main),
|
||||
)
|
||||
|
||||
|
||||
def add_cli_args(parser: FlexibleArgumentParser):
|
||||
subparsers = parser.add_subparsers(required=True, dest="sweep_type")
|
||||
|
||||
for cmd, entrypoint in SUBCOMMANDS:
|
||||
cmd_subparser = subparsers.add_parser(
|
||||
cmd.parser_name,
|
||||
description=cmd.parser_help,
|
||||
usage=f"vllm bench sweep {cmd.parser_name} [options]",
|
||||
)
|
||||
cmd_subparser.set_defaults(dispatch_function=entrypoint)
|
||||
cmd.add_cli_args(cmd_subparser)
|
||||
cmd_subparser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(
|
||||
subcmd=f"sweep {cmd.parser_name}"
|
||||
)
|
||||
|
||||
|
||||
def main(args: argparse.Namespace):
|
||||
args.dispatch_function(args)
|
||||
@@ -1,156 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
import os
|
||||
|
||||
|
||||
class ParameterSweep(list["ParameterSweepItem"]):
|
||||
@classmethod
|
||||
def read_json(cls, filepath: os.PathLike):
|
||||
with open(filepath, "rb") as f:
|
||||
data = json.load(f)
|
||||
|
||||
# Support both list and dict formats
|
||||
if isinstance(data, dict):
|
||||
return cls.read_from_dict(data)
|
||||
|
||||
return cls.from_records(data)
|
||||
|
||||
@classmethod
|
||||
def read_from_dict(cls, data: dict[str, dict[str, object]]):
|
||||
"""Read parameter sweep from a dict format where keys are names.
|
||||
|
||||
Example:
|
||||
{
|
||||
"experiment1": {"max_tokens": 100, "temperature": 0.7},
|
||||
"experiment2": {"max_tokens": 200, "temperature": 0.9}
|
||||
}
|
||||
|
||||
"""
|
||||
records = [{"_benchmark_name": name, **params} for name, params in data.items()]
|
||||
return cls.from_records(records)
|
||||
|
||||
@classmethod
|
||||
def from_records(cls, records: list[dict[str, object]]):
|
||||
if not isinstance(records, list):
|
||||
raise TypeError(
|
||||
f"The parameter sweep should be a list of dictionaries, "
|
||||
f"but found type: {type(records)}"
|
||||
)
|
||||
|
||||
# Validate that all _benchmark_name values are unique if provided
|
||||
names = [r["_benchmark_name"] for r in records if "_benchmark_name" in r]
|
||||
if names and len(names) != len(set(names)):
|
||||
duplicates = [name for name in names if names.count(name) > 1]
|
||||
raise ValueError(
|
||||
f"Duplicate _benchmark_name values found: {set(duplicates)}. "
|
||||
f"All _benchmark_name values must be unique."
|
||||
)
|
||||
|
||||
return cls(ParameterSweepItem.from_record(record) for record in records)
|
||||
|
||||
|
||||
class ParameterSweepItem(dict[str, object]):
|
||||
@classmethod
|
||||
def from_record(cls, record: dict[str, object]):
|
||||
if not isinstance(record, dict):
|
||||
raise TypeError(
|
||||
f"Each item in the parameter sweep should be a dictionary, "
|
||||
f"but found type: {type(record)}"
|
||||
)
|
||||
|
||||
return cls(record)
|
||||
|
||||
def __or__(self, other: dict[str, object], /) -> "ParameterSweepItem": # type: ignore[override]
|
||||
return ParameterSweepItem(super().__or__(other))
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
"""Get the name for this parameter sweep item.
|
||||
|
||||
Returns the '_benchmark_name' field if present, otherwise returns a text
|
||||
representation of all parameters.
|
||||
"""
|
||||
if "_benchmark_name" in self:
|
||||
return str(self["_benchmark_name"])
|
||||
|
||||
return self.as_text(sep="-")
|
||||
|
||||
# In JSON, we prefer "_"
|
||||
def _iter_param_key_candidates(self, param_key: str):
|
||||
# Inner config arguments are not converted by the CLI
|
||||
if "." in param_key:
|
||||
prefix, rest = param_key.split(".", 1)
|
||||
for prefix_candidate in self._iter_param_key_candidates(prefix):
|
||||
yield prefix_candidate + "." + rest
|
||||
|
||||
return
|
||||
|
||||
yield param_key
|
||||
yield param_key.replace("-", "_")
|
||||
yield param_key.replace("_", "-")
|
||||
|
||||
# In CLI, we prefer "-"
|
||||
def _iter_cmd_key_candidates(self, param_key: str):
|
||||
for k in reversed(tuple(self._iter_param_key_candidates(param_key))):
|
||||
yield "--" + k
|
||||
|
||||
def _normalize_cmd_key(self, param_key: str):
|
||||
return next(self._iter_cmd_key_candidates(param_key))
|
||||
|
||||
def has_param(self, param_key: str) -> bool:
|
||||
return any(k in self for k in self._iter_param_key_candidates(param_key))
|
||||
|
||||
def _normalize_cmd_kv_pair(self, k: str, v: object) -> list[str]:
|
||||
"""Normalize a key-value pair into command-line arguments.
|
||||
|
||||
Returns a list containing either:
|
||||
- A single element for boolean flags (e.g., ['--flag'] or ['--flag=true'])
|
||||
- Two elements for key-value pairs (e.g., ['--key', 'value'])
|
||||
"""
|
||||
if isinstance(v, bool):
|
||||
# For nested params (containing "."), use =true/false syntax
|
||||
if "." in k:
|
||||
return [f"{self._normalize_cmd_key(k)}={'true' if v else 'false'}"]
|
||||
else:
|
||||
return [self._normalize_cmd_key(k if v else "no-" + k)]
|
||||
else:
|
||||
return [self._normalize_cmd_key(k), str(v)]
|
||||
|
||||
def apply_to_cmd(self, cmd: list[str]) -> list[str]:
|
||||
cmd = list(cmd)
|
||||
|
||||
for k, v in self.items():
|
||||
# Skip the '_benchmark_name' field, not a parameter
|
||||
if k == "_benchmark_name":
|
||||
continue
|
||||
|
||||
# Serialize dict values as JSON
|
||||
if isinstance(v, dict):
|
||||
v = json.dumps(v)
|
||||
|
||||
for k_candidate in self._iter_cmd_key_candidates(k):
|
||||
try:
|
||||
k_idx = cmd.index(k_candidate)
|
||||
|
||||
# Replace existing parameter
|
||||
normalized = self._normalize_cmd_kv_pair(k, v)
|
||||
if len(normalized) == 1:
|
||||
# Boolean flag
|
||||
cmd[k_idx] = normalized[0]
|
||||
else:
|
||||
# Key-value pair
|
||||
cmd[k_idx] = normalized[0]
|
||||
cmd[k_idx + 1] = normalized[1]
|
||||
|
||||
break
|
||||
except ValueError:
|
||||
continue
|
||||
else:
|
||||
# Add new parameter
|
||||
cmd.extend(self._normalize_cmd_kv_pair(k, v))
|
||||
|
||||
return cmd
|
||||
|
||||
def as_text(self, sep: str = ", ") -> str:
|
||||
return sep.join(f"{k}={v}" for k, v in self.items() if k != "_benchmark_name")
|
||||
@@ -1,690 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
import json
|
||||
from abc import ABC, abstractmethod
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
from types import TracebackType
|
||||
from typing import TYPE_CHECKING, ClassVar, Self
|
||||
|
||||
from typing_extensions import override
|
||||
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
from vllm.utils.collection_utils import full_groupby
|
||||
from vllm.utils.import_utils import PlaceholderModule
|
||||
|
||||
from .utils import sanitize_filename
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import pandas as pd
|
||||
|
||||
|
||||
@dataclass
|
||||
class PlotFilterBase(ABC):
|
||||
var: str
|
||||
target: str
|
||||
|
||||
@classmethod
|
||||
def parse_str(cls, s: str):
|
||||
for op_key in PLOT_FILTERS:
|
||||
if op_key in s:
|
||||
key, value = s.split(op_key)
|
||||
return PLOT_FILTERS[op_key](
|
||||
key,
|
||||
value.removeprefix(op_key).strip("'").strip('"'),
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Invalid operator for plot filter '{s}'. "
|
||||
f"Valid operators are: {sorted(PLOT_FILTERS)}",
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
|
||||
"""Applies this filter to a DataFrame."""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
@dataclass
|
||||
class PlotEqualTo(PlotFilterBase):
|
||||
@override
|
||||
def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
|
||||
target: float | str
|
||||
try:
|
||||
target = float(self.target)
|
||||
except ValueError:
|
||||
target = self.target
|
||||
|
||||
return df[df[self.var] == target]
|
||||
|
||||
|
||||
@dataclass
|
||||
class PlotNotEqualTo(PlotFilterBase):
|
||||
@override
|
||||
def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
|
||||
target: float | str
|
||||
try:
|
||||
target = float(self.target)
|
||||
except ValueError:
|
||||
target = self.target
|
||||
|
||||
return df[df[self.var] != target]
|
||||
|
||||
|
||||
@dataclass
|
||||
class PlotLessThan(PlotFilterBase):
|
||||
@override
|
||||
def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
|
||||
return df[df[self.var] < float(self.target)]
|
||||
|
||||
|
||||
@dataclass
|
||||
class PlotLessThanOrEqualTo(PlotFilterBase):
|
||||
@override
|
||||
def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
|
||||
return df[df[self.var] <= float(self.target)]
|
||||
|
||||
|
||||
@dataclass
|
||||
class PlotGreaterThan(PlotFilterBase):
|
||||
@override
|
||||
def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
|
||||
return df[df[self.var] > float(self.target)]
|
||||
|
||||
|
||||
@dataclass
|
||||
class PlotGreaterThanOrEqualTo(PlotFilterBase):
|
||||
@override
|
||||
def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
|
||||
return df[df[self.var] >= float(self.target)]
|
||||
|
||||
|
||||
# NOTE: The ordering is important! Match longer op_keys first
|
||||
PLOT_FILTERS: dict[str, type[PlotFilterBase]] = {
|
||||
"==": PlotEqualTo,
|
||||
"!=": PlotNotEqualTo,
|
||||
"<=": PlotLessThanOrEqualTo,
|
||||
">=": PlotGreaterThanOrEqualTo,
|
||||
"<": PlotLessThan,
|
||||
">": PlotGreaterThan,
|
||||
}
|
||||
|
||||
|
||||
class PlotFilters(list[PlotFilterBase]):
|
||||
@classmethod
|
||||
def parse_str(cls, s: str):
|
||||
if not s:
|
||||
return cls()
|
||||
|
||||
return cls(PlotFilterBase.parse_str(e) for e in s.split(","))
|
||||
|
||||
def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
|
||||
for item in self:
|
||||
df = item.apply(df)
|
||||
|
||||
return df
|
||||
|
||||
|
||||
@dataclass
|
||||
class PlotBinner:
|
||||
var: str
|
||||
bin_size: float
|
||||
|
||||
@classmethod
|
||||
def parse_str(cls, s: str):
|
||||
for op_key in PLOT_BINNERS:
|
||||
if op_key in s:
|
||||
key, value = s.split(op_key)
|
||||
return PLOT_BINNERS[op_key](key, float(value.removeprefix(op_key)))
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Invalid operator for plot binner '{s}'. "
|
||||
f"Valid operators are: {sorted(PLOT_BINNERS)}",
|
||||
)
|
||||
|
||||
def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
|
||||
"""Applies this binner to a DataFrame."""
|
||||
df = df.copy()
|
||||
df[self.var] = df[self.var] // self.bin_size * self.bin_size
|
||||
return df
|
||||
|
||||
|
||||
PLOT_BINNERS: dict[str, type[PlotBinner]] = {
|
||||
"%": PlotBinner,
|
||||
}
|
||||
|
||||
|
||||
class PlotBinners(list[PlotBinner]):
|
||||
@classmethod
|
||||
def parse_str(cls, s: str):
|
||||
if not s:
|
||||
return cls()
|
||||
|
||||
return cls(PlotBinner.parse_str(e) for e in s.split(","))
|
||||
|
||||
def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
|
||||
for item in self:
|
||||
df = item.apply(df)
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def _json_load_bytes(path: Path) -> list[dict[str, object]]:
|
||||
with path.open("rb") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def _convert_inf_nan_strings(data: list[dict[str, object]]) -> list[dict[str, object]]:
|
||||
"""Convert string values "inf", "-inf", and "nan" to their float equivalents.
|
||||
|
||||
This handles the case where JSON serialization represents inf/nan as strings.
|
||||
"""
|
||||
converted_data = []
|
||||
for record in data:
|
||||
converted_record: dict[str, object] = {}
|
||||
for key, value in record.items():
|
||||
if isinstance(value, str):
|
||||
if value in ["inf", "-inf", "nan"]:
|
||||
converted_record[key] = float(value)
|
||||
else:
|
||||
converted_record[key] = value
|
||||
else:
|
||||
converted_record[key] = value
|
||||
converted_data.append(converted_record)
|
||||
return converted_data
|
||||
|
||||
|
||||
def _get_metric(run_data: dict[str, object], metric_key: str):
|
||||
try:
|
||||
return run_data[metric_key]
|
||||
except KeyError as exc:
|
||||
raise ValueError(f"Cannot find metric {metric_key!r} in {run_data=}") from exc
|
||||
|
||||
|
||||
def _get_group(run_data: dict[str, object], group_keys: list[str]):
|
||||
return tuple((k, str(_get_metric(run_data, k))) for k in group_keys)
|
||||
|
||||
|
||||
def _get_fig_path(fig_dir: Path, group: tuple[tuple[str, str], ...], fig_name: str):
|
||||
parts = list[str]()
|
||||
|
||||
# Start with figure name (always provided, defaults to "FIGURE")
|
||||
parts.append(fig_name)
|
||||
|
||||
# Always append group data if present
|
||||
if group:
|
||||
parts.extend(f"{k}={v}" for k, v in group)
|
||||
|
||||
return fig_dir / sanitize_filename("-".join(parts) + ".png")
|
||||
|
||||
|
||||
class DummyExecutor:
|
||||
map = map
|
||||
|
||||
def __enter__(self) -> Self:
|
||||
return self
|
||||
|
||||
def __exit__(
|
||||
self,
|
||||
exc_type: type[BaseException] | None,
|
||||
exc_value: BaseException | None,
|
||||
exc_traceback: TracebackType | None,
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _plot_fig(
|
||||
fig_dir: Path,
|
||||
fig_group_data: tuple[tuple[tuple[str, str], ...], list[dict[str, object]]],
|
||||
row_by: list[str],
|
||||
col_by: list[str],
|
||||
curve_by: list[str],
|
||||
*,
|
||||
var_x: str,
|
||||
var_y: str,
|
||||
filter_by: PlotFilters,
|
||||
bin_by: PlotBinners,
|
||||
scale_x: str | None,
|
||||
scale_y: str | None,
|
||||
dry_run: bool,
|
||||
fig_name: str,
|
||||
error_bars: bool,
|
||||
fig_height: float,
|
||||
fig_dpi: int,
|
||||
):
|
||||
# Lazy-import matplotlib/pandas/seaborn
|
||||
try:
|
||||
import matplotlib.pyplot as plt
|
||||
except ImportError:
|
||||
plt = PlaceholderModule("matplotlib").placeholder_attr("pyplot")
|
||||
try:
|
||||
import pandas as pd
|
||||
except ImportError:
|
||||
pd = PlaceholderModule("pandas")
|
||||
try:
|
||||
import seaborn as sns
|
||||
except ImportError:
|
||||
sns = PlaceholderModule("seaborn")
|
||||
|
||||
fig_group, fig_data = fig_group_data
|
||||
|
||||
row_groups = full_groupby(
|
||||
fig_data,
|
||||
key=lambda item: _get_group(item, row_by),
|
||||
)
|
||||
num_rows = len(row_groups)
|
||||
num_cols = max(
|
||||
len(full_groupby(row_data, key=lambda item: _get_group(item, col_by)))
|
||||
for _, row_data in row_groups
|
||||
)
|
||||
|
||||
fig_path = _get_fig_path(fig_dir, fig_group, fig_name)
|
||||
|
||||
print("[BEGIN FIGURE]")
|
||||
print(f"Group: {dict(fig_group)}")
|
||||
print(f"Grid: {num_rows} rows x {num_cols} cols")
|
||||
print(f"Output file: {fig_path}")
|
||||
|
||||
if dry_run:
|
||||
print("[END FIGURE]")
|
||||
return
|
||||
|
||||
# Convert string "inf", "-inf", and "nan" to their float equivalents
|
||||
fig_data = _convert_inf_nan_strings(fig_data)
|
||||
df = pd.DataFrame.from_records(fig_data)
|
||||
|
||||
if var_x not in df.columns:
|
||||
raise ValueError(
|
||||
f"Cannot find {var_x=!r} in parameter sweep results. "
|
||||
f"Available variables: {df.columns.tolist()}"
|
||||
)
|
||||
if var_y not in df.columns:
|
||||
raise ValueError(
|
||||
f"Cannot find {var_y=!r} in parameter sweep results. "
|
||||
f"Available variables: {df.columns.tolist()}"
|
||||
)
|
||||
for k in row_by:
|
||||
if k not in df.columns:
|
||||
raise ValueError(
|
||||
f"Cannot find row_by={k!r} in parameter sweep results. "
|
||||
f"Available variables: {df.columns.tolist()}"
|
||||
)
|
||||
for k in col_by:
|
||||
if k not in df.columns:
|
||||
raise ValueError(
|
||||
f"Cannot find col_by={k!r} in parameter sweep results. "
|
||||
f"Available variables: {df.columns.tolist()}"
|
||||
)
|
||||
for k in curve_by:
|
||||
if k not in df.columns:
|
||||
raise ValueError(
|
||||
f"Cannot find curve_by={k!r} in parameter sweep results. "
|
||||
f"Available variables: {df.columns.tolist()}"
|
||||
)
|
||||
|
||||
df = filter_by.apply(df)
|
||||
df = bin_by.apply(df)
|
||||
|
||||
if len(df) == 0:
|
||||
print(f"No data to plot. Filters: {filter_by}")
|
||||
print("[END FIGURE]")
|
||||
return
|
||||
|
||||
# Sort by curve_by columns alphabetically for consistent legend ordering
|
||||
if curve_by:
|
||||
df = df.sort_values(by=curve_by)
|
||||
|
||||
df["row_group"] = (
|
||||
pd.concat(
|
||||
[k + "=" + df[k].astype(str) for k in row_by],
|
||||
axis=1,
|
||||
).agg("\n".join, axis=1)
|
||||
if row_by
|
||||
else "(All)"
|
||||
)
|
||||
|
||||
df["col_group"] = (
|
||||
pd.concat(
|
||||
[k + "=" + df[k].astype(str) for k in col_by],
|
||||
axis=1,
|
||||
).agg("\n".join, axis=1)
|
||||
if col_by
|
||||
else "(All)"
|
||||
)
|
||||
|
||||
if len(curve_by) <= 3:
|
||||
hue, style, size, *_ = (*curve_by, None, None, None)
|
||||
|
||||
g = sns.relplot(
|
||||
df,
|
||||
x=var_x,
|
||||
y=var_y,
|
||||
hue=hue,
|
||||
style=style,
|
||||
size=size,
|
||||
markers=True,
|
||||
errorbar="sd" if error_bars else None,
|
||||
kind="line",
|
||||
row="row_group",
|
||||
col="col_group",
|
||||
height=fig_height,
|
||||
)
|
||||
else:
|
||||
df["curve_group"] = (
|
||||
pd.concat(
|
||||
[k + "=" + df[k].astype(str) for k in curve_by],
|
||||
axis=1,
|
||||
).agg("\n".join, axis=1)
|
||||
if curve_by
|
||||
else "(All)"
|
||||
)
|
||||
|
||||
g = sns.relplot(
|
||||
df,
|
||||
x=var_x,
|
||||
y=var_y,
|
||||
hue="curve_group",
|
||||
markers=True,
|
||||
errorbar="sd" if error_bars else None,
|
||||
kind="line",
|
||||
row="row_group",
|
||||
col="col_group",
|
||||
height=fig_height,
|
||||
)
|
||||
|
||||
if row_by and col_by:
|
||||
g.set_titles("{row_name}\n{col_name}")
|
||||
elif row_by:
|
||||
g.set_titles("{row_name}")
|
||||
elif col_by:
|
||||
g.set_titles("{col_name}")
|
||||
else:
|
||||
g.set_titles("")
|
||||
|
||||
if scale_x:
|
||||
g.set(xscale=scale_x)
|
||||
if scale_y:
|
||||
g.set(yscale=scale_y)
|
||||
|
||||
g.savefig(fig_path, dpi=fig_dpi)
|
||||
plt.close(g.figure)
|
||||
|
||||
print("[END FIGURE]")
|
||||
|
||||
|
||||
def plot(
|
||||
output_dir: Path,
|
||||
fig_dir: Path,
|
||||
fig_by: list[str],
|
||||
row_by: list[str],
|
||||
col_by: list[str],
|
||||
curve_by: list[str],
|
||||
*,
|
||||
var_x: str,
|
||||
var_y: str,
|
||||
filter_by: PlotFilters,
|
||||
bin_by: PlotBinners,
|
||||
scale_x: str | None,
|
||||
scale_y: str | None,
|
||||
dry_run: bool,
|
||||
fig_name: str = "FIGURE",
|
||||
error_bars: bool = True,
|
||||
fig_height: float = 6.4,
|
||||
fig_dpi: int = 300,
|
||||
):
|
||||
all_data = [
|
||||
run_data
|
||||
for path in output_dir.rglob("**/summary.json")
|
||||
for run_data in _json_load_bytes(path)
|
||||
]
|
||||
|
||||
if not all_data:
|
||||
raise ValueError(f"Did not find any parameter sweep results under {output_dir}")
|
||||
|
||||
fig_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
fig_groups = full_groupby(
|
||||
all_data,
|
||||
key=lambda item: _get_group(item, fig_by),
|
||||
)
|
||||
|
||||
with DummyExecutor() if len(fig_groups) <= 1 else ProcessPoolExecutor() as executor:
|
||||
# Resolve the iterable to ensure that the workers are run
|
||||
all(
|
||||
executor.map(
|
||||
partial(
|
||||
_plot_fig,
|
||||
fig_dir,
|
||||
row_by=row_by,
|
||||
col_by=col_by,
|
||||
curve_by=curve_by,
|
||||
var_x=var_x,
|
||||
var_y=var_y,
|
||||
filter_by=filter_by,
|
||||
bin_by=bin_by,
|
||||
scale_x=scale_x,
|
||||
scale_y=scale_y,
|
||||
dry_run=dry_run,
|
||||
fig_name=fig_name,
|
||||
error_bars=error_bars,
|
||||
fig_height=fig_height,
|
||||
fig_dpi=fig_dpi,
|
||||
),
|
||||
fig_groups,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SweepPlotArgs:
|
||||
output_dir: Path
|
||||
fig_dir: Path
|
||||
fig_by: list[str]
|
||||
row_by: list[str]
|
||||
col_by: list[str]
|
||||
curve_by: list[str]
|
||||
var_x: str
|
||||
var_y: str
|
||||
filter_by: PlotFilters
|
||||
bin_by: PlotBinners
|
||||
scale_x: str | None
|
||||
scale_y: str | None
|
||||
dry_run: bool
|
||||
fig_name: str = "FIGURE"
|
||||
error_bars: bool = True
|
||||
fig_height: float = 6.4
|
||||
fig_dpi: int = 300
|
||||
|
||||
parser_name: ClassVar[str] = "plot"
|
||||
parser_help: ClassVar[str] = "Plot performance curves from parameter sweep results."
|
||||
|
||||
@classmethod
|
||||
def from_cli_args(cls, args: argparse.Namespace):
|
||||
output_dir = Path(args.EXPERIMENT_DIR)
|
||||
if not output_dir.exists():
|
||||
raise ValueError(f"No parameter sweep results under {output_dir}")
|
||||
|
||||
curve_by = [] if not args.curve_by else args.curve_by.split(",")
|
||||
row_by = [] if not args.row_by else args.row_by.split(",")
|
||||
col_by = [] if not args.col_by else args.col_by.split(",")
|
||||
fig_by = [] if not args.fig_by else args.fig_by.split(",")
|
||||
|
||||
return cls(
|
||||
output_dir=output_dir,
|
||||
fig_dir=output_dir / args.fig_dir,
|
||||
fig_by=fig_by,
|
||||
row_by=row_by,
|
||||
col_by=col_by,
|
||||
curve_by=curve_by,
|
||||
var_x=args.var_x,
|
||||
var_y=args.var_y,
|
||||
filter_by=PlotFilters.parse_str(args.filter_by),
|
||||
bin_by=PlotBinners.parse_str(args.bin_by),
|
||||
scale_x=args.scale_x,
|
||||
scale_y=args.scale_y,
|
||||
dry_run=args.dry_run,
|
||||
fig_name=args.fig_name,
|
||||
error_bars=not args.no_error_bars,
|
||||
fig_height=args.fig_height,
|
||||
fig_dpi=args.fig_dpi,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def add_cli_args(cls, parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
|
||||
parser.add_argument(
|
||||
"EXPERIMENT_DIR",
|
||||
type=str,
|
||||
help="The directory containing the sweep results to plot.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--fig-dir",
|
||||
type=str,
|
||||
default="",
|
||||
help="The directory to save the figures, relative to `OUTPUT_DIR`. "
|
||||
"By default, the same directory is used.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--fig-by",
|
||||
type=str,
|
||||
default="",
|
||||
help="A comma-separated list of variables, such that a separate figure "
|
||||
"is created for each combination of these variables.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--row-by",
|
||||
type=str,
|
||||
default="",
|
||||
help="A comma-separated list of variables, such that a separate row "
|
||||
"is created for each combination of these variables.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--col-by",
|
||||
type=str,
|
||||
default="",
|
||||
help="A comma-separated list of variables, such that a separate column "
|
||||
"is created for each combination of these variables.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--curve-by",
|
||||
type=str,
|
||||
default=None,
|
||||
help="A comma-separated list of variables, such that a separate curve "
|
||||
"is created for each combination of these variables.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--var-x",
|
||||
type=str,
|
||||
default="total_token_throughput",
|
||||
help="The variable for the x-axis.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--var-y",
|
||||
type=str,
|
||||
default="median_ttft_ms",
|
||||
help="The variable for the y-axis",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--filter-by",
|
||||
type=str,
|
||||
default="",
|
||||
help="A comma-separated list of statements indicating values to filter by. "
|
||||
"This is useful to remove outliers. "
|
||||
"Example: `max_concurrency<1000,max_num_batched_tokens<=4096` means "
|
||||
"plot only the points where `max_concurrency` is less than 1000 and "
|
||||
"`max_num_batched_tokens` is no greater than 4096.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--bin-by",
|
||||
type=str,
|
||||
default="",
|
||||
help="A comma-separated list of statements indicating values to bin by. "
|
||||
"This is useful to avoid plotting points that are too close together. "
|
||||
"Example: `request_throughput%%1` means "
|
||||
"use a bin size of 1 for the `request_throughput` variable.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--scale-x",
|
||||
type=str,
|
||||
default=None,
|
||||
help="The scale to use for the x-axis. "
|
||||
"Currently only accepts string values such as 'log' and 'sqrt'. "
|
||||
"See also: https://seaborn.pydata.org/generated/seaborn.objects.Plot.scale.html",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--scale-y",
|
||||
type=str,
|
||||
default=None,
|
||||
help="The scale to use for the y-axis. "
|
||||
"Currently only accepts string values such as 'log' and 'sqrt'. "
|
||||
"See also: https://seaborn.pydata.org/generated/seaborn.objects.Plot.scale.html",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--fig-name",
|
||||
type=str,
|
||||
default="FIGURE",
|
||||
help="Name prefix for the output figure file. "
|
||||
"Group data is always appended when present. "
|
||||
"Default: 'FIGURE'. Example: --fig-name my_performance_plot",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-error-bars",
|
||||
action="store_true",
|
||||
help="If set, disables error bars on the plot. "
|
||||
"By default, error bars are shown.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--fig-height",
|
||||
type=float,
|
||||
default=6.4,
|
||||
help="Height of each subplot in inches. Default: 6.4",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--fig-dpi",
|
||||
type=int,
|
||||
default=300,
|
||||
help="Resolution of the output figure in dots per inch. Default: 300",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="If set, prints the information about each figure to plot, "
|
||||
"then exits without drawing them.",
|
||||
)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def run_main(args: SweepPlotArgs):
|
||||
return plot(
|
||||
output_dir=args.output_dir,
|
||||
fig_dir=args.fig_dir,
|
||||
fig_by=args.fig_by,
|
||||
row_by=args.row_by,
|
||||
col_by=args.col_by,
|
||||
curve_by=args.curve_by,
|
||||
var_x=args.var_x,
|
||||
var_y=args.var_y,
|
||||
filter_by=args.filter_by,
|
||||
bin_by=args.bin_by,
|
||||
scale_x=args.scale_x,
|
||||
scale_y=args.scale_y,
|
||||
dry_run=args.dry_run,
|
||||
fig_name=args.fig_name,
|
||||
error_bars=args.error_bars,
|
||||
fig_height=args.fig_height,
|
||||
fig_dpi=args.fig_dpi,
|
||||
)
|
||||
|
||||
|
||||
def main(args: argparse.Namespace):
|
||||
run_main(SweepPlotArgs.from_cli_args(args))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = FlexibleArgumentParser(description=SweepPlotArgs.parser_help)
|
||||
SweepPlotArgs.add_cli_args(parser)
|
||||
|
||||
main(parser.parse_args())
|
||||
@@ -1,401 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
import math
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
from vllm.utils.collection_utils import full_groupby
|
||||
from vllm.utils.import_utils import PlaceholderModule
|
||||
|
||||
from .plot import DummyExecutor, _json_load_bytes
|
||||
from .utils import sanitize_filename
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def _first_present(run_data: dict[str, object], keys: list[str]):
|
||||
for key in keys:
|
||||
for candidate in {key, key.replace("_", "-"), key.replace("-", "_")}:
|
||||
if candidate in run_data:
|
||||
return run_data[candidate]
|
||||
return None
|
||||
|
||||
|
||||
def _get_numeric(
|
||||
run_data: dict[str, object],
|
||||
keys: list[str],
|
||||
*,
|
||||
allow_zero: bool = True,
|
||||
) -> float | None:
|
||||
value = _first_present(run_data, keys)
|
||||
if value is None:
|
||||
return None
|
||||
|
||||
try:
|
||||
numeric = float(value)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError(
|
||||
f"Expected numeric value for one of {keys}, "
|
||||
f"but found {value!r} in {run_data=}"
|
||||
) from exc
|
||||
|
||||
if not allow_zero and numeric == 0:
|
||||
return None
|
||||
|
||||
return numeric
|
||||
|
||||
|
||||
def _infer_user_count(
|
||||
run_data: dict[str, object],
|
||||
user_count_var: str | None,
|
||||
) -> float | None:
|
||||
candidates = [user_count_var] if user_count_var else []
|
||||
candidates.extend(["request_rate"])
|
||||
user_count = _get_numeric(run_data, candidates, allow_zero=False)
|
||||
if user_count is not None:
|
||||
return user_count
|
||||
|
||||
# Fallback to the observed peak if configured value is missing.
|
||||
return _get_numeric(run_data, ["max_concurrent_requests"], allow_zero=False)
|
||||
|
||||
|
||||
def _infer_gpu_count(
|
||||
run_data: dict[str, object],
|
||||
gpu_count_var: str | None,
|
||||
) -> float:
|
||||
direct_candidates = [gpu_count_var] if gpu_count_var else []
|
||||
direct_gpu_count = _get_numeric(run_data, direct_candidates, allow_zero=False)
|
||||
if direct_gpu_count:
|
||||
return direct_gpu_count
|
||||
|
||||
tp_size = _get_numeric(run_data, ["tensor_parallel_size", "tp"])
|
||||
pp_size = _get_numeric(run_data, ["pipeline_parallel_size", "pp"])
|
||||
dp_size = _get_numeric(run_data, ["data_parallel_size", "dp"])
|
||||
world_size = 1.0
|
||||
if tp_size:
|
||||
world_size *= tp_size
|
||||
if pp_size:
|
||||
world_size *= pp_size
|
||||
if dp_size:
|
||||
world_size *= dp_size
|
||||
|
||||
return world_size
|
||||
|
||||
|
||||
def _get_throughput(
|
||||
run_data: dict[str, object],
|
||||
throughput_var: str,
|
||||
) -> float:
|
||||
throughput = _get_numeric(run_data, [throughput_var])
|
||||
if throughput is None:
|
||||
raise ValueError(
|
||||
f"Cannot find throughput metric {throughput_var!r} in run data. "
|
||||
f"Available keys: {sorted(run_data)}"
|
||||
)
|
||||
|
||||
return throughput
|
||||
|
||||
|
||||
def _prepare_records(
|
||||
all_data: list[dict[str, object]],
|
||||
*,
|
||||
user_count_var: str | None,
|
||||
gpu_count_var: str | None,
|
||||
) -> tuple[list[dict[str, object]], int]:
|
||||
prepared = []
|
||||
skipped_missing_users = 0
|
||||
|
||||
for record in all_data:
|
||||
throughput = _get_throughput(record, "output_throughput")
|
||||
user_count = _infer_user_count(record, user_count_var)
|
||||
if user_count is None:
|
||||
skipped_missing_users += 1
|
||||
continue
|
||||
|
||||
gpu_count = _infer_gpu_count(record, gpu_count_var)
|
||||
tokens_per_user = throughput / user_count
|
||||
tokens_per_gpu = throughput / gpu_count
|
||||
|
||||
prepared.append(
|
||||
{
|
||||
**record,
|
||||
"tokens_per_user": tokens_per_user,
|
||||
"tokens_per_gpu": tokens_per_gpu,
|
||||
"user_count_estimate": user_count,
|
||||
"gpu_count": gpu_count,
|
||||
}
|
||||
)
|
||||
|
||||
return prepared, skipped_missing_users
|
||||
|
||||
|
||||
def _pareto_frontier(
|
||||
df: "pd.DataFrame",
|
||||
x_col: str,
|
||||
y_col: str,
|
||||
*,
|
||||
epsilon: float = 1e-9,
|
||||
) -> "pd.DataFrame":
|
||||
sorted_df = df.sort_values([x_col, y_col], ascending=[False, False])
|
||||
frontier_indices = []
|
||||
best_y = -math.inf
|
||||
|
||||
for idx, row in sorted_df.iterrows():
|
||||
y_val = row[y_col]
|
||||
if y_val >= best_y - epsilon:
|
||||
frontier_indices.append(idx)
|
||||
best_y = max(best_y, y_val)
|
||||
|
||||
return df.loc[frontier_indices]
|
||||
|
||||
|
||||
def _get_fig_path(
|
||||
fig_dir: Path,
|
||||
fig_group: tuple[tuple[str, str], ...],
|
||||
) -> Path:
|
||||
parts = ["PARETO"]
|
||||
if fig_group:
|
||||
parts.extend(f"{k}={v}" for k, v in fig_group)
|
||||
filename = sanitize_filename("-".join(parts) + ".png")
|
||||
return fig_dir / filename
|
||||
|
||||
|
||||
def _plot_fig(
|
||||
fig_dir: Path,
|
||||
fig_group_data: tuple[tuple[tuple[str, str], ...], list[dict[str, object]]],
|
||||
label_by: list[str],
|
||||
*,
|
||||
dry_run: bool,
|
||||
):
|
||||
fig_group, fig_data = fig_group_data
|
||||
fig_path = _get_fig_path(fig_dir, fig_group)
|
||||
|
||||
print("[BEGIN FIGURE]")
|
||||
print(f"Group: {dict(fig_group)}")
|
||||
print(f"Output file: {fig_path}")
|
||||
|
||||
if dry_run:
|
||||
print("[END FIGURE]")
|
||||
return
|
||||
|
||||
# Lazy-import matplotlib/pandas/seaborn
|
||||
try:
|
||||
import matplotlib.pyplot as plt
|
||||
except ImportError:
|
||||
plt = PlaceholderModule("matplotlib").placeholder_attr("pyplot")
|
||||
try:
|
||||
import pandas as pd
|
||||
except ImportError:
|
||||
pd = PlaceholderModule("pandas")
|
||||
try:
|
||||
import seaborn as sns
|
||||
except ImportError:
|
||||
sns = PlaceholderModule("seaborn")
|
||||
|
||||
df = pd.DataFrame.from_records(fig_data)
|
||||
df = df.dropna(subset=["tokens_per_user", "tokens_per_gpu"])
|
||||
|
||||
if df.empty:
|
||||
print("No data points available after filtering; skipping.")
|
||||
print("[END FIGURE]")
|
||||
return
|
||||
|
||||
frontier = _pareto_frontier(df, "tokens_per_user", "tokens_per_gpu")
|
||||
frontier = frontier.sort_values("tokens_per_user")
|
||||
|
||||
fig, ax = plt.subplots()
|
||||
sns.scatterplot(
|
||||
data=df,
|
||||
x="tokens_per_user",
|
||||
y="tokens_per_gpu",
|
||||
color="0.5",
|
||||
alpha=0.6,
|
||||
ax=ax,
|
||||
label="All runs",
|
||||
)
|
||||
sns.lineplot(
|
||||
data=frontier,
|
||||
x="tokens_per_user",
|
||||
y="tokens_per_gpu",
|
||||
marker="o",
|
||||
ax=ax,
|
||||
label="Pareto frontier",
|
||||
)
|
||||
|
||||
if label_by:
|
||||
for _, row in frontier.iterrows():
|
||||
label_parts = []
|
||||
for key in label_by:
|
||||
if key in row:
|
||||
label_parts.append(f"{key}={row[key]}")
|
||||
if label_parts:
|
||||
ax.text(
|
||||
row["tokens_per_user"],
|
||||
row["tokens_per_gpu"],
|
||||
"\n".join(label_parts),
|
||||
fontsize=8,
|
||||
)
|
||||
|
||||
ax.set_xlabel("Tokens/s/user")
|
||||
ax.set_ylabel("Tokens/s/GPU")
|
||||
ax.grid(True, linestyle="--", linewidth=0.5, alpha=0.6)
|
||||
fig.tight_layout()
|
||||
fig.savefig(fig_path)
|
||||
plt.close(fig)
|
||||
|
||||
print(
|
||||
f"Plotted {len(df)} points; Pareto frontier size: {len(frontier)}.",
|
||||
)
|
||||
print("[END FIGURE]")
|
||||
|
||||
|
||||
def plot_pareto(
|
||||
output_dir: Path,
|
||||
user_count_var: str | None,
|
||||
gpu_count_var: str | None,
|
||||
label_by: list[str],
|
||||
*,
|
||||
dry_run: bool,
|
||||
):
|
||||
fig_dir = output_dir / "pareto"
|
||||
raw_data = [
|
||||
run_data
|
||||
for path in output_dir.rglob("**/summary.json")
|
||||
for run_data in _json_load_bytes(path)
|
||||
]
|
||||
|
||||
if not raw_data:
|
||||
raise ValueError(f"Did not find any parameter sweep results under {output_dir}")
|
||||
|
||||
fig_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
prepared_data, skipped_missing_users = _prepare_records(
|
||||
raw_data,
|
||||
user_count_var=user_count_var,
|
||||
gpu_count_var=gpu_count_var,
|
||||
)
|
||||
|
||||
if skipped_missing_users:
|
||||
print(
|
||||
f"Skipped {skipped_missing_users} runs without a user count "
|
||||
"(`max_concurrency` or `max_concurrent_requests`).",
|
||||
)
|
||||
|
||||
if not prepared_data:
|
||||
raise ValueError(
|
||||
"No data points with both throughput and user count available "
|
||||
"to plot Pareto frontier.",
|
||||
)
|
||||
|
||||
fig_groups = full_groupby(
|
||||
prepared_data,
|
||||
key=lambda item: tuple(),
|
||||
)
|
||||
|
||||
with DummyExecutor() if len(fig_groups) <= 1 else ProcessPoolExecutor() as executor:
|
||||
all(
|
||||
executor.map(
|
||||
partial(
|
||||
_plot_fig,
|
||||
fig_dir,
|
||||
label_by=label_by,
|
||||
dry_run=dry_run,
|
||||
),
|
||||
fig_groups,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SweepPlotParetoArgs:
|
||||
output_dir: Path
|
||||
user_count_var: str | None
|
||||
gpu_count_var: str | None
|
||||
label_by: list[str]
|
||||
dry_run: bool
|
||||
|
||||
parser_name: ClassVar[str] = "plot_pareto"
|
||||
parser_help: ClassVar[str] = (
|
||||
"Plot Pareto frontier between tokens/s/user and tokens/s/GPU "
|
||||
"from parameter sweep results."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_cli_args(cls, args: argparse.Namespace):
|
||||
output_dir = Path(args.EXPERIMENT_DIR)
|
||||
if not output_dir.exists():
|
||||
raise ValueError(f"No parameter sweep results under {output_dir}")
|
||||
|
||||
label_by = [] if not args.label_by else args.label_by.split(",")
|
||||
|
||||
return cls(
|
||||
output_dir=output_dir,
|
||||
user_count_var=args.user_count_var,
|
||||
gpu_count_var=args.gpu_count_var,
|
||||
label_by=label_by,
|
||||
dry_run=args.dry_run,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def add_cli_args(cls, parser: FlexibleArgumentParser):
|
||||
parser.add_argument(
|
||||
"EXPERIMENT_DIR",
|
||||
type=str,
|
||||
help="The directory containing the sweep results to plot.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--user-count-var",
|
||||
type=str,
|
||||
default="max_concurrency",
|
||||
help="Result key that stores concurrent user count. "
|
||||
"Falls back to max_concurrent_requests if missing.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--gpu-count-var",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Result key that stores GPU count. "
|
||||
"If not provided, falls back to num_gpus/gpu_count "
|
||||
"or tensor_parallel_size * pipeline_parallel_size.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--label-by",
|
||||
type=str,
|
||||
default="max_concurrency,gpu_count",
|
||||
help="Comma-separated list of fields to annotate on Pareto frontier "
|
||||
"points.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="If set, prints the figures to plot without drawing them.",
|
||||
)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def run_main(args: SweepPlotParetoArgs):
|
||||
return plot_pareto(
|
||||
output_dir=args.output_dir,
|
||||
user_count_var=args.user_count_var,
|
||||
gpu_count_var=args.gpu_count_var,
|
||||
label_by=args.label_by,
|
||||
dry_run=args.dry_run,
|
||||
)
|
||||
|
||||
|
||||
def main(args: argparse.Namespace):
|
||||
run_main(SweepPlotParetoArgs.from_cli_args(args))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = FlexibleArgumentParser(description=SweepPlotParetoArgs.parser_help)
|
||||
SweepPlotParetoArgs.add_cli_args(parser)
|
||||
|
||||
main(parser.parse_args())
|
||||
@@ -1,649 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
import contextlib
|
||||
import json
|
||||
import shlex
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import ClassVar
|
||||
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
from vllm.utils.import_utils import PlaceholderModule
|
||||
|
||||
from .param_sweep import ParameterSweep, ParameterSweepItem
|
||||
from .server import ServerProcess
|
||||
from .utils import sanitize_filename
|
||||
|
||||
try:
|
||||
import pandas as pd
|
||||
except ImportError:
|
||||
pd = PlaceholderModule("pandas")
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def run_server(
|
||||
serve_cmd: list[str],
|
||||
after_bench_cmd: list[str],
|
||||
*,
|
||||
show_stdout: bool,
|
||||
serve_overrides: ParameterSweepItem,
|
||||
dry_run: bool,
|
||||
server_ready_timeout: int = 300,
|
||||
):
|
||||
server_cmd = serve_overrides.apply_to_cmd(serve_cmd)
|
||||
|
||||
print("[BEGIN SERVER]")
|
||||
print(f"Server overrides: {serve_overrides}")
|
||||
print(f"Server command: {server_cmd}")
|
||||
|
||||
if dry_run:
|
||||
yield None
|
||||
print("[END SERVER]")
|
||||
return
|
||||
|
||||
with ServerProcess(server_cmd, after_bench_cmd, show_stdout=show_stdout) as server:
|
||||
server.wait_until_ready(timeout=server_ready_timeout)
|
||||
yield server
|
||||
|
||||
print("[END SERVER]")
|
||||
|
||||
|
||||
def _update_run_data(
|
||||
run_data: dict[str, object],
|
||||
serve_overrides: ParameterSweepItem,
|
||||
bench_overrides: ParameterSweepItem,
|
||||
run_number: int,
|
||||
):
|
||||
run_data["run_number"] = run_number
|
||||
run_data.update(serve_overrides)
|
||||
run_data.update(bench_overrides)
|
||||
|
||||
return run_data
|
||||
|
||||
|
||||
def run_benchmark(
|
||||
server: ServerProcess | None,
|
||||
bench_cmd: list[str],
|
||||
*,
|
||||
serve_overrides: ParameterSweepItem,
|
||||
bench_overrides: ParameterSweepItem,
|
||||
run_number: int,
|
||||
output_path: Path,
|
||||
dry_run: bool,
|
||||
):
|
||||
benchmark_cmd = [
|
||||
*bench_overrides.apply_to_cmd(bench_cmd),
|
||||
"--percentile-metrics",
|
||||
"ttft,tpot,itl,e2el",
|
||||
"--save-result",
|
||||
"--result-dir",
|
||||
str(output_path.parent),
|
||||
"--result-filename",
|
||||
output_path.name,
|
||||
]
|
||||
|
||||
print("[BEGIN BENCHMARK]")
|
||||
print(f"Benchmark overrides: {bench_overrides}")
|
||||
print(f"Run Number: {run_number}")
|
||||
print(f"Benchmark command: {benchmark_cmd}")
|
||||
print(f"Output file: {output_path}")
|
||||
|
||||
run_data: dict[str, object]
|
||||
|
||||
if output_path.exists():
|
||||
print("Found existing results.")
|
||||
print("[SKIPPED BENCHMARK]")
|
||||
|
||||
with output_path.open("rb") as f:
|
||||
run_data = json.load(f)
|
||||
return _update_run_data(
|
||||
run_data,
|
||||
serve_overrides,
|
||||
bench_overrides,
|
||||
run_number,
|
||||
)
|
||||
|
||||
if server is None:
|
||||
if not dry_run:
|
||||
raise ValueError(f"Cannot find results at {output_path}")
|
||||
|
||||
print("[END BENCHMARK]")
|
||||
return None
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
server.run_subcommand(benchmark_cmd)
|
||||
server.after_bench()
|
||||
|
||||
with output_path.open("rb") as f:
|
||||
run_data = json.load(f)
|
||||
|
||||
run_data = _update_run_data(
|
||||
run_data,
|
||||
serve_overrides,
|
||||
bench_overrides,
|
||||
run_number,
|
||||
)
|
||||
|
||||
with output_path.open("w") as f:
|
||||
json.dump(run_data, f, indent=4)
|
||||
|
||||
print("[END BENCHMARK]")
|
||||
|
||||
return run_data
|
||||
|
||||
|
||||
def _get_comb_base_path(
|
||||
experiment_dir: Path,
|
||||
serve_comb: ParameterSweepItem,
|
||||
bench_comb: ParameterSweepItem,
|
||||
*,
|
||||
extra_parts: tuple[str, ...] = (),
|
||||
):
|
||||
parts = list[str]()
|
||||
if serve_comb:
|
||||
parts.extend(("SERVE-", serve_comb.name))
|
||||
if bench_comb:
|
||||
parts.extend(("BENCH-", bench_comb.name))
|
||||
if extra_parts:
|
||||
parts.extend(extra_parts)
|
||||
|
||||
return experiment_dir / sanitize_filename("-".join(parts))
|
||||
|
||||
|
||||
def _get_comb_run_path(base_path: Path, run_number: int | None):
|
||||
if run_number is None:
|
||||
return base_path / "summary.json"
|
||||
|
||||
return base_path / f"run={run_number}.json"
|
||||
|
||||
|
||||
def _comb_needs_server(
|
||||
serve_comb: ParameterSweepItem,
|
||||
bench_combs: ParameterSweep,
|
||||
experiment_dir: Path,
|
||||
):
|
||||
for bench_comb in bench_combs:
|
||||
base_path = _get_comb_base_path(experiment_dir, serve_comb, bench_comb)
|
||||
if not _get_comb_run_path(base_path, run_number=None).exists():
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def server_ctx(
|
||||
serve_cmd: list[str],
|
||||
after_bench_cmd: list[str],
|
||||
*,
|
||||
show_stdout: bool,
|
||||
serve_comb: ParameterSweepItem,
|
||||
bench_params: ParameterSweep,
|
||||
experiment_dir: Path,
|
||||
dry_run: bool,
|
||||
server_ready_timeout: int = 300,
|
||||
):
|
||||
if not _comb_needs_server(serve_comb, bench_params, experiment_dir):
|
||||
return contextlib.nullcontext()
|
||||
|
||||
return run_server(
|
||||
serve_cmd,
|
||||
after_bench_cmd,
|
||||
show_stdout=show_stdout,
|
||||
serve_overrides=serve_comb,
|
||||
dry_run=dry_run,
|
||||
server_ready_timeout=server_ready_timeout,
|
||||
)
|
||||
|
||||
|
||||
def _comb_is_valid(
|
||||
serve_comb: ParameterSweepItem,
|
||||
bench_comb: ParameterSweepItem,
|
||||
link_vars: list[tuple[str, str]],
|
||||
) -> bool:
|
||||
return all(
|
||||
serve_key in serve_comb
|
||||
and bench_key in bench_comb
|
||||
and serve_comb[serve_key] == bench_comb[bench_key]
|
||||
for serve_key, bench_key in link_vars
|
||||
)
|
||||
|
||||
|
||||
def run_comb(
|
||||
server: ServerProcess | None,
|
||||
bench_cmd: list[str],
|
||||
*,
|
||||
serve_comb: ParameterSweepItem,
|
||||
bench_comb: ParameterSweepItem,
|
||||
link_vars: list[tuple[str, str]],
|
||||
base_path: Path,
|
||||
num_runs: int,
|
||||
dry_run: bool,
|
||||
warmup_num_prompts: int = 0,
|
||||
continue_on_error: bool = False,
|
||||
):
|
||||
if not _comb_is_valid(serve_comb, bench_comb, link_vars):
|
||||
return None
|
||||
|
||||
comb_data = list[dict[str, object]]()
|
||||
|
||||
def run_benchmark_with_error_handling(
|
||||
*,
|
||||
bench_overrides: ParameterSweepItem,
|
||||
run_number: int,
|
||||
output_path: Path,
|
||||
):
|
||||
try:
|
||||
return run_benchmark(
|
||||
server,
|
||||
bench_cmd,
|
||||
serve_overrides=serve_comb,
|
||||
bench_overrides=bench_overrides,
|
||||
run_number=run_number,
|
||||
output_path=output_path,
|
||||
dry_run=dry_run,
|
||||
)
|
||||
except Exception as exc:
|
||||
if not continue_on_error:
|
||||
raise
|
||||
failure_path = output_path.with_name(output_path.stem + ".failure.json")
|
||||
failure_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
failure_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"run_number": run_number,
|
||||
"error_type": type(exc).__name__,
|
||||
"error": str(exc),
|
||||
"serve_overrides": dict(serve_comb),
|
||||
"bench_overrides": dict(bench_overrides),
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(f"[FAILED BENCHMARK] run={run_number}: {type(exc).__name__}: {exc}")
|
||||
print(f"Failure details: {failure_path}")
|
||||
if server is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
server.after_bench()
|
||||
return None
|
||||
|
||||
if warmup_num_prompts > 0:
|
||||
warmup_overrides = bench_comb | {"num_prompts": warmup_num_prompts}
|
||||
run_benchmark_with_error_handling(
|
||||
bench_overrides=warmup_overrides,
|
||||
run_number=-1,
|
||||
output_path=base_path / "warmup.json",
|
||||
)
|
||||
|
||||
for run_number in range(num_runs):
|
||||
run_data = run_benchmark_with_error_handling(
|
||||
bench_overrides=bench_comb,
|
||||
run_number=run_number,
|
||||
output_path=_get_comb_run_path(base_path, run_number),
|
||||
)
|
||||
if run_data is not None:
|
||||
comb_data.append(run_data)
|
||||
|
||||
if dry_run:
|
||||
return None
|
||||
|
||||
if not comb_data:
|
||||
print("[NO SUCCESSFUL MEASURED RUNS] leaving combination retryable")
|
||||
return []
|
||||
|
||||
with _get_comb_run_path(base_path, run_number=None).open("w") as f:
|
||||
json.dump(comb_data, f, indent=4)
|
||||
|
||||
return comb_data
|
||||
|
||||
|
||||
def run_combs(
|
||||
serve_cmd: list[str],
|
||||
bench_cmd: list[str],
|
||||
after_bench_cmd: list[str],
|
||||
*,
|
||||
show_stdout: bool,
|
||||
server_ready_timeout: int,
|
||||
serve_params: ParameterSweep,
|
||||
bench_params: ParameterSweep,
|
||||
link_vars: list[tuple[str, str]],
|
||||
experiment_dir: Path,
|
||||
num_runs: int,
|
||||
warmup_num_prompts: int,
|
||||
dry_run: bool,
|
||||
continue_on_error: bool = False,
|
||||
):
|
||||
all_data = list[dict[str, object]]()
|
||||
for serve_comb in serve_params:
|
||||
try:
|
||||
with server_ctx(
|
||||
serve_cmd,
|
||||
after_bench_cmd,
|
||||
show_stdout=show_stdout,
|
||||
serve_comb=serve_comb,
|
||||
bench_params=bench_params,
|
||||
experiment_dir=experiment_dir,
|
||||
dry_run=dry_run,
|
||||
server_ready_timeout=server_ready_timeout,
|
||||
) as server:
|
||||
for bench_comb in bench_params:
|
||||
base_path = _get_comb_base_path(
|
||||
experiment_dir, serve_comb, bench_comb
|
||||
)
|
||||
|
||||
comb_data = run_comb(
|
||||
server,
|
||||
bench_cmd,
|
||||
serve_comb=serve_comb,
|
||||
bench_comb=bench_comb,
|
||||
link_vars=link_vars,
|
||||
base_path=base_path,
|
||||
num_runs=num_runs,
|
||||
warmup_num_prompts=warmup_num_prompts,
|
||||
dry_run=dry_run,
|
||||
continue_on_error=continue_on_error,
|
||||
)
|
||||
|
||||
if comb_data is not None:
|
||||
all_data.extend(comb_data)
|
||||
except Exception as exc:
|
||||
if not continue_on_error:
|
||||
raise
|
||||
failure_name = sanitize_filename(
|
||||
f"SERVE-{serve_comb.name or 'default'}-failure.json"
|
||||
)
|
||||
failure_path = experiment_dir / failure_name
|
||||
failure_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
failure_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"error_type": type(exc).__name__,
|
||||
"error": str(exc),
|
||||
"serve_overrides": dict(serve_comb),
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(
|
||||
"[FAILED SERVER COMBINATION] "
|
||||
f"{serve_comb.name or '(default)'}: "
|
||||
f"{type(exc).__name__}: {exc}"
|
||||
)
|
||||
print(f"Failure details: {failure_path}")
|
||||
|
||||
if dry_run:
|
||||
return None
|
||||
|
||||
combined_df = pd.DataFrame.from_records(all_data)
|
||||
combined_df.to_csv(experiment_dir / "summary.csv")
|
||||
|
||||
return combined_df
|
||||
|
||||
|
||||
@dataclass
|
||||
class SweepServeArgs:
|
||||
serve_cmd: list[str]
|
||||
bench_cmd: list[str]
|
||||
after_bench_cmd: list[str]
|
||||
show_stdout: bool
|
||||
server_ready_timeout: int
|
||||
serve_params: ParameterSweep
|
||||
bench_params: ParameterSweep
|
||||
link_vars: list[tuple[str, str]]
|
||||
output_dir: Path
|
||||
experiment_name: str
|
||||
num_runs: int
|
||||
warmup_num_prompts: int
|
||||
continue_on_error: bool
|
||||
dry_run: bool
|
||||
resume: bool
|
||||
|
||||
parser_name: ClassVar[str] = "serve"
|
||||
parser_help: ClassVar[str] = "Run vLLM server benchmark under multiple settings."
|
||||
|
||||
@classmethod
|
||||
def from_cli_args(cls, args: argparse.Namespace):
|
||||
serve_cmd = shlex.split(args.serve_cmd)
|
||||
bench_cmd = shlex.split(args.bench_cmd)
|
||||
after_bench_cmd = (
|
||||
[] if args.after_bench_cmd is None else shlex.split(args.after_bench_cmd)
|
||||
)
|
||||
|
||||
if args.serve_params:
|
||||
serve_params = ParameterSweep.read_json(args.serve_params)
|
||||
else:
|
||||
# i.e.: run serve_cmd without any modification
|
||||
serve_params = ParameterSweep.from_records([{}])
|
||||
|
||||
if args.bench_params:
|
||||
bench_params = ParameterSweep.read_json(args.bench_params)
|
||||
else:
|
||||
# i.e.: run bench_cmd without any modification
|
||||
bench_params = ParameterSweep.from_records([{}])
|
||||
|
||||
link_vars = cls.parse_link_vars(args.link_vars)
|
||||
|
||||
if args.experiment_name:
|
||||
experiment_name = args.experiment_name
|
||||
else:
|
||||
experiment_name = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
|
||||
num_runs = args.num_runs
|
||||
if num_runs < 1:
|
||||
raise ValueError("`num_runs` should be at least 1.")
|
||||
warmup_num_prompts = args.warmup_num_prompts
|
||||
if warmup_num_prompts < 0:
|
||||
raise ValueError("`warmup_num_prompts` should be at least 0.")
|
||||
|
||||
return cls(
|
||||
serve_cmd=serve_cmd,
|
||||
bench_cmd=bench_cmd,
|
||||
after_bench_cmd=after_bench_cmd,
|
||||
show_stdout=args.show_stdout,
|
||||
serve_params=serve_params,
|
||||
bench_params=bench_params,
|
||||
link_vars=link_vars,
|
||||
output_dir=Path(args.output_dir),
|
||||
experiment_name=experiment_name,
|
||||
num_runs=num_runs,
|
||||
warmup_num_prompts=warmup_num_prompts,
|
||||
continue_on_error=args.continue_on_error,
|
||||
dry_run=args.dry_run,
|
||||
resume=args.resume,
|
||||
server_ready_timeout=args.server_ready_timeout,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def add_cli_args(cls, parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
|
||||
parser.add_argument(
|
||||
"--serve-cmd",
|
||||
type=str,
|
||||
required=True,
|
||||
help="The command used to run the server: `vllm serve ...`",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--bench-cmd",
|
||||
type=str,
|
||||
required=True,
|
||||
help="The command used to run the benchmark: `vllm bench serve ...`",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--after-bench-cmd",
|
||||
type=str,
|
||||
default=None,
|
||||
help="After a benchmark run is complete, invoke this command instead of "
|
||||
"the default `ServerWrapper.clear_cache()`.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--show-stdout",
|
||||
action="store_true",
|
||||
help="If set, logs the standard output of subcommands. "
|
||||
"Useful for debugging but can be quite spammy.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--server-ready-timeout",
|
||||
type=int,
|
||||
default=300,
|
||||
help="Timeout in seconds to wait for the server to become ready.",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--serve-params",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to JSON file containing parameter combinations "
|
||||
"for the `vllm serve` command. Can be either a list of dicts or a dict "
|
||||
"where keys are benchmark names. "
|
||||
"If both `serve_params` and `bench_params` are given, "
|
||||
"this script will iterate over their Cartesian product.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--link-vars",
|
||||
type=str,
|
||||
default="",
|
||||
help=(
|
||||
"Comma-separated list of linked variables between serve and bench, "
|
||||
"e.g. max_num_seqs=max_concurrency,max_model_len=random_input_len"
|
||||
),
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--bench-params",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to JSON file containing parameter combinations "
|
||||
"for the `vllm bench serve` command. Can be either a list of dicts or "
|
||||
"a dict where keys are benchmark names. "
|
||||
"If both `serve_params` and `bench_params` are given, "
|
||||
"this script will iterate over their Cartesian product.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-o",
|
||||
"--output-dir",
|
||||
type=str,
|
||||
default="results",
|
||||
help="The main directory to which results are written.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-e",
|
||||
"--experiment-name",
|
||||
type=str,
|
||||
default=None,
|
||||
help="The name of this experiment (defaults to current timestamp). "
|
||||
"Results will be stored under `output_dir/experiment_name`.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num-runs",
|
||||
type=int,
|
||||
default=3,
|
||||
help="Number of runs per parameter combination.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--warmup-num-prompts",
|
||||
type=int,
|
||||
default=0,
|
||||
help=(
|
||||
"Number of prompts in one unmeasured warmup run before each "
|
||||
"parameter combination. The warmup is saved as warmup.json but "
|
||||
"excluded from summary statistics."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--continue-on-error",
|
||||
action="store_true",
|
||||
help=(
|
||||
"Record and skip a failed warmup or benchmark run instead of "
|
||||
"terminating the entire sweep."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="If set, prints the commands to run, "
|
||||
"then exits without executing them.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--resume",
|
||||
action="store_true",
|
||||
help="Resume a previous execution of this script, i.e., only run "
|
||||
"parameter combinations for which there are still no output files "
|
||||
"under `output_dir/experiment_name`.",
|
||||
)
|
||||
|
||||
return parser
|
||||
|
||||
@staticmethod
|
||||
def parse_link_vars(s: str) -> list[tuple[str, str]]:
|
||||
if not s:
|
||||
return []
|
||||
pairs = []
|
||||
for item in s.split(","):
|
||||
a, b = item.split("=")
|
||||
pairs.append((a.strip(), b.strip()))
|
||||
return pairs
|
||||
|
||||
def resolve_experiment_dir(self) -> Path:
|
||||
experiment_dir = self.output_dir / self.experiment_name
|
||||
|
||||
if self.resume:
|
||||
if not experiment_dir.exists():
|
||||
raise ValueError(f"Cannot resume from non-existent {experiment_dir=}")
|
||||
else:
|
||||
if experiment_dir.exists():
|
||||
raise ValueError(f"Cannot overwrite existing {experiment_dir=}")
|
||||
|
||||
return experiment_dir
|
||||
|
||||
@contextmanager
|
||||
def run_ctx(self, experiment_dir: Path):
|
||||
if self.dry_run:
|
||||
yield
|
||||
print(f"Experiment will be saved at: {experiment_dir}")
|
||||
return
|
||||
|
||||
try:
|
||||
yield
|
||||
print(f"Experiment has been saved at: {experiment_dir}")
|
||||
except BaseException as exc:
|
||||
raise RuntimeError(
|
||||
"The script was terminated early. Use `--resume` "
|
||||
"to continue the script from its last checkpoint."
|
||||
) from exc
|
||||
|
||||
|
||||
def run_main(args: SweepServeArgs):
|
||||
experiment_dir = args.resolve_experiment_dir()
|
||||
|
||||
with args.run_ctx(experiment_dir):
|
||||
return run_combs(
|
||||
serve_cmd=args.serve_cmd,
|
||||
bench_cmd=args.bench_cmd,
|
||||
link_vars=args.link_vars,
|
||||
after_bench_cmd=args.after_bench_cmd,
|
||||
show_stdout=args.show_stdout,
|
||||
server_ready_timeout=args.server_ready_timeout,
|
||||
serve_params=args.serve_params,
|
||||
bench_params=args.bench_params,
|
||||
experiment_dir=experiment_dir,
|
||||
num_runs=args.num_runs,
|
||||
warmup_num_prompts=args.warmup_num_prompts,
|
||||
dry_run=args.dry_run,
|
||||
continue_on_error=args.continue_on_error,
|
||||
)
|
||||
|
||||
|
||||
def main(args: argparse.Namespace):
|
||||
run_main(SweepServeArgs.from_cli_args(args))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = FlexibleArgumentParser(description=SweepServeArgs.parser_help)
|
||||
SweepServeArgs.add_cli_args(parser)
|
||||
|
||||
main(parser.parse_args())
|
||||
@@ -1,350 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
import math
|
||||
from dataclasses import dataclass, fields
|
||||
from pathlib import Path
|
||||
from typing import ClassVar, Literal, assert_never, get_args
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.benchmarks.datasets import DEFAULT_NUM_PROMPTS
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
from vllm.utils.import_utils import PlaceholderModule
|
||||
|
||||
from .param_sweep import ParameterSweep, ParameterSweepItem
|
||||
from .serve import (
|
||||
SweepServeArgs,
|
||||
_get_comb_base_path,
|
||||
run_comb,
|
||||
server_ctx,
|
||||
)
|
||||
from .server import ServerProcess
|
||||
|
||||
try:
|
||||
import pandas as pd
|
||||
except ImportError:
|
||||
pd = PlaceholderModule("pandas")
|
||||
|
||||
|
||||
WorkloadVariable = Literal["request_rate", "max_concurrency"]
|
||||
|
||||
|
||||
def _estimate_workload_value(
|
||||
run_data: dict[str, object],
|
||||
workload_var: WorkloadVariable,
|
||||
):
|
||||
request_throughput = float(run_data["request_throughput"]) # type: ignore
|
||||
if workload_var == "request_rate":
|
||||
return request_throughput
|
||||
if workload_var == "max_concurrency":
|
||||
mean_latency_ms = float(run_data["mean_e2el_ms"]) # type: ignore
|
||||
return request_throughput * mean_latency_ms / 1000
|
||||
|
||||
assert_never(workload_var)
|
||||
|
||||
|
||||
def _estimate_workload_avg(
|
||||
runs: list[dict[str, object]],
|
||||
workload_var: WorkloadVariable,
|
||||
):
|
||||
total = sum(_estimate_workload_value(run, workload_var) for run in runs)
|
||||
return total / len(runs)
|
||||
|
||||
|
||||
def run_comb_workload(
|
||||
server: ServerProcess | None,
|
||||
bench_cmd: list[str],
|
||||
*,
|
||||
serve_comb: ParameterSweepItem,
|
||||
bench_comb: ParameterSweepItem,
|
||||
link_vars: list[tuple[str, str]],
|
||||
experiment_dir: Path,
|
||||
num_runs: int,
|
||||
dry_run: bool,
|
||||
warmup_num_prompts: int,
|
||||
continue_on_error: bool,
|
||||
workload_var: WorkloadVariable,
|
||||
workload_value: int,
|
||||
) -> list[dict[str, object]] | None:
|
||||
bench_comb_workload = bench_comb | {workload_var: workload_value}
|
||||
|
||||
return run_comb(
|
||||
server,
|
||||
bench_cmd,
|
||||
serve_comb=serve_comb,
|
||||
bench_comb=bench_comb_workload,
|
||||
link_vars=link_vars,
|
||||
base_path=_get_comb_base_path(
|
||||
experiment_dir,
|
||||
serve_comb,
|
||||
bench_comb,
|
||||
extra_parts=("WL-", f"{workload_var}={workload_value}"),
|
||||
),
|
||||
num_runs=num_runs,
|
||||
dry_run=dry_run,
|
||||
warmup_num_prompts=warmup_num_prompts,
|
||||
continue_on_error=continue_on_error,
|
||||
)
|
||||
|
||||
|
||||
def explore_comb_workloads(
|
||||
server: ServerProcess | None,
|
||||
bench_cmd: list[str],
|
||||
*,
|
||||
serve_comb: ParameterSweepItem,
|
||||
bench_comb: ParameterSweepItem,
|
||||
link_vars: list[tuple[str, str]],
|
||||
workload_var: WorkloadVariable,
|
||||
workload_iters: int,
|
||||
experiment_dir: Path,
|
||||
num_runs: int,
|
||||
dry_run: bool,
|
||||
warmup_num_prompts: int,
|
||||
continue_on_error: bool,
|
||||
):
|
||||
print("[WL START]")
|
||||
print(f"Serve parameters: {serve_comb.as_text() or '(None)'}")
|
||||
print(f"Bench parameters: {bench_comb.as_text() or '(None)'}")
|
||||
print(f"Number of workload iterations: {workload_iters}")
|
||||
|
||||
if workload_iters < 2:
|
||||
raise ValueError("`workload_iters` should be at least 2")
|
||||
|
||||
dataset_size = DEFAULT_NUM_PROMPTS
|
||||
if "num_prompts" in bench_comb:
|
||||
dataset_size = int(bench_comb["num_prompts"]) # type: ignore
|
||||
else:
|
||||
for i, arg in enumerate(bench_cmd):
|
||||
if arg == "--num-prompts" and i + 1 < len(bench_cmd):
|
||||
dataset_size = int(bench_cmd[i + 1])
|
||||
break
|
||||
elif arg.startswith("--num-prompts="):
|
||||
dataset_size = int(arg.split("=", 1)[1])
|
||||
break
|
||||
|
||||
print(f"Dataset size: {dataset_size}")
|
||||
|
||||
serial_workload_data = run_comb_workload(
|
||||
server,
|
||||
bench_cmd,
|
||||
serve_comb=serve_comb,
|
||||
bench_comb=bench_comb | {"max_concurrency": 1},
|
||||
link_vars=link_vars,
|
||||
experiment_dir=experiment_dir,
|
||||
num_runs=num_runs,
|
||||
dry_run=dry_run,
|
||||
warmup_num_prompts=warmup_num_prompts,
|
||||
continue_on_error=continue_on_error,
|
||||
workload_var=workload_var,
|
||||
workload_value=1,
|
||||
)
|
||||
batch_workload_data = run_comb_workload(
|
||||
server,
|
||||
bench_cmd,
|
||||
serve_comb=serve_comb,
|
||||
bench_comb=bench_comb | {"max_concurrency": dataset_size},
|
||||
link_vars=link_vars,
|
||||
experiment_dir=experiment_dir,
|
||||
num_runs=num_runs,
|
||||
dry_run=dry_run,
|
||||
warmup_num_prompts=warmup_num_prompts,
|
||||
continue_on_error=continue_on_error,
|
||||
workload_var=workload_var,
|
||||
workload_value=dataset_size,
|
||||
)
|
||||
|
||||
if not serial_workload_data or not batch_workload_data:
|
||||
if dry_run:
|
||||
print("Omitting intermediate Workload iterations.")
|
||||
else:
|
||||
print(
|
||||
"Unable to establish both Workload Explorer endpoints; "
|
||||
"skipping intermediate workload levels."
|
||||
)
|
||||
print("[WL END]")
|
||||
return []
|
||||
|
||||
serial_workload_value = math.ceil(
|
||||
_estimate_workload_avg(serial_workload_data, workload_var)
|
||||
)
|
||||
print(f"Serial inference: {workload_var}={serial_workload_value}")
|
||||
|
||||
batch_workload_value = math.floor(
|
||||
_estimate_workload_avg(batch_workload_data, workload_var)
|
||||
)
|
||||
print(f"Batch inference: {workload_var}={batch_workload_value}")
|
||||
|
||||
# Avoid duplicated runs for intermediate values if the range between
|
||||
# `serial_workload_value` and `batch_workload_value` is small
|
||||
inter_workload_values = np.linspace(
|
||||
serial_workload_value, batch_workload_value, workload_iters
|
||||
)[1:-1]
|
||||
inter_workload_values = sorted(set(map(round, inter_workload_values)))
|
||||
|
||||
inter_workloads_data: list[dict[str, object]] = []
|
||||
for inter_workload_value in inter_workload_values:
|
||||
print(f"Exploring: {workload_var}={inter_workload_value}")
|
||||
inter_workload_data = run_comb_workload(
|
||||
server,
|
||||
bench_cmd,
|
||||
serve_comb=serve_comb,
|
||||
bench_comb=bench_comb,
|
||||
link_vars=link_vars,
|
||||
experiment_dir=experiment_dir,
|
||||
num_runs=num_runs,
|
||||
dry_run=dry_run,
|
||||
warmup_num_prompts=warmup_num_prompts,
|
||||
continue_on_error=continue_on_error,
|
||||
workload_var=workload_var,
|
||||
workload_value=inter_workload_value,
|
||||
)
|
||||
if inter_workload_data is not None:
|
||||
inter_workloads_data.extend(inter_workload_data)
|
||||
|
||||
print("[WL END]")
|
||||
|
||||
return serial_workload_data + inter_workloads_data + batch_workload_data
|
||||
|
||||
|
||||
def explore_combs_workloads(
|
||||
serve_cmd: list[str],
|
||||
bench_cmd: list[str],
|
||||
after_bench_cmd: list[str],
|
||||
*,
|
||||
show_stdout: bool,
|
||||
server_ready_timeout: int,
|
||||
serve_params: ParameterSweep,
|
||||
bench_params: ParameterSweep,
|
||||
link_vars: list[tuple[str, str]],
|
||||
workload_var: WorkloadVariable,
|
||||
workload_iters: int,
|
||||
experiment_dir: Path,
|
||||
num_runs: int,
|
||||
dry_run: bool,
|
||||
warmup_num_prompts: int,
|
||||
continue_on_error: bool,
|
||||
):
|
||||
if any(bench_comb.has_param(workload_var) for bench_comb in bench_params):
|
||||
raise ValueError(
|
||||
f"You should not override `{workload_var}` in `bench_params` "
|
||||
"since it is supposed to be explored automatically."
|
||||
)
|
||||
|
||||
all_data = list[dict[str, object]]()
|
||||
for serve_comb in serve_params:
|
||||
with server_ctx(
|
||||
serve_cmd,
|
||||
after_bench_cmd,
|
||||
show_stdout=show_stdout,
|
||||
server_ready_timeout=server_ready_timeout,
|
||||
serve_comb=serve_comb,
|
||||
bench_params=bench_params,
|
||||
experiment_dir=experiment_dir,
|
||||
dry_run=dry_run,
|
||||
) as server:
|
||||
for bench_comb in bench_params:
|
||||
comb_data = explore_comb_workloads(
|
||||
server,
|
||||
bench_cmd,
|
||||
serve_comb=serve_comb,
|
||||
bench_comb=bench_comb,
|
||||
link_vars=link_vars,
|
||||
workload_var=workload_var,
|
||||
workload_iters=workload_iters,
|
||||
experiment_dir=experiment_dir,
|
||||
num_runs=num_runs,
|
||||
dry_run=dry_run,
|
||||
warmup_num_prompts=warmup_num_prompts,
|
||||
continue_on_error=continue_on_error,
|
||||
)
|
||||
|
||||
if comb_data is not None:
|
||||
all_data.extend(comb_data)
|
||||
|
||||
if dry_run:
|
||||
return None
|
||||
|
||||
combined_df = pd.DataFrame.from_records(all_data)
|
||||
combined_df.to_csv(experiment_dir / "summary.csv")
|
||||
|
||||
return combined_df
|
||||
|
||||
|
||||
@dataclass
|
||||
class SweepServeWorkloadArgs(SweepServeArgs):
|
||||
workload_var: WorkloadVariable
|
||||
workload_iters: int
|
||||
|
||||
parser_name: ClassVar[str] = "serve_workload"
|
||||
parser_help: ClassVar[str] = (
|
||||
"Explore the latency-throughput tradeoff for different workload levels."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_cli_args(cls, args: argparse.Namespace):
|
||||
# NOTE: Don't use super() as `from_cli_args` calls `cls()`
|
||||
base_args = SweepServeArgs.from_cli_args(args)
|
||||
|
||||
return cls(
|
||||
**{f.name: getattr(base_args, f.name) for f in fields(base_args)},
|
||||
workload_var=args.workload_var,
|
||||
workload_iters=args.workload_iters,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def add_cli_args(cls, parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
|
||||
parser = super().add_cli_args(parser)
|
||||
|
||||
workload_group = parser.add_argument_group("workload options")
|
||||
workload_group.add_argument(
|
||||
"--workload-var",
|
||||
type=str,
|
||||
choices=get_args(WorkloadVariable),
|
||||
default="request_rate",
|
||||
help="The variable to adjust in each iteration.",
|
||||
)
|
||||
workload_group.add_argument(
|
||||
"--workload-iters",
|
||||
type=int,
|
||||
default=10,
|
||||
help="Number of workload levels to explore. "
|
||||
"This includes the first two iterations used to interpolate the value of "
|
||||
"`workload_var` for remaining iterations.",
|
||||
)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def run_main(args: SweepServeWorkloadArgs):
|
||||
experiment_dir = args.resolve_experiment_dir()
|
||||
|
||||
with args.run_ctx(experiment_dir):
|
||||
return explore_combs_workloads(
|
||||
serve_cmd=args.serve_cmd,
|
||||
bench_cmd=args.bench_cmd,
|
||||
after_bench_cmd=args.after_bench_cmd,
|
||||
show_stdout=args.show_stdout,
|
||||
server_ready_timeout=args.server_ready_timeout,
|
||||
serve_params=args.serve_params,
|
||||
bench_params=args.bench_params,
|
||||
link_vars=args.link_vars,
|
||||
workload_var=args.workload_var,
|
||||
workload_iters=args.workload_iters,
|
||||
experiment_dir=experiment_dir,
|
||||
num_runs=args.num_runs,
|
||||
dry_run=args.dry_run,
|
||||
warmup_num_prompts=args.warmup_num_prompts,
|
||||
continue_on_error=args.continue_on_error,
|
||||
)
|
||||
|
||||
|
||||
def main(args: argparse.Namespace):
|
||||
run_main(SweepServeWorkloadArgs.from_cli_args(args))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = FlexibleArgumentParser(description=SweepServeWorkloadArgs.parser_help)
|
||||
SweepServeWorkloadArgs.add_cli_args(parser)
|
||||
|
||||
main(parser.parse_args())
|
||||
@@ -1,142 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import contextlib
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import time
|
||||
from types import TracebackType
|
||||
from typing import Self
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
class ServerProcess:
|
||||
VLLM_RESET_CACHE_ENDPOINTS = [
|
||||
"/reset_prefix_cache",
|
||||
"/reset_mm_cache",
|
||||
"/reset_encoder_cache",
|
||||
]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
server_cmd: list[str],
|
||||
after_bench_cmd: list[str],
|
||||
*,
|
||||
show_stdout: bool,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.server_cmd = server_cmd
|
||||
self.after_bench_cmd = after_bench_cmd
|
||||
self.show_stdout = show_stdout
|
||||
|
||||
def __enter__(self) -> Self:
|
||||
self.start()
|
||||
return self
|
||||
|
||||
def __exit__(
|
||||
self,
|
||||
exc_type: type[BaseException] | None,
|
||||
exc_value: BaseException | None,
|
||||
exc_traceback: TracebackType | None,
|
||||
) -> None:
|
||||
self.stop()
|
||||
|
||||
def start(self):
|
||||
# Create new process for clean termination
|
||||
self._server_process = subprocess.Popen(
|
||||
self.server_cmd,
|
||||
start_new_session=True,
|
||||
stdout=None if self.show_stdout else subprocess.DEVNULL,
|
||||
# Need `VLLM_SERVER_DEV_MODE=1` for `_reset_caches`
|
||||
env=os.environ | {"VLLM_SERVER_DEV_MODE": "1"},
|
||||
)
|
||||
|
||||
def stop(self):
|
||||
server_process = self._server_process
|
||||
|
||||
if server_process.poll() is None:
|
||||
# In case only some processes have been terminated
|
||||
with contextlib.suppress(ProcessLookupError):
|
||||
# We need to kill both API Server and Engine processes
|
||||
os.killpg(os.getpgid(server_process.pid), signal.SIGKILL)
|
||||
|
||||
def run_subcommand(self, cmd: list[str]):
|
||||
return subprocess.run(
|
||||
cmd,
|
||||
stdout=None if self.show_stdout else subprocess.DEVNULL,
|
||||
check=True,
|
||||
)
|
||||
|
||||
def after_bench(self) -> None:
|
||||
if not self.after_bench_cmd:
|
||||
self.reset_caches()
|
||||
return
|
||||
|
||||
self.run_subcommand(self.after_bench_cmd)
|
||||
|
||||
def _get_vllm_server_address(self) -> str:
|
||||
server_cmd = self.server_cmd
|
||||
|
||||
for host_key in ("--host",):
|
||||
if host_key in server_cmd:
|
||||
host = server_cmd[server_cmd.index(host_key) + 1]
|
||||
break
|
||||
else:
|
||||
host = "localhost"
|
||||
|
||||
for port_key in ("-p", "--port"):
|
||||
if port_key in server_cmd:
|
||||
port = int(server_cmd[server_cmd.index(port_key) + 1])
|
||||
break
|
||||
else:
|
||||
port = 8000 # The default value in vllm serve
|
||||
|
||||
return f"http://{host}:{port}"
|
||||
|
||||
def is_server_ready(self) -> bool:
|
||||
server_address = self._get_vllm_server_address()
|
||||
try:
|
||||
response = requests.get(f"{server_address}/health")
|
||||
return response.status_code == 200
|
||||
except requests.RequestException:
|
||||
return False
|
||||
|
||||
def wait_until_ready(self, timeout: int) -> None:
|
||||
start_time = time.monotonic()
|
||||
while not self.is_server_ready():
|
||||
# Check if server process has crashed
|
||||
if self._server_process.poll() is not None:
|
||||
returncode = self._server_process.returncode
|
||||
raise RuntimeError(
|
||||
f"Server process crashed with return code {returncode}"
|
||||
)
|
||||
if time.monotonic() - start_time > timeout:
|
||||
raise TimeoutError(
|
||||
f"Server failed to become ready within {timeout} seconds."
|
||||
)
|
||||
time.sleep(1)
|
||||
|
||||
def reset_caches(self) -> None:
|
||||
server_cmd = self.server_cmd
|
||||
|
||||
# Use `.endswith()` to match `/bin/...`
|
||||
if server_cmd[0].endswith("vllm"):
|
||||
server_address = self._get_vllm_server_address()
|
||||
print(f"Resetting caches at {server_address}")
|
||||
|
||||
for endpoint in self.VLLM_RESET_CACHE_ENDPOINTS:
|
||||
res = requests.post(server_address + endpoint)
|
||||
res.raise_for_status()
|
||||
elif server_cmd[0].endswith("infinity_emb"):
|
||||
if "--vector-disk-cache" in server_cmd:
|
||||
raise NotImplementedError(
|
||||
"Infinity server uses caching but does not expose a method "
|
||||
"to reset the cache"
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
f"No implementation of `reset_caches` for `{server_cmd[0]}` server. "
|
||||
"Please specify a custom command via `--after-bench-cmd`."
|
||||
)
|
||||
@@ -1,443 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
import json
|
||||
import shlex
|
||||
import subprocess
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import ClassVar
|
||||
|
||||
from vllm.benchmarks.startup import add_cli_args as add_startup_cli_args
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
from vllm.utils.import_utils import PlaceholderModule
|
||||
|
||||
from .param_sweep import ParameterSweep, ParameterSweepItem
|
||||
from .utils import sanitize_filename
|
||||
|
||||
try:
|
||||
import pandas as pd
|
||||
except ImportError:
|
||||
pd = PlaceholderModule("pandas")
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _get_supported_startup_keys() -> set[str]:
|
||||
parser = FlexibleArgumentParser(add_help=False)
|
||||
add_startup_cli_args(parser)
|
||||
|
||||
supported: set[str] = {"config"}
|
||||
for action in parser._actions:
|
||||
if action.dest and action.dest is not argparse.SUPPRESS:
|
||||
supported.add(action.dest)
|
||||
for option in action.option_strings:
|
||||
if option.startswith("--"):
|
||||
supported.add(option.lstrip("-").replace("-", "_"))
|
||||
|
||||
return supported
|
||||
|
||||
|
||||
def _is_supported_param(param_key: str, supported: set[str]) -> bool:
|
||||
if param_key == "_benchmark_name":
|
||||
return True
|
||||
prefix = param_key.split(".", 1)[0]
|
||||
normalized = prefix.replace("-", "_")
|
||||
return normalized in supported
|
||||
|
||||
|
||||
def _filter_params(
|
||||
params: ParameterSweep, *, supported: set[str], strict: bool
|
||||
) -> ParameterSweep:
|
||||
filtered = []
|
||||
for item in params:
|
||||
kept: dict[str, object] = {}
|
||||
dropped: list[str] = []
|
||||
for key, value in item.items():
|
||||
if _is_supported_param(key, supported):
|
||||
kept[key] = value
|
||||
else:
|
||||
dropped.append(key)
|
||||
|
||||
if dropped:
|
||||
label = item.get("_benchmark_name") or item.as_text()
|
||||
message = (
|
||||
"Ignoring unsupported startup params"
|
||||
f"{' for ' + str(label) if label else ''}: "
|
||||
f"{', '.join(sorted(dropped))}"
|
||||
)
|
||||
if strict:
|
||||
raise ValueError(message)
|
||||
print(message)
|
||||
|
||||
filtered.append(ParameterSweepItem.from_record(kept))
|
||||
|
||||
return ParameterSweep(filtered)
|
||||
|
||||
|
||||
def _update_run_data(
|
||||
run_data: dict[str, object],
|
||||
serve_overrides: ParameterSweepItem,
|
||||
startup_overrides: ParameterSweepItem,
|
||||
run_number: int,
|
||||
) -> dict[str, object]:
|
||||
run_data["run_number"] = run_number
|
||||
run_data.update(serve_overrides)
|
||||
run_data.update(startup_overrides)
|
||||
return run_data
|
||||
|
||||
|
||||
def _strip_arg(cmd: list[str], keys: tuple[str, ...]) -> list[str]:
|
||||
stripped: list[str] = []
|
||||
skip_next = False
|
||||
for arg in cmd:
|
||||
if skip_next:
|
||||
skip_next = False
|
||||
continue
|
||||
if arg in keys:
|
||||
skip_next = True
|
||||
continue
|
||||
if any(arg.startswith(f"{key}=") for key in keys):
|
||||
continue
|
||||
stripped.append(arg)
|
||||
return stripped
|
||||
|
||||
|
||||
def _apply_output_json(cmd: list[str], output_path: Path) -> list[str]:
|
||||
keys = ("--output-json", "--output_json")
|
||||
cmd = _strip_arg(cmd, keys)
|
||||
return [*cmd, keys[0], str(output_path)]
|
||||
|
||||
|
||||
def _get_comb_base_path(
|
||||
experiment_dir: Path,
|
||||
serve_comb: ParameterSweepItem,
|
||||
startup_comb: ParameterSweepItem,
|
||||
) -> Path:
|
||||
parts = list[str]()
|
||||
if serve_comb:
|
||||
parts.extend(("SERVE-", serve_comb.name))
|
||||
if startup_comb:
|
||||
parts.extend(("STARTUP-", startup_comb.name))
|
||||
|
||||
return experiment_dir / sanitize_filename("-".join(parts))
|
||||
|
||||
|
||||
def _get_comb_run_path(base_path: Path, run_number: int | None) -> Path:
|
||||
if run_number is None:
|
||||
return base_path / "summary.json"
|
||||
return base_path / f"run={run_number}.json"
|
||||
|
||||
|
||||
def run_benchmark(
|
||||
startup_cmd: list[str],
|
||||
*,
|
||||
serve_overrides: ParameterSweepItem,
|
||||
startup_overrides: ParameterSweepItem,
|
||||
run_number: int,
|
||||
output_path: Path,
|
||||
show_stdout: bool,
|
||||
dry_run: bool,
|
||||
) -> dict[str, object] | None:
|
||||
cmd = serve_overrides.apply_to_cmd(startup_cmd)
|
||||
cmd = startup_overrides.apply_to_cmd(cmd)
|
||||
cmd = _apply_output_json(cmd, output_path)
|
||||
|
||||
print("[BEGIN BENCHMARK]")
|
||||
print(f"Serve overrides: {serve_overrides}")
|
||||
print(f"Startup overrides: {startup_overrides}")
|
||||
print(f"Run Number: {run_number}")
|
||||
print(f"Benchmark command: {cmd}")
|
||||
print(f"Output file: {output_path}")
|
||||
|
||||
if output_path.exists():
|
||||
print("Found existing results.")
|
||||
print("[SKIPPED BENCHMARK]")
|
||||
|
||||
with output_path.open("r", encoding="utf-8") as f:
|
||||
run_data = json.load(f)
|
||||
return _update_run_data(
|
||||
run_data, serve_overrides, startup_overrides, run_number
|
||||
)
|
||||
|
||||
if dry_run:
|
||||
print("[END BENCHMARK]")
|
||||
return None
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
subprocess.run(
|
||||
cmd,
|
||||
stdout=None if show_stdout else subprocess.DEVNULL,
|
||||
check=True,
|
||||
)
|
||||
|
||||
with output_path.open("r", encoding="utf-8") as f:
|
||||
run_data = json.load(f)
|
||||
|
||||
run_data = _update_run_data(
|
||||
run_data, serve_overrides, startup_overrides, run_number
|
||||
)
|
||||
|
||||
with output_path.open("w", encoding="utf-8") as f:
|
||||
json.dump(run_data, f, indent=4)
|
||||
|
||||
print("[END BENCHMARK]")
|
||||
return run_data
|
||||
|
||||
|
||||
def run_comb(
|
||||
startup_cmd: list[str],
|
||||
*,
|
||||
serve_comb: ParameterSweepItem,
|
||||
startup_comb: ParameterSweepItem,
|
||||
base_path: Path,
|
||||
num_runs: int,
|
||||
show_stdout: bool,
|
||||
dry_run: bool,
|
||||
) -> list[dict[str, object]] | None:
|
||||
comb_data = list[dict[str, object]]()
|
||||
for run_number in range(num_runs):
|
||||
run_data = run_benchmark(
|
||||
startup_cmd,
|
||||
serve_overrides=serve_comb,
|
||||
startup_overrides=startup_comb,
|
||||
run_number=run_number,
|
||||
output_path=_get_comb_run_path(base_path, run_number),
|
||||
show_stdout=show_stdout,
|
||||
dry_run=dry_run,
|
||||
)
|
||||
if run_data is not None:
|
||||
comb_data.append(run_data)
|
||||
|
||||
if dry_run:
|
||||
return None
|
||||
|
||||
with _get_comb_run_path(base_path, run_number=None).open(
|
||||
"w", encoding="utf-8"
|
||||
) as f:
|
||||
json.dump(comb_data, f, indent=4)
|
||||
|
||||
return comb_data
|
||||
|
||||
|
||||
def run_combs(
|
||||
startup_cmd: list[str],
|
||||
*,
|
||||
serve_params: ParameterSweep,
|
||||
startup_params: ParameterSweep,
|
||||
experiment_dir: Path,
|
||||
num_runs: int,
|
||||
show_stdout: bool,
|
||||
dry_run: bool,
|
||||
) -> "pd.DataFrame | None":
|
||||
all_data = list[dict[str, object]]()
|
||||
for serve_comb in serve_params:
|
||||
for startup_comb in startup_params:
|
||||
base_path = _get_comb_base_path(experiment_dir, serve_comb, startup_comb)
|
||||
comb_data = run_comb(
|
||||
startup_cmd,
|
||||
serve_comb=serve_comb,
|
||||
startup_comb=startup_comb,
|
||||
base_path=base_path,
|
||||
num_runs=num_runs,
|
||||
show_stdout=show_stdout,
|
||||
dry_run=dry_run,
|
||||
)
|
||||
if comb_data is not None:
|
||||
all_data.extend(comb_data)
|
||||
|
||||
if dry_run:
|
||||
return None
|
||||
|
||||
combined_df = pd.DataFrame.from_records(all_data)
|
||||
combined_df.to_csv(experiment_dir / "summary.csv")
|
||||
return combined_df
|
||||
|
||||
|
||||
@dataclass
|
||||
class SweepStartupArgs:
|
||||
startup_cmd: list[str]
|
||||
serve_params: ParameterSweep
|
||||
startup_params: ParameterSweep
|
||||
output_dir: Path
|
||||
experiment_name: str
|
||||
num_runs: int
|
||||
show_stdout: bool
|
||||
dry_run: bool
|
||||
resume: bool
|
||||
|
||||
parser_name: ClassVar[str] = "startup"
|
||||
parser_help: ClassVar[str] = (
|
||||
"Benchmark vLLM startup time over parameter combinations."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_cli_args(cls, args: argparse.Namespace):
|
||||
startup_cmd = shlex.split(args.startup_cmd)
|
||||
|
||||
if args.serve_params:
|
||||
serve_params = ParameterSweep.read_json(args.serve_params)
|
||||
else:
|
||||
serve_params = ParameterSweep.from_records([{}])
|
||||
|
||||
if args.startup_params:
|
||||
startup_params = ParameterSweep.read_json(args.startup_params)
|
||||
else:
|
||||
startup_params = ParameterSweep.from_records([{}])
|
||||
|
||||
supported = _get_supported_startup_keys()
|
||||
strict_params = args.strict_params
|
||||
serve_params = _filter_params(
|
||||
serve_params, supported=supported, strict=strict_params
|
||||
)
|
||||
startup_params = _filter_params(
|
||||
startup_params, supported=supported, strict=strict_params
|
||||
)
|
||||
|
||||
if args.experiment_name:
|
||||
experiment_name = args.experiment_name
|
||||
else:
|
||||
experiment_name = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
|
||||
if args.num_runs < 1:
|
||||
raise ValueError("`num_runs` should be at least 1.")
|
||||
|
||||
return cls(
|
||||
startup_cmd=startup_cmd,
|
||||
serve_params=serve_params,
|
||||
startup_params=startup_params,
|
||||
output_dir=Path(args.output_dir),
|
||||
experiment_name=experiment_name,
|
||||
num_runs=args.num_runs,
|
||||
show_stdout=args.show_stdout,
|
||||
dry_run=args.dry_run,
|
||||
resume=args.resume,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def add_cli_args(cls, parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
|
||||
parser.add_argument(
|
||||
"--startup-cmd",
|
||||
type=str,
|
||||
default="vllm bench startup",
|
||||
help="The command used to run the startup benchmark.",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--serve-params",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to JSON file containing parameter combinations "
|
||||
"for the `vllm serve` command. Only parameters supported by "
|
||||
"`vllm bench startup` will be applied.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--startup-params",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to JSON file containing parameter combinations "
|
||||
"for the `vllm bench startup` command.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--strict-params",
|
||||
action="store_true",
|
||||
help="If set, unknown parameters in sweep files raise an error "
|
||||
"instead of being ignored.",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"-o",
|
||||
"--output-dir",
|
||||
type=str,
|
||||
default="results",
|
||||
help="The main directory to which results are written.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-e",
|
||||
"--experiment-name",
|
||||
type=str,
|
||||
default=None,
|
||||
help="The name of this experiment (defaults to current timestamp). "
|
||||
"Results will be stored under `output_dir/experiment_name`.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num-runs",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Number of runs per parameter combination.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--show-stdout",
|
||||
action="store_true",
|
||||
help="If set, logs the standard output of subcommands.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="If set, prints the commands to run, "
|
||||
"then exits without executing them.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--resume",
|
||||
action="store_true",
|
||||
help="Resume a previous execution of this script, i.e., only run "
|
||||
"parameter combinations for which there are still no output files "
|
||||
"under `output_dir/experiment_name`.",
|
||||
)
|
||||
|
||||
return parser
|
||||
|
||||
def resolve_experiment_dir(self) -> Path:
|
||||
experiment_dir = self.output_dir / self.experiment_name
|
||||
|
||||
if self.resume:
|
||||
if not experiment_dir.exists():
|
||||
raise ValueError(f"Cannot resume from non-existent {experiment_dir=}")
|
||||
else:
|
||||
if experiment_dir.exists():
|
||||
raise ValueError(f"Cannot overwrite existing {experiment_dir=}")
|
||||
|
||||
return experiment_dir
|
||||
|
||||
@contextmanager
|
||||
def run_ctx(self, experiment_dir: Path):
|
||||
if self.dry_run:
|
||||
yield
|
||||
print(f"Experiment will be saved at: {experiment_dir}")
|
||||
return
|
||||
|
||||
try:
|
||||
yield
|
||||
print(f"Experiment has been saved at: {experiment_dir}")
|
||||
except BaseException as exc:
|
||||
raise RuntimeError(
|
||||
"The script was terminated early. Use `--resume` "
|
||||
"to continue the script from its last checkpoint."
|
||||
) from exc
|
||||
|
||||
|
||||
def run_main(args: SweepStartupArgs):
|
||||
experiment_dir = args.resolve_experiment_dir()
|
||||
|
||||
with args.run_ctx(experiment_dir):
|
||||
return run_combs(
|
||||
startup_cmd=args.startup_cmd,
|
||||
serve_params=args.serve_params,
|
||||
startup_params=args.startup_params,
|
||||
experiment_dir=experiment_dir,
|
||||
num_runs=args.num_runs,
|
||||
show_stdout=args.show_stdout,
|
||||
dry_run=args.dry_run,
|
||||
)
|
||||
|
||||
|
||||
def main(args: argparse.Namespace):
|
||||
run_main(SweepStartupArgs.from_cli_args(args))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = FlexibleArgumentParser(description=SweepStartupArgs.parser_help)
|
||||
SweepStartupArgs.add_cli_args(parser)
|
||||
main(parser.parse_args())
|
||||
@@ -1,4 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
def sanitize_filename(filename: str) -> str:
|
||||
return filename.replace("/", "_").replace("..", "__").strip("'").strip('"')
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,629 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
from torch._higher_order_ops.auto_functionalize import auto_functionalized
|
||||
|
||||
from vllm.config import VllmConfig, get_layers_from_vllm_config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention.mla_attention import MLAAttention
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kFp8Dynamic64Sym,
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8StaticTensorSym,
|
||||
kNvfp4Dynamic,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.utils.torch_utils import _USE_LAYERNAME, _encode_layer_name
|
||||
|
||||
from ..vllm_inductor_pass import VllmFusionPatternMatcherPass, VllmPatternReplacement
|
||||
from .matcher_utils import MatcherQuantFP8
|
||||
from .rms_quant_fusion import QUANT_OPS
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
FP4_DTYPE = torch.uint8
|
||||
|
||||
MLA_ATTN_OP = torch.ops.vllm.unified_mla_attention_with_output.default
|
||||
|
||||
|
||||
class MLAAttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
|
||||
"""Fusion for MLA Attention+Fp8StaticQuant.
|
||||
|
||||
Matches the pattern: MLA attention -> static FP8 quant, and replaces
|
||||
it with MLA attention(output_scale=scale, output=fp8_buffer).
|
||||
"""
|
||||
|
||||
def __init__(self, layer: MLAAttention, dtype: torch.dtype) -> None:
|
||||
self._layer_name = layer.layer_name
|
||||
self._num_heads = layer.num_heads
|
||||
self._v_head_dim = layer.v_head_dim
|
||||
self._kv_lora_rank = layer.kv_lora_rank
|
||||
self._qk_rope_head_dim = layer.qk_rope_head_dim
|
||||
self._qk_head_dim = layer.qk_nope_head_dim + layer.qk_rope_head_dim
|
||||
self._output_dim = layer.num_heads * layer.v_head_dim
|
||||
self._dtype = dtype
|
||||
self._quant_matcher = MatcherQuantFP8(kFp8StaticTensorSym)
|
||||
|
||||
@property
|
||||
def pattern(self) -> Callable[..., torch.Tensor]:
|
||||
_ln = _encode_layer_name(self._layer_name)
|
||||
|
||||
if _USE_LAYERNAME:
|
||||
|
||||
def _pattern_with_ln( # type: ignore[misc]
|
||||
q,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
output_attn,
|
||||
scale,
|
||||
kv_cache_dummy_dep,
|
||||
layer_name,
|
||||
):
|
||||
at1 = auto_functionalized(
|
||||
MLA_ATTN_OP,
|
||||
q=q,
|
||||
kv_c_normed=kv_c_normed,
|
||||
k_pe=k_pe,
|
||||
output=output_attn,
|
||||
layer_name=layer_name,
|
||||
output_scale=None,
|
||||
output_block_scale=None,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
)
|
||||
# MLA output is already 2D (T, N*V), no reshape needed
|
||||
return self._quant_matcher(at1[1], scale)[0]
|
||||
|
||||
return _pattern_with_ln
|
||||
|
||||
def _pattern(q, kv_c_normed, k_pe, output_attn, scale, kv_cache_dummy_dep):
|
||||
at1 = auto_functionalized(
|
||||
MLA_ATTN_OP,
|
||||
q=q,
|
||||
kv_c_normed=kv_c_normed,
|
||||
k_pe=k_pe,
|
||||
output=output_attn,
|
||||
layer_name=_ln,
|
||||
output_scale=None,
|
||||
output_block_scale=None,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
)
|
||||
# MLA output is already 2D (T, N*V), no reshape needed
|
||||
return self._quant_matcher(at1[1], scale)[0]
|
||||
|
||||
return _pattern
|
||||
|
||||
@property
|
||||
def replacement(self) -> Callable[..., torch.Tensor]:
|
||||
_ln = _encode_layer_name(self._layer_name)
|
||||
|
||||
if _USE_LAYERNAME:
|
||||
|
||||
def _replacement_with_ln( # type: ignore[misc]
|
||||
q,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
output_attn,
|
||||
scale,
|
||||
kv_cache_dummy_dep,
|
||||
layer_name,
|
||||
):
|
||||
# MLA output in quant_dtype
|
||||
output_attn = torch.empty(
|
||||
[q.shape[0], self._output_dim],
|
||||
dtype=FP8_DTYPE,
|
||||
device=q.device,
|
||||
)
|
||||
at1 = auto_functionalized(
|
||||
MLA_ATTN_OP,
|
||||
q=q,
|
||||
kv_c_normed=kv_c_normed,
|
||||
k_pe=k_pe,
|
||||
output=output_attn,
|
||||
layer_name=layer_name,
|
||||
output_scale=scale,
|
||||
output_block_scale=None,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
)
|
||||
return at1[1]
|
||||
|
||||
return _replacement_with_ln
|
||||
|
||||
def _replacement(q, kv_c_normed, k_pe, output_attn, scale, kv_cache_dummy_dep):
|
||||
# MLA output in quant_dtype
|
||||
output_attn = torch.empty(
|
||||
[q.shape[0], self._output_dim],
|
||||
dtype=FP8_DTYPE,
|
||||
device=q.device,
|
||||
)
|
||||
at1 = auto_functionalized(
|
||||
MLA_ATTN_OP,
|
||||
q=q,
|
||||
kv_c_normed=kv_c_normed,
|
||||
k_pe=k_pe,
|
||||
output=output_attn,
|
||||
layer_name=_ln,
|
||||
output_scale=scale,
|
||||
output_block_scale=None,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
)
|
||||
return at1[1]
|
||||
|
||||
return _replacement
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
inputs: list = [
|
||||
self.empty(5, self._num_heads, self._qk_head_dim, dtype=self._dtype),
|
||||
self.empty(5, self._kv_lora_rank, dtype=self._dtype),
|
||||
self.empty(5, 1, self._qk_rope_head_dim, dtype=self._dtype),
|
||||
self.empty(5, self._output_dim, dtype=self._dtype),
|
||||
self.empty_fp32(1, 1),
|
||||
self.empty(0, dtype=self._dtype),
|
||||
]
|
||||
if _USE_LAYERNAME:
|
||||
inputs.append(_encode_layer_name(self._layer_name))
|
||||
return inputs
|
||||
|
||||
|
||||
class MLAAttnNvfp4QuantPattern(
|
||||
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor]]
|
||||
):
|
||||
"""Fusion for MLA Attention+Nvfp4Quant.
|
||||
|
||||
Matches the pattern: MLA attention -> NVFP4 quant, and replaces
|
||||
it with MLA attention(output_scale=scale, output_block_scale=block_scale,
|
||||
output=fp4_buffer).
|
||||
"""
|
||||
|
||||
def __init__(self, layer: MLAAttention, dtype: torch.dtype) -> None:
|
||||
self._layer_name = layer.layer_name
|
||||
self._num_heads = layer.num_heads
|
||||
self._v_head_dim = layer.v_head_dim
|
||||
self._kv_lora_rank = layer.kv_lora_rank
|
||||
self._qk_rope_head_dim = layer.qk_rope_head_dim
|
||||
self._qk_head_dim = layer.qk_nope_head_dim + layer.qk_rope_head_dim
|
||||
self._output_dim = layer.num_heads * layer.v_head_dim
|
||||
self._dtype = dtype
|
||||
self._QUANT_OP = QUANT_OPS[kNvfp4Dynamic]
|
||||
|
||||
@property
|
||||
def pattern(
|
||||
self,
|
||||
) -> Callable[..., tuple[torch.Tensor, torch.Tensor]]:
|
||||
_ln = _encode_layer_name(self._layer_name)
|
||||
|
||||
if _USE_LAYERNAME:
|
||||
|
||||
def _pattern_with_ln( # type: ignore[misc]
|
||||
q,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
output_attn,
|
||||
output_quant,
|
||||
output_scale,
|
||||
input_scale,
|
||||
kv_cache_dummy_dep,
|
||||
layer_name,
|
||||
):
|
||||
at1 = auto_functionalized(
|
||||
MLA_ATTN_OP,
|
||||
q=q,
|
||||
kv_c_normed=kv_c_normed,
|
||||
k_pe=k_pe,
|
||||
output=output_attn,
|
||||
layer_name=layer_name,
|
||||
output_scale=None,
|
||||
output_block_scale=None,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
)
|
||||
at2 = auto_functionalized(
|
||||
self._QUANT_OP,
|
||||
input=at1[1],
|
||||
input_scale=input_scale,
|
||||
is_sf_swizzled_layout=True,
|
||||
output=output_quant,
|
||||
output_scale=output_scale,
|
||||
)
|
||||
output_scale_view = torch.ops.aten.view.dtype(at2[2], FP8_DTYPE)
|
||||
return at2[1], output_scale_view
|
||||
|
||||
return _pattern_with_ln
|
||||
|
||||
def _pattern(
|
||||
q,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
output_attn,
|
||||
output_quant,
|
||||
output_scale,
|
||||
input_scale,
|
||||
kv_cache_dummy_dep,
|
||||
):
|
||||
at1 = auto_functionalized(
|
||||
MLA_ATTN_OP,
|
||||
q=q,
|
||||
kv_c_normed=kv_c_normed,
|
||||
k_pe=k_pe,
|
||||
output=output_attn,
|
||||
layer_name=_ln,
|
||||
output_scale=None,
|
||||
output_block_scale=None,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
)
|
||||
at2 = auto_functionalized(
|
||||
self._QUANT_OP,
|
||||
input=at1[1],
|
||||
input_scale=input_scale,
|
||||
is_sf_swizzled_layout=True,
|
||||
output=output_quant,
|
||||
output_scale=output_scale,
|
||||
)
|
||||
output_scale_view = torch.ops.aten.view.dtype(at2[2], FP8_DTYPE)
|
||||
return at2[1], output_scale_view
|
||||
|
||||
return _pattern
|
||||
|
||||
@property
|
||||
def replacement(
|
||||
self,
|
||||
) -> Callable[..., tuple[torch.Tensor, torch.Tensor]]:
|
||||
_ln = _encode_layer_name(self._layer_name)
|
||||
|
||||
if _USE_LAYERNAME:
|
||||
|
||||
def _replacement_with_ln( # type: ignore[misc]
|
||||
q,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
output_attn,
|
||||
_output_quant,
|
||||
output_scale,
|
||||
input_scale,
|
||||
kv_cache_dummy_dep,
|
||||
layer_name,
|
||||
):
|
||||
# MLA output in quant_dtype (FP4 packed as uint8)
|
||||
output_attn = torch.empty(
|
||||
[q.shape[0], self._output_dim // 2],
|
||||
dtype=FP4_DTYPE,
|
||||
device=q.device,
|
||||
)
|
||||
output_scale_view = torch.ops.aten.view.dtype(output_scale, FP8_DTYPE)
|
||||
at2 = auto_functionalized(
|
||||
MLA_ATTN_OP,
|
||||
q=q,
|
||||
kv_c_normed=kv_c_normed,
|
||||
k_pe=k_pe,
|
||||
output=output_attn,
|
||||
layer_name=layer_name,
|
||||
output_scale=input_scale,
|
||||
output_block_scale=output_scale_view,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
)
|
||||
return at2[1], at2[2]
|
||||
|
||||
return _replacement_with_ln
|
||||
|
||||
def _replacement(
|
||||
q,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
output_attn,
|
||||
_output_quant,
|
||||
output_scale,
|
||||
input_scale,
|
||||
kv_cache_dummy_dep,
|
||||
):
|
||||
# MLA output in quant_dtype (FP4 packed as uint8)
|
||||
output_attn = torch.empty(
|
||||
[q.shape[0], self._output_dim // 2],
|
||||
dtype=FP4_DTYPE,
|
||||
device=q.device,
|
||||
)
|
||||
output_scale_view = torch.ops.aten.view.dtype(output_scale, FP8_DTYPE)
|
||||
at2 = auto_functionalized(
|
||||
MLA_ATTN_OP,
|
||||
q=q,
|
||||
kv_c_normed=kv_c_normed,
|
||||
k_pe=k_pe,
|
||||
output=output_attn,
|
||||
layer_name=_ln,
|
||||
output_scale=input_scale,
|
||||
output_block_scale=output_scale_view,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
)
|
||||
return at2[1], at2[2]
|
||||
|
||||
return _replacement
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
inputs: list = [
|
||||
self.empty(5, self._num_heads, self._qk_head_dim, dtype=self._dtype),
|
||||
self.empty(5, self._kv_lora_rank, dtype=self._dtype),
|
||||
self.empty(5, 1, self._qk_rope_head_dim, dtype=self._dtype),
|
||||
self.empty(5, self._output_dim, dtype=self._dtype),
|
||||
self.empty(5, self._output_dim // 2, dtype=FP4_DTYPE),
|
||||
self.empty_i32(128, round_up(self._output_dim // 16, 4)),
|
||||
self.empty_fp32(1, 1),
|
||||
self.empty(0, dtype=self._dtype),
|
||||
]
|
||||
if _USE_LAYERNAME:
|
||||
inputs.append(_encode_layer_name(self._layer_name))
|
||||
return inputs
|
||||
|
||||
|
||||
class MLAAttnFp8GroupQuantPattern(
|
||||
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor]]
|
||||
):
|
||||
"""Fusion for MLA Attention+Fp8GroupQuant (per-group dynamic FP8).
|
||||
|
||||
Matches the pattern: MLA attention -> per_token_group_fp8_quant, and
|
||||
replaces it with MLA attention(output_block_scale=group_scale_buffer).
|
||||
Used by models with block FP8 quantization (e.g. DeepSeek V3).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
layer: MLAAttention,
|
||||
dtype: torch.dtype,
|
||||
quant_key: QuantKey,
|
||||
has_col_major_scales: bool,
|
||||
is_e8m0: bool,
|
||||
is_tma_aligned: bool,
|
||||
) -> None:
|
||||
self._layer_name = layer.layer_name
|
||||
self._num_heads = layer.num_heads
|
||||
self._v_head_dim = layer.v_head_dim
|
||||
self._kv_lora_rank = layer.kv_lora_rank
|
||||
self._qk_rope_head_dim = layer.qk_rope_head_dim
|
||||
self._qk_head_dim = layer.qk_nope_head_dim + layer.qk_rope_head_dim
|
||||
self._output_dim = layer.num_heads * layer.v_head_dim
|
||||
self._dtype = dtype
|
||||
self._layer = layer
|
||||
self._group_size = quant_key.scale.group_shape[1]
|
||||
self._has_col_major_scales = has_col_major_scales
|
||||
self._is_e8m0 = is_e8m0
|
||||
self._is_tma_aligned = is_tma_aligned
|
||||
|
||||
self._quant_matcher = MatcherQuantFP8(
|
||||
quant_key,
|
||||
has_col_major_scales=has_col_major_scales,
|
||||
is_e8m0=is_e8m0,
|
||||
is_tma_aligned=is_tma_aligned,
|
||||
)
|
||||
|
||||
@property
|
||||
def pattern(
|
||||
self,
|
||||
) -> Callable[..., tuple[torch.Tensor, torch.Tensor]]:
|
||||
_ln = _encode_layer_name(self._layer_name)
|
||||
|
||||
if _USE_LAYERNAME:
|
||||
|
||||
def _pattern_with_ln( # type: ignore[misc]
|
||||
q,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
output_attn,
|
||||
kv_cache_dummy_dep,
|
||||
scale,
|
||||
layer_name,
|
||||
):
|
||||
at1 = auto_functionalized(
|
||||
MLA_ATTN_OP,
|
||||
q=q,
|
||||
kv_c_normed=kv_c_normed,
|
||||
k_pe=k_pe,
|
||||
output=output_attn,
|
||||
layer_name=layer_name,
|
||||
output_scale=None,
|
||||
output_block_scale=None,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
)
|
||||
attn_out = at1[1]
|
||||
result = torch.empty(
|
||||
attn_out.shape, device=attn_out.device, dtype=FP8_DTYPE
|
||||
)
|
||||
finfo = torch.finfo(FP8_DTYPE)
|
||||
_, result, scale = auto_functionalized(
|
||||
self._quant_matcher.QUANT_OP,
|
||||
input=attn_out,
|
||||
output_q=result,
|
||||
output_s=scale,
|
||||
group_size=self._group_size,
|
||||
eps=1e-10,
|
||||
fp8_min=finfo.min,
|
||||
fp8_max=finfo.max,
|
||||
scale_ue8m0=self._is_e8m0,
|
||||
dummy_is_scale_transposed=self._has_col_major_scales,
|
||||
dummy_is_tma_aligned=self._is_tma_aligned,
|
||||
)
|
||||
return result, scale
|
||||
|
||||
return _pattern_with_ln
|
||||
|
||||
def _pattern(
|
||||
q: torch.Tensor,
|
||||
kv_c_normed: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
output_attn: torch.Tensor,
|
||||
kv_cache_dummy_dep: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
at1 = auto_functionalized(
|
||||
MLA_ATTN_OP,
|
||||
q=q,
|
||||
kv_c_normed=kv_c_normed,
|
||||
k_pe=k_pe,
|
||||
output=output_attn,
|
||||
layer_name=_ln,
|
||||
output_scale=None,
|
||||
output_block_scale=None,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
)
|
||||
attn_out = at1[1]
|
||||
result = torch.empty(
|
||||
attn_out.shape, device=attn_out.device, dtype=FP8_DTYPE
|
||||
)
|
||||
finfo = torch.finfo(FP8_DTYPE)
|
||||
_, result, scale = auto_functionalized(
|
||||
self._quant_matcher.QUANT_OP,
|
||||
input=attn_out,
|
||||
output_q=result,
|
||||
output_s=scale,
|
||||
group_size=self._group_size,
|
||||
eps=1e-10,
|
||||
fp8_min=finfo.min,
|
||||
fp8_max=finfo.max,
|
||||
scale_ue8m0=self._is_e8m0,
|
||||
dummy_is_scale_transposed=self._has_col_major_scales,
|
||||
dummy_is_tma_aligned=self._is_tma_aligned,
|
||||
)
|
||||
return result, scale
|
||||
|
||||
return _pattern
|
||||
|
||||
@property
|
||||
def replacement(
|
||||
self,
|
||||
) -> Callable[..., tuple[torch.Tensor, torch.Tensor]]:
|
||||
_ln = _encode_layer_name(self._layer_name)
|
||||
|
||||
if _USE_LAYERNAME:
|
||||
|
||||
def _replacement_with_ln( # type: ignore[misc]
|
||||
q,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
output_attn,
|
||||
kv_cache_dummy_dep,
|
||||
scale,
|
||||
layer_name,
|
||||
):
|
||||
output_attn = torch.empty(
|
||||
[q.shape[0], self._output_dim],
|
||||
dtype=FP8_DTYPE,
|
||||
device=q.device,
|
||||
)
|
||||
at1 = auto_functionalized(
|
||||
MLA_ATTN_OP,
|
||||
q=q,
|
||||
kv_c_normed=kv_c_normed,
|
||||
k_pe=k_pe,
|
||||
output=output_attn,
|
||||
layer_name=layer_name,
|
||||
output_scale=None,
|
||||
output_block_scale=scale,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
quant_group_size=self._group_size,
|
||||
quant_scale_ue8m0=self._is_e8m0,
|
||||
quant_col_major=self._has_col_major_scales,
|
||||
quant_tma_aligned=self._is_tma_aligned,
|
||||
)
|
||||
return at1[1], at1[2]
|
||||
|
||||
return _replacement_with_ln
|
||||
|
||||
def _replacement(q, kv_c_normed, k_pe, output_attn, kv_cache_dummy_dep, scale):
|
||||
output_attn = torch.empty(
|
||||
[q.shape[0], self._output_dim],
|
||||
dtype=FP8_DTYPE,
|
||||
device=q.device,
|
||||
)
|
||||
at1 = auto_functionalized(
|
||||
MLA_ATTN_OP,
|
||||
q=q,
|
||||
kv_c_normed=kv_c_normed,
|
||||
k_pe=k_pe,
|
||||
output=output_attn,
|
||||
layer_name=_ln,
|
||||
output_scale=None,
|
||||
output_block_scale=scale,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
quant_group_size=self._group_size,
|
||||
quant_scale_ue8m0=self._is_e8m0,
|
||||
quant_col_major=self._has_col_major_scales,
|
||||
quant_tma_aligned=self._is_tma_aligned,
|
||||
)
|
||||
return at1[1], at1[2]
|
||||
|
||||
return _replacement
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
inputs: list = [
|
||||
self.empty(5, self._num_heads, self._qk_head_dim, dtype=self._dtype),
|
||||
self.empty(5, self._kv_lora_rank, dtype=self._dtype),
|
||||
self.empty(5, 1, self._qk_rope_head_dim, dtype=self._dtype),
|
||||
self.empty(5, self._output_dim, dtype=self._dtype),
|
||||
self.empty(0, dtype=self._dtype),
|
||||
self._quant_matcher.empty_f32(1, 1),
|
||||
]
|
||||
if _USE_LAYERNAME:
|
||||
inputs.append(_encode_layer_name(self._layer_name))
|
||||
return inputs
|
||||
|
||||
|
||||
class MLAAttnQuantFusionPass(VllmFusionPatternMatcherPass):
|
||||
"""This pass fuses post-attention quantization onto MLA attention if supported.
|
||||
|
||||
It uses the pattern matcher and matches each MLA layer manually, as strings
|
||||
cannot be wildcarded. This also lets us check support on attention layers
|
||||
upon registration instead of during pattern matching.
|
||||
"""
|
||||
|
||||
def __init__(self, config: VllmConfig) -> None:
|
||||
super().__init__(config, "mla_attn_quant_fusion")
|
||||
|
||||
dtype = config.model_config.dtype
|
||||
layers = list(get_layers_from_vllm_config(config, MLAAttention).values())
|
||||
|
||||
if len(layers) == 0:
|
||||
logger.warning(
|
||||
"MLA attention + quant fusion is enabled, but no MLA "
|
||||
"attention layers were found in "
|
||||
"CompilationConfig.static_forward_context "
|
||||
"so no fusion patterns were registered."
|
||||
)
|
||||
|
||||
# When _USE_LAYERNAME is enabled, layer_name is a wildcard so all
|
||||
# layers produce the same pattern — register once then break.
|
||||
for layer in layers:
|
||||
if layer.impl.fused_output_quant_supported(kFp8StaticTensorSym):
|
||||
self.register(MLAAttnFp8StaticQuantPattern(layer, dtype))
|
||||
if _USE_LAYERNAME:
|
||||
break
|
||||
|
||||
if current_platform.is_cuda() and hasattr(torch.ops._C, "scaled_fp4_quant"):
|
||||
for layer in layers:
|
||||
if layer.impl.fused_output_quant_supported(kNvfp4Dynamic):
|
||||
self.register(MLAAttnNvfp4QuantPattern(layer, dtype))
|
||||
if _USE_LAYERNAME:
|
||||
break
|
||||
|
||||
# Per-group FP8 (block quant) — register all flag combinations.
|
||||
if current_platform.is_cuda():
|
||||
for quant_key in [kFp8Dynamic128Sym, kFp8Dynamic64Sym]:
|
||||
for col_major in [True, False]:
|
||||
for is_e8m0 in [True, False]:
|
||||
for tma_aligned in [False, True]:
|
||||
for layer in layers:
|
||||
if layer.impl.fused_output_quant_supported(quant_key):
|
||||
self.register(
|
||||
MLAAttnFp8GroupQuantPattern(
|
||||
layer,
|
||||
dtype,
|
||||
quant_key,
|
||||
col_major,
|
||||
is_e8m0,
|
||||
tma_aligned,
|
||||
)
|
||||
)
|
||||
if _USE_LAYERNAME:
|
||||
break
|
||||
|
||||
self.dump_patterns(config, self.pm_pass)
|
||||
@@ -1,274 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
from torch._higher_order_ops.auto_functionalize import auto_functionalized
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
from vllm.config import VllmConfig, get_layers_from_vllm_config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import MLAAttention
|
||||
from vllm.model_executor.layers.attention.attention import get_attention_context
|
||||
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
|
||||
from vllm.utils.torch_utils import (
|
||||
_USE_LAYERNAME,
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
_resolve_layer_name,
|
||||
direct_register_custom_op,
|
||||
)
|
||||
|
||||
from ..vllm_inductor_pass import VllmFusionPatternMatcherPass, VllmPatternReplacement
|
||||
from .matcher_utils import MatcherDeepseekScalingRotaryEmbedding, MatcherRotaryEmbedding
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def fused_rope_unified_mla_kv_cache_update_impl(
|
||||
positions: torch.Tensor,
|
||||
q_pe: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
kv_c: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
is_neox: bool,
|
||||
kv_cache_dtype: str,
|
||||
kv_cache_scale: torch.Tensor,
|
||||
layer_name: LayerNameType,
|
||||
) -> torch.Tensor:
|
||||
layer_name = _resolve_layer_name(layer_name)
|
||||
attn_metadata, attn_layer, kv_cache, layer_slot_mapping = get_attention_context(
|
||||
layer_name
|
||||
)
|
||||
attn_layer.prepare_kv_cache_update(attn_metadata)
|
||||
if layer_slot_mapping is not None:
|
||||
ops.concat_and_cache_mla_rope_fused(
|
||||
positions,
|
||||
q_pe,
|
||||
k_pe,
|
||||
kv_c,
|
||||
cos_sin_cache,
|
||||
is_neox,
|
||||
layer_slot_mapping,
|
||||
kv_cache.squeeze(1),
|
||||
kv_cache_dtype,
|
||||
kv_cache_scale,
|
||||
)
|
||||
return torch.empty(0, device=kv_c.device, dtype=kv_c.dtype)
|
||||
|
||||
|
||||
def fused_rope_unified_mla_kv_cache_update_fake(
|
||||
positions: torch.Tensor,
|
||||
q_pe: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
kv_c: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
is_neox: bool,
|
||||
kv_cache_dtype: str,
|
||||
kv_cache_scale: torch.Tensor,
|
||||
layer_name: LayerNameType,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty(0, dtype=kv_c.dtype, device=kv_c.device)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="fused_rope_unified_mla_kv_cache_update",
|
||||
op_func=fused_rope_unified_mla_kv_cache_update_impl,
|
||||
fake_impl=fused_rope_unified_mla_kv_cache_update_fake,
|
||||
mutates_args=["q_pe", "k_pe"],
|
||||
)
|
||||
|
||||
|
||||
class MLARoPEKVCacheCatPattern(VllmPatternReplacement):
|
||||
FUSED_OP = torch.ops.vllm.fused_rope_unified_mla_kv_cache_update.default
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
layer: MLAAttention,
|
||||
is_neox: bool,
|
||||
use_flashinfer: bool = False,
|
||||
use_deepseek_scaling: bool = False,
|
||||
) -> None:
|
||||
self.layer_name = layer.layer_name
|
||||
self.kv_cache_dtype = layer.kv_cache_dtype
|
||||
self.num_heads = layer.num_heads
|
||||
self.num_kv_heads = layer.num_kv_heads
|
||||
self.kv_lora_rank = layer.kv_lora_rank
|
||||
self.qk_rope_head_dim = layer.qk_rope_head_dim
|
||||
self.is_neox = is_neox
|
||||
self.use_flashinfer = use_flashinfer
|
||||
self._ln = _encode_layer_name(self.layer_name)
|
||||
|
||||
if use_deepseek_scaling:
|
||||
self.rope_matcher = MatcherDeepseekScalingRotaryEmbedding(
|
||||
is_neox=self.is_neox,
|
||||
head_size=self.qk_rope_head_dim,
|
||||
num_heads=self.num_heads,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
use_flashinfer=self.use_flashinfer,
|
||||
)
|
||||
else:
|
||||
self.rope_matcher = MatcherRotaryEmbedding( # type: ignore
|
||||
is_neox=self.is_neox,
|
||||
head_size=self.qk_rope_head_dim,
|
||||
num_heads=self.num_heads,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
use_flashinfer=self.use_flashinfer,
|
||||
)
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
T = 5
|
||||
L = 4096
|
||||
q_pe = self.empty_bf16(T, self.num_heads, self.qk_rope_head_dim)
|
||||
k_pe = self.empty_bf16(T, self.qk_rope_head_dim)
|
||||
kv_c_normed = self.empty_bf16(T, self.kv_lora_rank)
|
||||
cos_sin_cache = self.empty_bf16(L, self.qk_rope_head_dim)
|
||||
positions = self.empty(T, dtype=torch.int64)
|
||||
k_scale = self.empty(0, dtype=torch.float32)
|
||||
inputs = [
|
||||
q_pe,
|
||||
k_pe,
|
||||
kv_c_normed,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
k_scale,
|
||||
]
|
||||
if _USE_LAYERNAME:
|
||||
inputs.append(self._ln)
|
||||
return inputs
|
||||
|
||||
@property
|
||||
def pattern(self):
|
||||
_ln = self._ln
|
||||
|
||||
if _USE_LAYERNAME:
|
||||
|
||||
def _pattern_with_ln(
|
||||
q_pe: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
kv_c_normed: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
k_scale: torch.Tensor,
|
||||
layer_name: LayerNameType,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
k_pe_unsqueezed = k_pe.unsqueeze(1)
|
||||
q_pe, k_pe = self.rope_matcher(
|
||||
positions, q_pe, k_pe_unsqueezed, cos_sin_cache
|
||||
)
|
||||
dummy = torch.ops.vllm.unified_mla_kv_cache_update(
|
||||
kv_c_normed, k_pe, layer_name, self.kv_cache_dtype, k_scale
|
||||
)
|
||||
return dummy, q_pe, k_pe
|
||||
|
||||
return _pattern_with_ln
|
||||
|
||||
def _pattern(
|
||||
q_pe: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
kv_c_normed: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
k_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
k_pe_unsqueezed = k_pe.unsqueeze(1)
|
||||
q_pe, k_pe = self.rope_matcher(
|
||||
positions, q_pe, k_pe_unsqueezed, cos_sin_cache
|
||||
)
|
||||
dummy = torch.ops.vllm.unified_mla_kv_cache_update(
|
||||
kv_c_normed, k_pe, _ln, self.kv_cache_dtype, k_scale
|
||||
)
|
||||
return dummy, q_pe, k_pe
|
||||
|
||||
return _pattern
|
||||
|
||||
@property
|
||||
def replacement(self):
|
||||
_ln = self._ln
|
||||
|
||||
if _USE_LAYERNAME:
|
||||
|
||||
def _replacement_with_ln(
|
||||
q_pe: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
kv_c_normed: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
k_scale: torch.Tensor,
|
||||
layer_name: LayerNameType,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
at = auto_functionalized(
|
||||
self.FUSED_OP,
|
||||
positions=positions,
|
||||
q_pe=q_pe,
|
||||
k_pe=k_pe,
|
||||
kv_c=kv_c_normed,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
is_neox=self.is_neox,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
kv_cache_scale=k_scale,
|
||||
layer_name=layer_name,
|
||||
)
|
||||
dummy, q_pe, k_pe_squeezed = at
|
||||
k_pe = k_pe_squeezed.unsqueeze(1)
|
||||
return dummy, q_pe, k_pe
|
||||
|
||||
return _replacement_with_ln
|
||||
|
||||
def _replacement(
|
||||
q_pe: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
kv_c_normed: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
k_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
at = auto_functionalized(
|
||||
self.FUSED_OP,
|
||||
positions=positions,
|
||||
q_pe=q_pe,
|
||||
k_pe=k_pe,
|
||||
kv_c=kv_c_normed,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
is_neox=self.is_neox,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
kv_cache_scale=k_scale,
|
||||
layer_name=_ln,
|
||||
)
|
||||
dummy, q_pe, k_pe_squeezed = at
|
||||
k_pe = k_pe_squeezed.unsqueeze(1)
|
||||
return dummy, q_pe, k_pe
|
||||
|
||||
return _replacement
|
||||
|
||||
|
||||
class MLARoPEKVCacheCatFusionPass(VllmFusionPatternMatcherPass):
|
||||
def __init__(self, config: VllmConfig) -> None:
|
||||
super().__init__(config, "mla_rope_kv_cache_fusion_pass")
|
||||
|
||||
attn_layers = get_layers_from_vllm_config(config, MLAAttention)
|
||||
|
||||
for _, layer in attn_layers.items():
|
||||
for is_neox in [False, True]:
|
||||
for use_deepseek_scaling in [False, True]:
|
||||
if RotaryEmbedding.enabled():
|
||||
for use_flashinfer in [False, True]:
|
||||
self.register(
|
||||
MLARoPEKVCacheCatPattern(
|
||||
layer,
|
||||
is_neox,
|
||||
use_flashinfer,
|
||||
use_deepseek_scaling,
|
||||
)
|
||||
)
|
||||
else:
|
||||
self.register(
|
||||
MLARoPEKVCacheCatPattern(
|
||||
layer,
|
||||
is_neox,
|
||||
use_deepseek_scaling=use_deepseek_scaling,
|
||||
)
|
||||
)
|
||||
|
||||
if _USE_LAYERNAME:
|
||||
break
|
||||
|
||||
self.dump_patterns(config, self.pm_pass)
|
||||
@@ -40,8 +40,6 @@ if current_platform.is_cuda_alike() or current_platform.is_xpu():
|
||||
if current_platform.is_cuda_alike():
|
||||
from .fusion.act_quant_fusion import ActivationQuantFusionPass
|
||||
from .fusion.attn_quant_fusion import AttnQuantFusionPass
|
||||
from .fusion.mla_attn_quant_fusion import MLAAttnQuantFusionPass
|
||||
from .fusion.mla_rope_kvcache_cat_fusion import MLARoPEKVCacheCatFusionPass
|
||||
from .fusion.qk_norm_rope_kvcache_fusion import QkNormRopeKvCacheFusionPass
|
||||
from .fusion.rms_quant_fusion import RMSNormQuantFusionPass
|
||||
from .fusion.rope_kvcache_fusion import RopeKVCacheFusionPass
|
||||
@@ -206,12 +204,8 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
|
||||
self.passes += [ScatterSplitReplacementPass(config)]
|
||||
self.passes += [RopeKVCacheFusionPass(config)]
|
||||
|
||||
if self.pass_config.fuse_rope_kvcache_cat_mla:
|
||||
self.passes += [MLARoPEKVCacheCatFusionPass(config)]
|
||||
|
||||
if self.pass_config.fuse_attn_quant:
|
||||
self.passes += [AttnQuantFusionPass(config)]
|
||||
self.passes += [MLAAttnQuantFusionPass(config)]
|
||||
|
||||
if self.pass_config.enable_qk_norm_rope_fusion:
|
||||
self.passes += [SplitCoalescingPass(config)]
|
||||
|
||||
+5
-23
@@ -1072,30 +1072,12 @@ class ModelConfig:
|
||||
self, mm_device_do_normalize: bool | None
|
||||
) -> bool:
|
||||
if mm_device_do_normalize is None:
|
||||
if envs.VLLM_USE_RUST_FRONTEND:
|
||||
logger.debug(
|
||||
"VLLM_USE_RUST_FRONTEND is set. "
|
||||
"Rust frontend does not currently support mm_device_do_normalize, "
|
||||
"forcing mm_device_do_normalize = False."
|
||||
)
|
||||
mm_device_do_normalize = False
|
||||
else:
|
||||
mm_device_do_normalize = (
|
||||
self._model_info.supports_mm_device_do_normalize
|
||||
)
|
||||
logger.debug(
|
||||
"mm_device_do_normalize is %s by default.",
|
||||
"enabled" if mm_device_do_normalize else "disabled",
|
||||
)
|
||||
mm_device_do_normalize = self._model_info.supports_mm_device_do_normalize
|
||||
logger.debug(
|
||||
"mm_device_do_normalize is %s by default.",
|
||||
"enabled" if mm_device_do_normalize else "disabled",
|
||||
)
|
||||
else:
|
||||
if mm_device_do_normalize and envs.VLLM_USE_RUST_FRONTEND:
|
||||
logger.warning(
|
||||
"VLLM_USE_RUST_FRONTEND is set. "
|
||||
"Rust frontend does not currently support mm_device_do_normalize, "
|
||||
"forcing mm_device_do_normalize = False."
|
||||
)
|
||||
mm_device_do_normalize = False
|
||||
|
||||
if (
|
||||
mm_device_do_normalize
|
||||
and not self._model_info.supports_mm_device_do_normalize
|
||||
|
||||
@@ -1,457 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import safetensors
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorBase_V1,
|
||||
KVConnectorMetadata,
|
||||
KVConnectorRole,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention.mla_attention import MLACommonMetadata
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.hashing import safe_hash
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.forward_context import ForwardContext
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReqMeta:
|
||||
# Request tokens
|
||||
token_ids: torch.Tensor
|
||||
# Slot mappings, should have the same length as token_ids
|
||||
slot_mapping: torch.Tensor
|
||||
# Is store or load
|
||||
is_store: bool
|
||||
mm_hashes: list[str]
|
||||
|
||||
@staticmethod
|
||||
def make_meta(
|
||||
token_ids: list[int],
|
||||
block_ids: list[int],
|
||||
block_size: int,
|
||||
is_store: bool,
|
||||
mm_hashes: list[str],
|
||||
) -> "ReqMeta":
|
||||
valid_num_tokens = align_to_block_size(len(token_ids), block_size)
|
||||
token_ids_tensor = torch.tensor(token_ids)[:valid_num_tokens]
|
||||
block_ids_tensor = torch.tensor(block_ids)
|
||||
num_blocks = block_ids_tensor.shape[0]
|
||||
block_offsets = torch.arange(0, block_size)
|
||||
slot_mapping = (
|
||||
block_offsets.reshape((1, block_size))
|
||||
+ block_ids_tensor.reshape((num_blocks, 1)) * block_size
|
||||
)
|
||||
slot_mapping = slot_mapping.flatten()[:valid_num_tokens]
|
||||
return ReqMeta(
|
||||
token_ids=token_ids_tensor,
|
||||
slot_mapping=slot_mapping,
|
||||
is_store=is_store,
|
||||
mm_hashes=mm_hashes,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExampleConnectorMetadata(KVConnectorMetadata):
|
||||
requests: list[ReqMeta] = field(default_factory=list)
|
||||
|
||||
def add_request(
|
||||
self,
|
||||
token_ids: list[int],
|
||||
block_ids: list[int],
|
||||
block_size: int,
|
||||
is_store: bool,
|
||||
mm_hashes: list[str],
|
||||
) -> None:
|
||||
self.requests.append(
|
||||
ReqMeta.make_meta(token_ids, block_ids, block_size, is_store, mm_hashes)
|
||||
)
|
||||
|
||||
|
||||
class ExampleConnector(KVConnectorBase_V1):
|
||||
# NOTE: This is Simple debug implementation of the KV connector.
|
||||
# It save / load the KV cache to / from the disk.
|
||||
# It does extra work which will overwrite the existing prefix-cache in GPU
|
||||
# - to remove the overhead, need to add some "mask" in the ReqMeta class
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: "VllmConfig",
|
||||
role: KVConnectorRole,
|
||||
kv_cache_config: "KVCacheConfig",
|
||||
):
|
||||
super().__init__(
|
||||
vllm_config=vllm_config,
|
||||
role=role,
|
||||
kv_cache_config=kv_cache_config,
|
||||
)
|
||||
self._block_size = vllm_config.cache_config.block_size
|
||||
self._requests_need_load: dict[str, Request] = {}
|
||||
self._storage_path = self._kv_transfer_config.get_from_extra_config(
|
||||
"shared_storage_path", "/tmp"
|
||||
)
|
||||
logger.info(self._kv_transfer_config)
|
||||
logger.info("Shared storage path is %s", self._storage_path)
|
||||
|
||||
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
|
||||
"""Start loading the KV cache from the connector buffer to vLLM's
|
||||
paged KV buffer.
|
||||
|
||||
Args:
|
||||
forward_context (ForwardContext): the forward context.
|
||||
**kwargs: additional arguments for the load operation
|
||||
|
||||
Note:
|
||||
The number of elements in kv_caches and layer_names should be
|
||||
the same.
|
||||
|
||||
"""
|
||||
|
||||
def inject_kv_into_layer(
|
||||
dst_kv_cache_layer: torch.Tensor,
|
||||
src_kv_cache: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata,
|
||||
) -> None:
|
||||
"""Inject the KV cache into the layer.
|
||||
|
||||
Args:
|
||||
dst_kv_cache_layer (torch.Tensor): the destination KV cache layer,
|
||||
a standardized [B, H, N, C] per-layer view (H == 1 for MLA).
|
||||
src_kv_cache (torch.Tensor): the source KV cache.
|
||||
slot_mapping (torch.Tensor): the slot mapping. In shape
|
||||
[num_tokens].
|
||||
attn_metadata (AttentionMetadata): attention metadata for the
|
||||
current forward pass.
|
||||
|
||||
"""
|
||||
# Deliberate bulk transfers in this reference connector.
|
||||
with gpu_sync_allowed():
|
||||
slot_mapping = slot_mapping.to(
|
||||
dst_kv_cache_layer.device, non_blocking=True
|
||||
)
|
||||
if isinstance(attn_metadata, MLACommonMetadata):
|
||||
# [B, 1, N, C] -> [B * N, C]; slot_mapping indexes B * N slots.
|
||||
dst_kv_cache_layer = dst_kv_cache_layer.reshape(
|
||||
-1, dst_kv_cache_layer.shape[-1]
|
||||
)
|
||||
dst_kv_cache_layer[slot_mapping, ...] = src_kv_cache
|
||||
else:
|
||||
block_idxs = slot_mapping // self._block_size
|
||||
offsets = slot_mapping % self._block_size
|
||||
dst_kv_cache_layer[block_idxs, :, offsets] = src_kv_cache
|
||||
|
||||
# Get the metadata
|
||||
metadata: KVConnectorMetadata = self._get_connector_metadata()
|
||||
assert isinstance(metadata, ExampleConnectorMetadata)
|
||||
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
if attn_metadata is None:
|
||||
logger.warning("In connector.start_load_kv, but the attn_metadata is None")
|
||||
return
|
||||
|
||||
# Load the KV for each request each layer
|
||||
for request in metadata.requests:
|
||||
if request.is_store:
|
||||
continue
|
||||
logger.info(
|
||||
"Inject KV cache of %d tokens to the paged memory",
|
||||
len(request.slot_mapping),
|
||||
)
|
||||
for layer_name in forward_context.no_compile_layers:
|
||||
layer = forward_context.no_compile_layers[layer_name]
|
||||
|
||||
# Only process layers that have kv_cache
|
||||
# attribute (attention layers) Skip non-attention
|
||||
# layers like FusedMoEFactory/MLP etc.
|
||||
kv_cache_layer = getattr(layer, "kv_cache", None)
|
||||
if kv_cache_layer is None:
|
||||
continue
|
||||
|
||||
filename = self._generate_filename_debug(
|
||||
layer_name, request.token_ids, request.mm_hashes
|
||||
)
|
||||
# Deliberate bulk load in this reference connector.
|
||||
with gpu_sync_allowed():
|
||||
kv_cache_cpu = safetensors.torch.load_file(filename)["kv_cache"]
|
||||
kv_cache = kv_cache_cpu.to(kv_cache_layer.device, non_blocking=True)
|
||||
if isinstance(attn_metadata, dict):
|
||||
inject_kv_into_layer(
|
||||
kv_cache_layer,
|
||||
kv_cache,
|
||||
request.slot_mapping,
|
||||
attn_metadata[layer_name],
|
||||
)
|
||||
|
||||
def wait_for_layer_load(self, layer_name: str) -> None:
|
||||
"""Blocking until the KV for a specific layer is loaded into vLLM's
|
||||
paged buffer.
|
||||
|
||||
This interface will be useful for layer-by-layer pipelining.
|
||||
|
||||
Args:
|
||||
layer_name: the name of that layer
|
||||
|
||||
"""
|
||||
return
|
||||
|
||||
def save_kv_layer(
|
||||
self,
|
||||
layer_name: str,
|
||||
kv_layer: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""Start saving the KV cache of the layer from vLLM's paged buffer
|
||||
to the connector.
|
||||
|
||||
Args:
|
||||
layer_name (str): the name of the layer.
|
||||
kv_layer (torch.Tensor): the paged KV buffer of the current
|
||||
layer in vLLM.
|
||||
attn_metadata (AttentionMetadata): the attention metadata.
|
||||
**kwargs: additional arguments for the save operation.
|
||||
|
||||
"""
|
||||
|
||||
def extract_kv_from_layer(
|
||||
layer: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Extract the KV cache from the layer.
|
||||
|
||||
The layer is a standardized [B, H, N, C] per-layer view (H == 1 for MLA).
|
||||
"""
|
||||
# Deliberate bulk transfer in this reference connector.
|
||||
with gpu_sync_allowed():
|
||||
slot_mapping = slot_mapping.to(layer.device, non_blocking=True)
|
||||
if isinstance(attn_metadata, MLACommonMetadata):
|
||||
# [B, 1, N, C] -> [B * N, C]; slot_mapping indexes B * N slots.
|
||||
return layer.reshape(-1, layer.shape[-1])[slot_mapping, ...]
|
||||
block_idxs = slot_mapping // self._block_size
|
||||
offsets = slot_mapping % self._block_size
|
||||
return layer[block_idxs, :, offsets]
|
||||
|
||||
connector_metadata = self._get_connector_metadata()
|
||||
assert isinstance(connector_metadata, ExampleConnectorMetadata)
|
||||
for request in connector_metadata.requests:
|
||||
if request.is_store:
|
||||
filename = self._generate_filename_debug(
|
||||
layer_name, request.token_ids, request.mm_hashes
|
||||
)
|
||||
kv_cache = extract_kv_from_layer(kv_layer, request.slot_mapping)
|
||||
with gpu_sync_allowed():
|
||||
tensors = {"kv_cache": kv_cache.detach().cpu()}
|
||||
safetensors.torch.save_file(tensors, filename)
|
||||
|
||||
def wait_for_save(self):
|
||||
return
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self,
|
||||
request: "Request",
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[int | None, bool]:
|
||||
"""Get number of new tokens that can be loaded from the
|
||||
external KV cache beyond the num_computed_tokens.
|
||||
|
||||
Args:
|
||||
request (Request): the request object.
|
||||
num_computed_tokens (int): the number of locally
|
||||
computed tokens for this request
|
||||
|
||||
Returns:
|
||||
the number of tokens that can be loaded from the
|
||||
external KV cache beyond what is already computed.
|
||||
|
||||
"""
|
||||
# NOTE: in this debug implementation, we assume that the prompt is
|
||||
# cached_prompt + newly_generated_single_token
|
||||
# Therefore, we use prompt_token_ids[:-1] to determine the folder name
|
||||
|
||||
# NOTE: in current v1 scheduler, the num_computed_tokens is aligned
|
||||
# with the block granularity. And it expects the returned blocks and
|
||||
# num_computed_tokens to also be aligned with the block granularity.
|
||||
if not self._found_match_for_request(request):
|
||||
return 0, False
|
||||
|
||||
logger.info("External Cache Hit!")
|
||||
|
||||
# Now, first num_tokens_to_check tokens are hit, we need to prepare
|
||||
# the metadata for the worker connector to correctly load the KV
|
||||
token_ids = request.prompt_token_ids or []
|
||||
num_tokens_to_check = align_to_block_size(len(token_ids) - 1, self._block_size)
|
||||
|
||||
return num_tokens_to_check - num_computed_tokens, False
|
||||
|
||||
def update_state_after_alloc(
|
||||
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
|
||||
):
|
||||
"""Update KVConnector state after block allocation.
|
||||
|
||||
If blocks were allocated, add to _requests_need_load,
|
||||
such that we load the KVs in the next forward pass.
|
||||
"""
|
||||
if num_external_tokens > 0:
|
||||
self._requests_need_load[request.request_id] = request
|
||||
|
||||
def build_connector_meta(
|
||||
self,
|
||||
scheduler_output: SchedulerOutput,
|
||||
) -> KVConnectorMetadata:
|
||||
"""Build the connector metadata for this step.
|
||||
|
||||
This function should NOT modify any fields in the scheduler_output.
|
||||
Also, calling this function will reset the state of the connector.
|
||||
|
||||
Args:
|
||||
scheduler_output (SchedulerOutput): the scheduler output object.
|
||||
|
||||
"""
|
||||
meta = ExampleConnectorMetadata()
|
||||
|
||||
total_need_load = 0
|
||||
for new_req in scheduler_output.scheduled_new_reqs:
|
||||
token_ids = new_req.prompt_token_ids or []
|
||||
mm_hashes = [f.identifier for f in new_req.mm_features]
|
||||
if new_req.req_id in self._requests_need_load:
|
||||
meta.add_request(
|
||||
token_ids=token_ids,
|
||||
block_ids=new_req.block_ids[0],
|
||||
block_size=self._block_size,
|
||||
is_store=False,
|
||||
mm_hashes=mm_hashes,
|
||||
)
|
||||
total_need_load += 1
|
||||
else:
|
||||
# NOTE: here, we set the store and load being exclusive,
|
||||
# but a single request can have both store and load.
|
||||
# NOTE(rob): for this debug implementation, we only cache
|
||||
# the original prompt tokens.
|
||||
if not self._found_match_for_prompt(token_ids, mm_hashes):
|
||||
meta.add_request(
|
||||
token_ids=token_ids,
|
||||
block_ids=new_req.block_ids[0],
|
||||
block_size=self._block_size,
|
||||
is_store=True,
|
||||
mm_hashes=mm_hashes,
|
||||
)
|
||||
|
||||
cached_reqs = scheduler_output.scheduled_cached_reqs
|
||||
for i, req_id in enumerate(cached_reqs.req_ids):
|
||||
resumed_from_preemption = req_id in cached_reqs.resumed_req_ids
|
||||
if not resumed_from_preemption or req_id not in self._requests_need_load:
|
||||
continue
|
||||
|
||||
num_computed_tokens = cached_reqs.num_computed_tokens[i]
|
||||
num_new_tokens = scheduler_output.num_scheduled_tokens[req_id]
|
||||
new_block_ids = cached_reqs.new_block_ids[i]
|
||||
|
||||
# NOTE(rob): cached_req_data does not have the full
|
||||
# list of token ids (only new tokens). So we look it
|
||||
# up in the actual request object.
|
||||
request = self._requests_need_load[req_id]
|
||||
total_tokens = num_computed_tokens + num_new_tokens
|
||||
token_ids = request.all_token_ids[:total_tokens]
|
||||
|
||||
# NOTE(rob): For resumed req, new_block_ids is all
|
||||
# of the block_ids for the request.
|
||||
assert new_block_ids is not None
|
||||
block_ids = new_block_ids[0]
|
||||
|
||||
meta.add_request(
|
||||
token_ids=token_ids,
|
||||
block_ids=block_ids,
|
||||
block_size=self._block_size,
|
||||
is_store=False,
|
||||
mm_hashes=[f.identifier for f in request.mm_features],
|
||||
)
|
||||
total_need_load += 1
|
||||
|
||||
assert total_need_load == len(self._requests_need_load)
|
||||
self._requests_need_load.clear()
|
||||
return meta
|
||||
|
||||
# ==============================
|
||||
# Helper functions
|
||||
# ==============================
|
||||
|
||||
def _found_match_for_request(
|
||||
self,
|
||||
request: "Request",
|
||||
) -> bool:
|
||||
"""Check if the cache is hit for the request."""
|
||||
return self._found_match_for_prompt(
|
||||
list(request.prompt_token_ids or []),
|
||||
[f.identifier for f in request.mm_features],
|
||||
)
|
||||
|
||||
def _found_match_for_prompt(
|
||||
self,
|
||||
prompt_token_ids: list[int],
|
||||
mm_hashes: list[str],
|
||||
) -> bool:
|
||||
num_tokens_to_check = align_to_block_size(
|
||||
len(prompt_token_ids) - 1, self._block_size
|
||||
)
|
||||
foldername = self._generate_foldername_debug(
|
||||
torch.tensor(prompt_token_ids)[:num_tokens_to_check],
|
||||
mm_hashes,
|
||||
create_folder=False,
|
||||
)
|
||||
return os.path.exists(foldername)
|
||||
|
||||
def _generate_foldername_debug(
|
||||
self,
|
||||
token_ids: torch.Tensor,
|
||||
mm_hashes: list[str],
|
||||
create_folder=False,
|
||||
) -> str:
|
||||
"""Generate a folder name based on the hash of the bytes of the input
|
||||
ids.
|
||||
"""
|
||||
token_bytes = token_ids.numpy().tobytes()
|
||||
# Add mm_hashes to the bytes being hashed to avoid path traversal and
|
||||
# to create a canonical key.
|
||||
if mm_hashes:
|
||||
mm_str = "-".join(mm_hashes)
|
||||
token_bytes += mm_str.encode("utf-8")
|
||||
input_ids_hash = safe_hash(token_bytes, usedforsecurity=False).hexdigest()
|
||||
|
||||
foldername = os.path.join(self._storage_path, input_ids_hash)
|
||||
if create_folder:
|
||||
os.makedirs(foldername, exist_ok=True)
|
||||
return foldername
|
||||
|
||||
def _generate_filename_debug(
|
||||
self,
|
||||
layer_name: str,
|
||||
token_ids: torch.Tensor,
|
||||
mm_hashes: list[str],
|
||||
) -> str:
|
||||
"""Generate a file name based on the layer name and the hash
|
||||
of the bytes of the input ids.
|
||||
"""
|
||||
foldername = self._generate_foldername_debug(
|
||||
token_ids, mm_hashes=mm_hashes, create_folder=True
|
||||
)
|
||||
return os.path.join(foldername, f"{layer_name}.safetensors")
|
||||
|
||||
|
||||
def align_to_block_size(num_tokens: int, block_size) -> int:
|
||||
"""Align the number of tokens to the block size."""
|
||||
return (num_tokens - 1) // block_size * block_size
|
||||
@@ -1,27 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
|
||||
from vllm.entrypoints.cli.types import CLISubcommand
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
|
||||
class BenchmarkSubcommandBase(CLISubcommand):
|
||||
"""The base class of subcommands for `vllm bench`."""
|
||||
|
||||
help: str
|
||||
|
||||
@classmethod
|
||||
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
|
||||
"""Add the CLI arguments to the parser."""
|
||||
raise NotImplementedError
|
||||
|
||||
@staticmethod
|
||||
def cmd(args: argparse.Namespace) -> None:
|
||||
"""Run the benchmark.
|
||||
|
||||
Args:
|
||||
args: The arguments to the command.
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
@@ -1,22 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
|
||||
from vllm.benchmarks.forward.main import add_cli_args, main
|
||||
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
|
||||
class BenchmarkForwardSubcommand(BenchmarkSubcommandBase):
|
||||
"""The `forward` subcommand for `vllm bench`."""
|
||||
|
||||
name = "forward"
|
||||
help = "Benchmark the model forward of exact batches as CUDA graphs."
|
||||
|
||||
@classmethod
|
||||
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
|
||||
add_cli_args(parser)
|
||||
|
||||
@staticmethod
|
||||
def cmd(args: argparse.Namespace) -> None:
|
||||
main(args)
|
||||
@@ -1,22 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
|
||||
from vllm.benchmarks.latency import add_cli_args, main
|
||||
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
|
||||
class BenchmarkLatencySubcommand(BenchmarkSubcommandBase):
|
||||
"""The `latency` subcommand for `vllm bench`."""
|
||||
|
||||
name = "latency"
|
||||
help = "Benchmark the latency of a single batch of requests."
|
||||
|
||||
@classmethod
|
||||
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
|
||||
add_cli_args(parser)
|
||||
|
||||
@staticmethod
|
||||
def cmd(args: argparse.Namespace) -> None:
|
||||
main(args)
|
||||
@@ -1,100 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import typing
|
||||
|
||||
from vllm import envs
|
||||
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
|
||||
from vllm.entrypoints.cli.types import CLISubcommand
|
||||
from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
else:
|
||||
FlexibleArgumentParser = argparse.ArgumentParser
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def maybe_exec_rust_bench() -> None:
|
||||
if sys.argv[1:3] != ["bench", "serve"] or not envs.VLLM_USE_RUST_BENCH:
|
||||
return
|
||||
|
||||
rust_cli = envs.VLLM_RUST_FRONTEND_PATH
|
||||
if rust_cli is None:
|
||||
raise RuntimeError(
|
||||
"VLLM_USE_RUST_BENCH=1 requires VLLM_RUST_FRONTEND_PATH "
|
||||
"to resolve to the vllm-rs binary."
|
||||
)
|
||||
|
||||
logger.info("Delegating `vllm bench serve` to Rust binary at %s.", rust_cli)
|
||||
os.execv(rust_cli, [rust_cli, "bench", "serve", *sys.argv[3:]])
|
||||
|
||||
|
||||
def _import_bench_subcommand_modules() -> None:
|
||||
# Imported lazily so `BenchmarkSubcommandBase` subclasses register only
|
||||
# when `vllm bench` is actually invoked.
|
||||
import vllm.entrypoints.cli.benchmark.forward # noqa: F401
|
||||
import vllm.entrypoints.cli.benchmark.latency # noqa: F401
|
||||
import vllm.entrypoints.cli.benchmark.mm_processor # noqa: F401
|
||||
import vllm.entrypoints.cli.benchmark.serve # noqa: F401
|
||||
import vllm.entrypoints.cli.benchmark.startup # noqa: F401
|
||||
import vllm.entrypoints.cli.benchmark.sweep # noqa: F401
|
||||
import vllm.entrypoints.cli.benchmark.throughput # noqa: F401
|
||||
|
||||
|
||||
class BenchmarkSubcommand(CLISubcommand):
|
||||
"""The `bench` subcommand for the vLLM CLI."""
|
||||
|
||||
name = "bench"
|
||||
help = "vLLM bench subcommand."
|
||||
|
||||
@staticmethod
|
||||
def cmd(args: argparse.Namespace) -> None:
|
||||
args.dispatch_function(args)
|
||||
|
||||
def validate(self, args: argparse.Namespace) -> None:
|
||||
pass
|
||||
|
||||
def subparser_init(
|
||||
self, subparsers: argparse._SubParsersAction
|
||||
) -> FlexibleArgumentParser:
|
||||
bench_parser = subparsers.add_parser(
|
||||
self.name,
|
||||
help=self.help,
|
||||
description=self.help,
|
||||
usage=f"vllm {self.name} <bench_type> [options]",
|
||||
)
|
||||
bench_subparsers = bench_parser.add_subparsers(required=True, dest="bench_type")
|
||||
|
||||
# Only build the nested bench subparsers when the user is actually
|
||||
# invoking `bench`; otherwise we'd drag in imports
|
||||
# unnecessarily on every `vllm --help` and `vllm serve`.
|
||||
# Scan for the first positional arg so global flags (e.g. `-v`)
|
||||
# before the subcommand don't break detection.
|
||||
first_positional = next(
|
||||
(arg for arg in sys.argv[1:] if not arg.startswith("-")), None
|
||||
)
|
||||
if first_positional == self.name:
|
||||
_import_bench_subcommand_modules()
|
||||
for cmd_cls in BenchmarkSubcommandBase.__subclasses__():
|
||||
cmd_subparser = bench_subparsers.add_parser(
|
||||
cmd_cls.name,
|
||||
help=cmd_cls.help,
|
||||
description=cmd_cls.help,
|
||||
usage=f"vllm {self.name} {cmd_cls.name} [options]",
|
||||
)
|
||||
cmd_subparser.set_defaults(dispatch_function=cmd_cls.cmd)
|
||||
cmd_cls.add_cli_args(cmd_subparser)
|
||||
cmd_subparser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(
|
||||
subcmd=f"{self.name} {cmd_cls.name}"
|
||||
)
|
||||
return bench_parser
|
||||
|
||||
|
||||
def cmd_init() -> list[CLISubcommand]:
|
||||
return [BenchmarkSubcommand()]
|
||||
@@ -1,69 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
import inspect
|
||||
|
||||
from vllm.benchmarks.mm_processor import add_cli_args, main
|
||||
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
|
||||
class BenchmarkMMProcessorSubcommand(BenchmarkSubcommandBase):
|
||||
r"""`vllm bench mm-processor` profiles the multimodal input processor pipeline of
|
||||
vision-language models. It measures per-stage latency from the HuggingFace
|
||||
processor through to the encoder forward pass, helping you identify
|
||||
preprocessing bottlenecks and understand how different image resolutions or
|
||||
item counts affect end-to-end request time.
|
||||
|
||||
The benchmark supports two data sources: synthetic random multimodal inputs
|
||||
(`random-mm`) and HuggingFace datasets (`hf`). Warmup requests are run before
|
||||
measurement to ensure stable results.
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
vllm bench mm-processor \
|
||||
--model Qwen/Qwen2-VL-7B-Instruct \
|
||||
--dataset-name random-mm \
|
||||
--num-prompts 50 \
|
||||
--random-input-len 300 \
|
||||
--random-output-len 40 \
|
||||
--random-mm-base-items-per-request 2 \
|
||||
--random-mm-limit-mm-per-prompt '{"image": 3, "video": 0}' \
|
||||
--random-mm-bucket-config '{(256, 256, 1): 0.7, (720, 1280, 1): 0.3}'
|
||||
```
|
||||
|
||||
## Measured Stages
|
||||
|
||||
| Stage | Description |
|
||||
| ----- | ----------- |
|
||||
| `get_mm_hashes_secs` | Time spent hashing multimodal inputs |
|
||||
| `get_cache_missing_items_secs` | Time spent looking up the processor cache |
|
||||
| `apply_hf_processor_secs` | Time spent in the HuggingFace processor |
|
||||
| `merge_mm_kwargs_secs` | Time spent merging multimodal kwargs |
|
||||
| `apply_prompt_updates_secs` | Time spent updating prompt tokens |
|
||||
| `preprocessor_total_secs` | Total preprocessing time |
|
||||
| `encoder_forward_secs` | Time spent in the encoder model forward pass |
|
||||
| `num_encoder_calls` | Number of encoder invocations per request |
|
||||
|
||||
The benchmark also reports end-to-end latency (TTFT + decode time) per
|
||||
request. Use `--metric-percentiles` to select which percentiles to report
|
||||
(default: p99) and `--output-json` to save results.
|
||||
|
||||
For more examples (HF datasets, warmup, JSON output), see the
|
||||
[multimodal processor benchmark guide](https://docs.vllm.ai/en/latest/benchmarking/cli/#multimodal-processor-benchmark).
|
||||
"""
|
||||
|
||||
name = "mm-processor"
|
||||
help = "Benchmark multimodal processor latency across different configurations."
|
||||
|
||||
@classmethod
|
||||
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
|
||||
# The class docstring is the page overview / `--help` description.
|
||||
if cls.__doc__:
|
||||
parser.description = inspect.cleandoc(cls.__doc__)
|
||||
add_cli_args(parser)
|
||||
|
||||
@staticmethod
|
||||
def cmd(args: argparse.Namespace) -> None:
|
||||
main(args)
|
||||
@@ -1,22 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
|
||||
from vllm.benchmarks.serve import add_cli_args, main
|
||||
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
|
||||
class BenchmarkServingSubcommand(BenchmarkSubcommandBase):
|
||||
"""The `serve` subcommand for `vllm bench`."""
|
||||
|
||||
name = "serve"
|
||||
help = "Benchmark the online serving throughput."
|
||||
|
||||
@classmethod
|
||||
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
|
||||
add_cli_args(parser)
|
||||
|
||||
@staticmethod
|
||||
def cmd(args: argparse.Namespace) -> None:
|
||||
main(args)
|
||||
@@ -1,22 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
|
||||
from vllm.benchmarks.startup import add_cli_args, main
|
||||
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
|
||||
class BenchmarkStartupSubcommand(BenchmarkSubcommandBase):
|
||||
"""The `startup` subcommand for `vllm bench`."""
|
||||
|
||||
name = "startup"
|
||||
help = "Benchmark the startup time of vLLM models."
|
||||
|
||||
@classmethod
|
||||
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
|
||||
add_cli_args(parser)
|
||||
|
||||
@staticmethod
|
||||
def cmd(args: argparse.Namespace) -> None:
|
||||
main(args)
|
||||
@@ -1,22 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
|
||||
from vllm.benchmarks.sweep.cli import add_cli_args, main
|
||||
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
|
||||
class BenchmarkSweepSubcommand(BenchmarkSubcommandBase):
|
||||
"""The `sweep` subcommand for `vllm bench`."""
|
||||
|
||||
name = "sweep"
|
||||
help = "Benchmark for a parameter sweep."
|
||||
|
||||
@classmethod
|
||||
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
|
||||
add_cli_args(parser)
|
||||
|
||||
@staticmethod
|
||||
def cmd(args: argparse.Namespace) -> None:
|
||||
main(args)
|
||||
@@ -1,22 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
|
||||
from vllm.benchmarks.throughput import add_cli_args, main
|
||||
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
|
||||
class BenchmarkThroughputSubcommand(BenchmarkSubcommandBase):
|
||||
"""The `throughput` subcommand for `vllm bench`."""
|
||||
|
||||
name = "throughput"
|
||||
help = "Benchmark offline inference throughput."
|
||||
|
||||
@classmethod
|
||||
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
|
||||
add_cli_args(parser)
|
||||
|
||||
@staticmethod
|
||||
def cmd(args: argparse.Namespace) -> None:
|
||||
main(args)
|
||||
@@ -40,7 +40,6 @@ def main():
|
||||
if maybe_run_omni():
|
||||
return
|
||||
|
||||
import vllm.entrypoints.cli.benchmark.main
|
||||
import vllm.entrypoints.cli.collect_env
|
||||
import vllm.entrypoints.cli.download_kernels
|
||||
import vllm.entrypoints.cli.launch
|
||||
@@ -59,7 +58,6 @@ def main():
|
||||
vllm.entrypoints.cli.openai,
|
||||
vllm.entrypoints.cli.serve,
|
||||
vllm.entrypoints.cli.launch,
|
||||
vllm.entrypoints.cli.benchmark.main,
|
||||
vllm.entrypoints.cli.collect_env,
|
||||
vllm.entrypoints.cli.download_kernels,
|
||||
vllm.entrypoints.cli.preload,
|
||||
@@ -70,24 +68,6 @@ def main():
|
||||
if sys.argv[1:2] != ["snapshot"]:
|
||||
cli_env_setup()
|
||||
|
||||
vllm.entrypoints.cli.benchmark.main.maybe_exec_rust_bench()
|
||||
|
||||
# For 'vllm bench *': use CPU instead of UnspecifiedPlatform by default
|
||||
if len(sys.argv) > 1 and sys.argv[1] == "bench":
|
||||
logger.debug(
|
||||
"Bench command detected, must ensure current platform is not "
|
||||
"UnspecifiedPlatform to avoid device type inference error"
|
||||
)
|
||||
from vllm import platforms
|
||||
|
||||
if platforms.current_platform.is_unspecified():
|
||||
from vllm.platforms.cpu import CpuPlatform
|
||||
|
||||
platforms.current_platform = CpuPlatform()
|
||||
logger.info(
|
||||
"Unspecified platform detected, switching to CPU Platform instead."
|
||||
)
|
||||
|
||||
parser = FlexibleArgumentParser(
|
||||
description="vLLM CLI",
|
||||
epilog=VLLM_SUBCMD_PARSER_EPILOG.format(subcmd="[subcommand]"),
|
||||
|
||||
@@ -17,7 +17,6 @@ from vllm.entrypoints.launchers.cli_args import (
|
||||
validate_parsed_serve_args,
|
||||
)
|
||||
from vllm.entrypoints.launchers.dp_supervisor import run_dp_supervisor
|
||||
from vllm.entrypoints.launchers.launcher import create_server_socket
|
||||
from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
|
||||
from vllm.logger import init_logger
|
||||
from vllm.reasoning import ReasoningParserManager
|
||||
@@ -30,7 +29,6 @@ from vllm.v1.executor.multiproc_executor import MultiprocExecutor
|
||||
from vllm.v1.metrics.prometheus import setup_multiprocess_prometheus
|
||||
from vllm.v1.utils import (
|
||||
APIServerProcessManager,
|
||||
RustFrontendProcessManager,
|
||||
wait_for_completion_or_failure,
|
||||
)
|
||||
|
||||
@@ -62,10 +60,6 @@ class ServeSubcommand(CLISubcommand):
|
||||
uvloop.run(serve_grpc(args))
|
||||
return
|
||||
|
||||
rust_frontend_path = (
|
||||
envs.VLLM_RUST_FRONTEND_PATH if envs.VLLM_USE_RUST_FRONTEND else None
|
||||
)
|
||||
|
||||
if args.headless:
|
||||
if args.api_server_count is not None and args.api_server_count > 0:
|
||||
raise ValueError(
|
||||
@@ -106,12 +100,11 @@ class ServeSubcommand(CLISubcommand):
|
||||
|
||||
# Default api_server_count if not explicitly set.
|
||||
# - Multi-port: 1 (supervisor spawns one server per local DP rank)
|
||||
# - Rust frontend: 1 (not applicable as it's multithreaded)
|
||||
# - External LB: 1 (external LB handles distribution)
|
||||
# - Hybrid LB: Use local DP size (internal LB for local ranks only)
|
||||
# - Internal LB: Use full DP size
|
||||
if args.api_server_count is None:
|
||||
if is_multi_port or is_external_lb or rust_frontend_path:
|
||||
if is_multi_port or is_external_lb:
|
||||
args.api_server_count = 1
|
||||
elif is_hybrid_lb:
|
||||
args.api_server_count = args.data_parallel_size_local or 1
|
||||
@@ -128,12 +121,6 @@ class ServeSubcommand(CLISubcommand):
|
||||
"Defaulting api_server_count to data_parallel_size (%d).",
|
||||
args.api_server_count,
|
||||
)
|
||||
elif rust_frontend_path and args.api_server_count > 1:
|
||||
logger.warning(
|
||||
"Ignoring --api-server-count=%d when using rust front-end process",
|
||||
args.api_server_count,
|
||||
)
|
||||
args.api_server_count = 1
|
||||
|
||||
# Elastic EP currently only supports running with at most one API server.
|
||||
if getattr(args, "enable_elastic_ep", False) and args.api_server_count > 1:
|
||||
@@ -148,7 +135,7 @@ class ServeSubcommand(CLISubcommand):
|
||||
run_dp_supervisor(args)
|
||||
elif args.api_server_count < 1:
|
||||
run_headless(args)
|
||||
elif args.api_server_count > 1 or rust_frontend_path:
|
||||
elif args.api_server_count > 1:
|
||||
run_multi_api_server(args)
|
||||
else:
|
||||
# Single API server (this process).
|
||||
@@ -268,17 +255,9 @@ def run_headless(args: argparse.Namespace):
|
||||
|
||||
def run_multi_api_server(args: argparse.Namespace):
|
||||
assert not args.headless
|
||||
rust_frontend_path = (
|
||||
envs.VLLM_RUST_FRONTEND_PATH if envs.VLLM_USE_RUST_FRONTEND else None
|
||||
)
|
||||
num_api_servers: int = args.api_server_count
|
||||
assert num_api_servers > 0
|
||||
|
||||
if rust_frontend_path and num_api_servers > 1:
|
||||
raise ValueError(
|
||||
"VLLM_RUST_FRONTEND_PATH does not support api_server_count > 1"
|
||||
)
|
||||
|
||||
if num_api_servers > 1:
|
||||
setup_multiprocess_prometheus()
|
||||
|
||||
@@ -296,13 +275,6 @@ def run_multi_api_server(args: argparse.Namespace):
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
|
||||
listen_address, sock = setup_server(args, reuse_port=num_api_servers > 1)
|
||||
# `--grpc-port` is only accepted for the Rust frontend, which inherits this
|
||||
# listener like the HTTP one. gRPC follows the HTTP TCP host, or IPv4
|
||||
# loopback when HTTP uses a Unix socket.
|
||||
grpc_sock = None
|
||||
if args.grpc_port is not None:
|
||||
grpc_host = "127.0.0.1" if args.uds else (args.host or "")
|
||||
grpc_sock = create_server_socket((grpc_host, args.grpc_port), reuse_port=False)
|
||||
|
||||
engine_args = vllm.AsyncEngineArgs.from_cli_args(args)
|
||||
propagate_flash_late_interaction(args, engine_args)
|
||||
@@ -324,9 +296,7 @@ def run_multi_api_server(args: argparse.Namespace):
|
||||
dp_rank = parallel_config.data_parallel_rank
|
||||
assert parallel_config.local_engines_only or dp_rank == 0
|
||||
|
||||
api_server_manager: APIServerProcessManager | RustFrontendProcessManager | None = (
|
||||
None
|
||||
)
|
||||
api_server_manager: APIServerProcessManager | None = None
|
||||
|
||||
from vllm.v1.engine.utils import bind_engine_zmq_listeners
|
||||
|
||||
@@ -343,38 +313,16 @@ def run_multi_api_server(args: argparse.Namespace):
|
||||
coordinator.get_stats_publish_address() if coordinator else None
|
||||
)
|
||||
|
||||
if rust_frontend_path:
|
||||
if parallel_config.local_engines_only:
|
||||
expected_engine_start_index = parallel_config.data_parallel_rank
|
||||
expected_engine_count = parallel_config.data_parallel_size_local
|
||||
else:
|
||||
expected_engine_start_index = 0
|
||||
expected_engine_count = parallel_config.data_parallel_size
|
||||
# Start rust front-end process.
|
||||
api_server_manager = RustFrontendProcessManager(
|
||||
binary_path=rust_frontend_path,
|
||||
sock=sock,
|
||||
grpc_sock=grpc_sock,
|
||||
args=args,
|
||||
input_listener=zmq_listeners.inputs[0],
|
||||
output_listener=zmq_listeners.outputs[0],
|
||||
engine_start_index=expected_engine_start_index,
|
||||
engine_count=expected_engine_count,
|
||||
data_parallel_size=parallel_config.data_parallel_size,
|
||||
stats_update_address=stats_update_address,
|
||||
)
|
||||
else:
|
||||
# Start API server(s).
|
||||
api_server_manager = APIServerProcessManager(
|
||||
listen_address=listen_address,
|
||||
sock=sock,
|
||||
args=args,
|
||||
num_servers=num_api_servers,
|
||||
input_listeners=zmq_listeners.inputs,
|
||||
output_listeners=zmq_listeners.outputs,
|
||||
stats_update_address=stats_update_address,
|
||||
tensor_queue=engine_launch.tensor_queue,
|
||||
)
|
||||
api_server_manager = APIServerProcessManager(
|
||||
listen_address=listen_address,
|
||||
sock=sock,
|
||||
args=args,
|
||||
num_servers=num_api_servers,
|
||||
input_listeners=zmq_listeners.inputs,
|
||||
output_listeners=zmq_listeners.outputs,
|
||||
stats_update_address=stats_update_address,
|
||||
tensor_queue=engine_launch.tensor_queue,
|
||||
)
|
||||
|
||||
# Set frontend processes to watch during engine startup.
|
||||
# If any of these processes exit before the engines are up, the engine startup
|
||||
|
||||
@@ -18,7 +18,7 @@ from vllm.plugins.endpoint_plugins.interface import init_endpoint_plugins_state
|
||||
from vllm.renderers.chat_utils import load_chat_template
|
||||
from vllm.renderers.online_derenderer import OnlineDerenderer
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.tasks import FALLBACK_SUPPORTED_TASKS, POOLING_TASKS, SupportedTask
|
||||
from vllm.tasks import FALLBACK_SUPPORTED_TASKS, SupportedTask
|
||||
|
||||
|
||||
async def init_app_state(
|
||||
@@ -152,11 +152,6 @@ async def init_app_state(
|
||||
engine_client, state, args, request_logger, supported_tasks
|
||||
)
|
||||
|
||||
if any(task in POOLING_TASKS for task in supported_tasks):
|
||||
from vllm.entrypoints.pooling.factories import init_pooling_state
|
||||
|
||||
init_pooling_state(engine_client, state, args, request_logger, supported_tasks)
|
||||
|
||||
from vllm.entrypoints.generate.structured_decisions.api_router import (
|
||||
init_structured_decisions_state,
|
||||
)
|
||||
|
||||
@@ -6,7 +6,7 @@ from fastapi import FastAPI
|
||||
|
||||
from vllm import envs
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.tasks import POOLING_TASKS, SupportedTask
|
||||
from vllm.tasks import SupportedTask
|
||||
|
||||
|
||||
def register_api_routers(
|
||||
@@ -77,11 +77,6 @@ def register_api_routers(
|
||||
|
||||
register_speech_to_text_api_routers(app, supported_tasks)
|
||||
|
||||
if any(task in POOLING_TASKS for task in supported_tasks):
|
||||
from vllm.entrypoints.pooling.factories import register_pooling_api_routers
|
||||
|
||||
register_pooling_api_routers(app, supported_tasks, model_config)
|
||||
|
||||
if getattr(args, "enable_fault_tolerance", False):
|
||||
from vllm.entrypoints.serve.fault_tolerance.api_router import (
|
||||
register_fault_tolerance_api_router,
|
||||
|
||||
@@ -285,11 +285,6 @@ class FrontendArgs(BaseFrontendArgs):
|
||||
"""Host name."""
|
||||
port: int = 8000
|
||||
"""Port number."""
|
||||
grpc_port: int | None = None
|
||||
"""Enable the Rust frontend's additional gRPC Inference and Control services
|
||||
on this port. Requires `VLLM_USE_RUST_FRONTEND=1 vllm serve`; HTTP remains on
|
||||
`--port`. Binds on `--host`, or 127.0.0.1 with `--uds`. Cannot be combined
|
||||
with the Python gRPC server's `--grpc` flag."""
|
||||
data_parallel_supervisor_port: int = 9256
|
||||
"""HTTP port for aggregated health endpoints in multi-port external LB
|
||||
mode."""
|
||||
@@ -468,38 +463,6 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
|
||||
return parser
|
||||
|
||||
|
||||
def validate_grpc_port_arg(args: argparse.Namespace) -> None:
|
||||
"""Validate the Rust frontend's optional gRPC listener."""
|
||||
if getattr(args, "grpc_port", None) is None:
|
||||
return
|
||||
|
||||
if args.grpc:
|
||||
raise ValueError(
|
||||
"--grpc and --grpc-port are mutually exclusive. Use --grpc "
|
||||
"--port for the Python gRPC server, or VLLM_USE_RUST_FRONTEND=1 "
|
||||
"with --grpc-port for the Rust frontend."
|
||||
)
|
||||
if not envs.VLLM_USE_RUST_FRONTEND or getattr(args, "subparser", None) != "serve":
|
||||
raise ValueError("--grpc-port requires VLLM_USE_RUST_FRONTEND=1 vllm serve")
|
||||
if args.headless or (
|
||||
args.api_server_count is not None and args.api_server_count <= 0
|
||||
):
|
||||
raise ValueError(
|
||||
"--grpc-port requires a Rust frontend; remove --headless "
|
||||
"and non-positive --api-server-count"
|
||||
)
|
||||
if args.data_parallel_multi_port_external_lb:
|
||||
raise ValueError(
|
||||
"--grpc-port is incompatible with "
|
||||
"--data-parallel-multi-port-external-lb: "
|
||||
"its frontends would share the same gRPC port"
|
||||
)
|
||||
if not 0 <= args.grpc_port <= 65535:
|
||||
raise ValueError("--grpc-port must be between 0 and 65535")
|
||||
if args.grpc_port != 0 and args.grpc_port == args.port and not args.uds:
|
||||
raise ValueError("--grpc-port must differ from --port when HTTP uses TCP")
|
||||
|
||||
|
||||
def validate_parsed_serve_args(args: argparse.Namespace):
|
||||
"""Quick checks for model serve args that raise prior to loading."""
|
||||
# `vllm launch <component>` builds its parser with make_arg_parser too (see
|
||||
@@ -518,8 +481,6 @@ def validate_parsed_serve_args(args: argparse.Namespace):
|
||||
"the checkpoint's output format."
|
||||
)
|
||||
|
||||
validate_grpc_port_arg(args)
|
||||
|
||||
# Enable auto tool needs a tool call parser to be valid
|
||||
if args.enable_auto_tool_choice and not args.tool_call_parser:
|
||||
raise TypeError("Error: --enable-auto-tool-choice requires --tool-call-parser")
|
||||
|
||||
@@ -22,7 +22,6 @@ import uvicorn
|
||||
import uvloop
|
||||
from fastapi import FastAPI, Response
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import configure_logging_from_args, init_logger
|
||||
from vllm.utils.system_utils import (
|
||||
decorate_logs,
|
||||
@@ -243,12 +242,6 @@ def _run_python_vllm_dp_server(child_args: argparse.Namespace) -> None:
|
||||
uvloop.run(run_server(child_args))
|
||||
|
||||
|
||||
def _run_rust_vllm_dp_server(child_args: argparse.Namespace) -> None:
|
||||
from vllm.entrypoints.cli.serve import run_multi_api_server
|
||||
|
||||
run_multi_api_server(child_args)
|
||||
|
||||
|
||||
def _run_vllm_dp_server(child_args: argparse.Namespace) -> None:
|
||||
"""Entrypoint function for the vLLM DP Server."""
|
||||
configure_logging_from_args(child_args)
|
||||
@@ -260,10 +253,7 @@ def _run_vllm_dp_server(child_args: argparse.Namespace) -> None:
|
||||
name = f"APIServer_DP{child_args.data_parallel_rank}"
|
||||
set_process_title(name)
|
||||
decorate_logs(name)
|
||||
if envs.VLLM_USE_RUST_FRONTEND and envs.VLLM_RUST_FRONTEND_PATH:
|
||||
_run_rust_vllm_dp_server(child_args)
|
||||
else:
|
||||
_run_python_vllm_dp_server(child_args)
|
||||
_run_python_vllm_dp_server(child_args)
|
||||
|
||||
|
||||
class DPSupervisor:
|
||||
|
||||
@@ -35,16 +35,6 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionResponse,
|
||||
)
|
||||
from vllm.entrypoints.pooling.embed.protocol import (
|
||||
EmbeddingRequest,
|
||||
EmbeddingResponse,
|
||||
)
|
||||
from vllm.entrypoints.pooling.scoring.protocol import (
|
||||
RerankRequest,
|
||||
RerankResponse,
|
||||
ScoreRequest,
|
||||
ScoreResponse,
|
||||
)
|
||||
from vllm.entrypoints.serve import create_error_response
|
||||
from vllm.entrypoints.serve.engine.protocol import (
|
||||
ErrorInfo,
|
||||
@@ -147,12 +137,7 @@ class BatchTranslationRequest(TranslationRequest):
|
||||
|
||||
|
||||
BatchRequestInputBody: TypeAlias = (
|
||||
ChatCompletionRequest
|
||||
| EmbeddingRequest
|
||||
| ScoreRequest
|
||||
| RerankRequest
|
||||
| BatchTranscriptionRequest
|
||||
| BatchTranslationRequest
|
||||
ChatCompletionRequest | BatchTranscriptionRequest | BatchTranslationRequest
|
||||
)
|
||||
|
||||
|
||||
@@ -184,12 +169,6 @@ class BatchRequestInput(OpenAIBaseModel):
|
||||
url: str = info.data["url"]
|
||||
if url == "/v1/chat/completions":
|
||||
return ChatCompletionRequest.model_validate(value)
|
||||
if url == "/v1/embeddings":
|
||||
return TypeAdapter(EmbeddingRequest).validate_python(value)
|
||||
if url.endswith("/score"):
|
||||
return TypeAdapter(ScoreRequest).validate_python(value)
|
||||
if url.endswith("/rerank"):
|
||||
return RerankRequest.model_validate(value)
|
||||
if url == "/v1/audio/transcriptions":
|
||||
return BatchTranscriptionRequest.model_validate(value)
|
||||
if url == "/v1/audio/translations":
|
||||
@@ -199,9 +178,6 @@ class BatchRequestInput(OpenAIBaseModel):
|
||||
|
||||
AllResponse: TypeAlias = (
|
||||
ChatCompletionResponse
|
||||
| EmbeddingResponse
|
||||
| ScoreResponse
|
||||
| RerankResponse
|
||||
| TranscriptionResponseVariant
|
||||
| TranslationResponse
|
||||
| TranslationResponseVerbose
|
||||
@@ -738,8 +714,6 @@ async def build_endpoint_registry(
|
||||
openai_serving_chat = getattr(state, "openai_serving_chat", None)
|
||||
openai_serving_transcription = getattr(state, "openai_serving_transcription", None)
|
||||
openai_serving_translation = getattr(state, "openai_serving_translation", None)
|
||||
serving_embedding = getattr(state, "serving_embedding", None)
|
||||
serving_scores = getattr(state, "serving_scores", None)
|
||||
|
||||
allowed_media_domains = getattr(args, "allowed_media_domains", None)
|
||||
|
||||
@@ -754,27 +728,6 @@ async def build_endpoint_registry(
|
||||
),
|
||||
"wrapper_fn": None,
|
||||
},
|
||||
"embeddings": {
|
||||
"url_matcher": lambda url: url == "/v1/embeddings",
|
||||
"handler_getter": lambda: (
|
||||
serving_embedding if serving_embedding is not None else None
|
||||
),
|
||||
"wrapper_fn": None,
|
||||
},
|
||||
"score": {
|
||||
"url_matcher": lambda url: url.endswith("/score"),
|
||||
"handler_getter": lambda: (
|
||||
serving_scores if serving_scores is not None else None
|
||||
),
|
||||
"wrapper_fn": None,
|
||||
},
|
||||
"rerank": {
|
||||
"url_matcher": lambda url: url.endswith("/rerank"),
|
||||
"handler_getter": lambda: (
|
||||
serving_scores if serving_scores is not None else None
|
||||
),
|
||||
"wrapper_fn": None,
|
||||
},
|
||||
"transcriptions": {
|
||||
"url_matcher": lambda url: url == "/v1/audio/transcriptions",
|
||||
"handler_getter": lambda: (
|
||||
@@ -859,9 +812,8 @@ async def run_batch(
|
||||
make_async_error_request_output(
|
||||
request,
|
||||
error_msg=f"URL {request.url} was used. "
|
||||
"Supported endpoints: /v1/chat/completions, /v1/embeddings,"
|
||||
" /v1/audio/transcriptions, /v1/audio/translations, /score, "
|
||||
" /rerank.",
|
||||
"Supported endpoints: /v1/chat/completions,"
|
||||
" /v1/audio/transcriptions, /v1/audio/translations.",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -30,7 +30,6 @@ from vllm.config.quantization import QuantizationConfigArgs
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.entrypoints.common.offline import _O, _R, OfflineInferenceMixin
|
||||
from vllm.entrypoints.generate.beam_search.offline import BeamSearchOfflineMixin
|
||||
from vllm.entrypoints.pooling.offline import PoolingOfflineMixin
|
||||
from vllm.entrypoints.rl.offline import RLOfflineMixin
|
||||
from vllm.entrypoints.serve.utils.api_utils import log_non_default_args
|
||||
from vllm.inputs import PromptType
|
||||
@@ -60,9 +59,7 @@ if TYPE_CHECKING:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class LLM(
|
||||
BeamSearchOfflineMixin, PoolingOfflineMixin, RLOfflineMixin, OfflineInferenceMixin
|
||||
):
|
||||
class LLM(BeamSearchOfflineMixin, RLOfflineMixin, OfflineInferenceMixin):
|
||||
"""An LLM for generating texts from given prompts and sampling parameters.
|
||||
|
||||
This class includes a tokenizer, a language model (possibly distributed
|
||||
@@ -373,8 +370,6 @@ class LLM(
|
||||
self.model_config.renderer_num_workers,
|
||||
)
|
||||
|
||||
PoolingOfflineMixin.__init__(self)
|
||||
|
||||
# Cache for __repr__ to avoid repeated collective_rpc calls
|
||||
self._cached_repr: str | None = None
|
||||
|
||||
|
||||
@@ -1,356 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Sequence
|
||||
from concurrent.futures import Executor
|
||||
from typing import Any, Final, cast
|
||||
|
||||
from vllm import (
|
||||
PoolingParams,
|
||||
PoolingRequestOutput,
|
||||
)
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.renderers import BaseRenderer, merge_kwargs
|
||||
from vllm.renderers.chat_utils import (
|
||||
ChatTemplateConfig,
|
||||
)
|
||||
from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq
|
||||
from vllm.utils.async_utils import make_async
|
||||
from vllm.utils.mistral import is_mistral_tokenizer
|
||||
|
||||
from ..typing import (
|
||||
AnyOfflineInputsContext,
|
||||
AnyRenderParam,
|
||||
EncodeChatRenderParams,
|
||||
EncodeCMPLRenderParams,
|
||||
OfflineEncodeInputsContext,
|
||||
OfflineOutputsContext,
|
||||
PoolingChatLikeRequest,
|
||||
PoolingCompletionLikeRequest,
|
||||
PoolingEngineInput,
|
||||
PoolingServeContext,
|
||||
RequestFactory,
|
||||
RequestGenerator,
|
||||
ScoringRenderParams,
|
||||
)
|
||||
|
||||
|
||||
class PoolingIOProcessor:
|
||||
"""Processor for handling preprocessing & postprocessing ops for pooling requests.
|
||||
|
||||
This class manages both online (serving) and offline (batch) processing of pooling
|
||||
requests, handling chat and completion formats.
|
||||
"""
|
||||
|
||||
name: str
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
renderer: BaseRenderer,
|
||||
chat_template_config: ChatTemplateConfig,
|
||||
):
|
||||
self.vllm_config = vllm_config
|
||||
self.model_config = vllm_config.model_config
|
||||
self.renderer = renderer
|
||||
|
||||
self.chat_template = chat_template_config.chat_template
|
||||
self.chat_template_content_format: Final = (
|
||||
chat_template_config.chat_template_content_format
|
||||
)
|
||||
self.trust_request_chat_template = (
|
||||
chat_template_config.trust_request_chat_template
|
||||
)
|
||||
|
||||
self.template_kwargs = None
|
||||
self.tool_dicts = None
|
||||
|
||||
# Shared thread pool executor for preprocessing
|
||||
self._executor: Executor = self.renderer._executor
|
||||
self.render_async = make_async(self.render, executor=self._executor)
|
||||
|
||||
#######################################
|
||||
# online APIs
|
||||
|
||||
def create_pooling_params(self, request):
|
||||
return request.to_pooling_params()
|
||||
|
||||
def get_request_factory_online(
|
||||
self, ctx: PoolingServeContext
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
request = ctx.request
|
||||
renderer = self.renderer
|
||||
requests: Sequence[AnyRenderParam]
|
||||
|
||||
if isinstance(request, PoolingChatLikeRequest):
|
||||
self._validate_chat_template(
|
||||
request_chat_template=request.chat_template,
|
||||
chat_template_kwargs=request.chat_template_kwargs,
|
||||
trust_request_chat_template=self.trust_request_chat_template,
|
||||
)
|
||||
|
||||
num_requests = 1
|
||||
default_template_kwargs = merge_kwargs(
|
||||
self.template_kwargs,
|
||||
dict(
|
||||
tools=self.tool_dicts,
|
||||
tokenize=is_mistral_tokenizer(renderer.tokenizer),
|
||||
),
|
||||
)
|
||||
|
||||
mm_config = self.model_config.multimodal_config
|
||||
tok_params = request.build_tok_params(self.model_config)
|
||||
chat_params = request.build_chat_params(
|
||||
self.chat_template, self.chat_template_content_format
|
||||
).with_defaults(
|
||||
default_template_kwargs,
|
||||
default_media_io_kwargs=(
|
||||
mm_config.media_io_kwargs if mm_config else None
|
||||
),
|
||||
)
|
||||
|
||||
params_seq = self._params_to_seq(ctx.pooling_params, num_requests)
|
||||
seq_lora_requests = self._lora_request_to_seq(
|
||||
ctx.lora_request, num_requests
|
||||
)
|
||||
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
|
||||
|
||||
requests = [
|
||||
EncodeChatRenderParams(
|
||||
conversations=request.messages,
|
||||
chat_params=chat_params,
|
||||
tok_params=tok_params,
|
||||
prompt_extras=ctx.prompt_extras,
|
||||
skip_mm_cache=False,
|
||||
params=params_seq[i],
|
||||
lora_requests=seq_lora_requests[i],
|
||||
priorities=seq_priority[i],
|
||||
)
|
||||
for i in range(num_requests)
|
||||
]
|
||||
|
||||
return requests
|
||||
|
||||
elif isinstance(request, PoolingCompletionLikeRequest):
|
||||
model_config = self.model_config
|
||||
prompts_seq = prompt_to_seq(request.input)
|
||||
num_requests = len(prompts_seq)
|
||||
|
||||
parsed_prompts = [
|
||||
(
|
||||
prompt
|
||||
if isinstance(prompt, bytes)
|
||||
else parse_model_prompt(model_config, prompt)
|
||||
)
|
||||
for prompt in prompts_seq
|
||||
]
|
||||
tok_params = request.build_tok_params(model_config)
|
||||
|
||||
params_seq = self._params_to_seq(ctx.pooling_params, num_requests)
|
||||
seq_lora_requests = self._lora_request_to_seq(
|
||||
ctx.lora_request, num_requests
|
||||
)
|
||||
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
|
||||
|
||||
requests = [
|
||||
EncodeCMPLRenderParams(
|
||||
prompts=parsed_prompts[i],
|
||||
tok_params=tok_params,
|
||||
prompt_extras=ctx.prompt_extras,
|
||||
skip_mm_cache=False,
|
||||
params=params_seq[i],
|
||||
lora_requests=seq_lora_requests[i],
|
||||
priorities=seq_priority[i],
|
||||
)
|
||||
for i in range(num_requests)
|
||||
]
|
||||
|
||||
return requests
|
||||
else:
|
||||
raise ValueError(f"Invalid {self.name} request type")
|
||||
|
||||
def post_process_online(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
):
|
||||
pass
|
||||
|
||||
#######################################
|
||||
# offline APIs
|
||||
|
||||
def get_request_factory_offline(
|
||||
self, ctx: AnyOfflineInputsContext
|
||||
) -> tuple[RequestFactory, int]:
|
||||
assert isinstance(ctx, OfflineEncodeInputsContext)
|
||||
|
||||
prompts_seq = prompt_to_seq(ctx.prompts)
|
||||
num_requests = len(prompts_seq)
|
||||
pooling_task = ctx.pooling_task
|
||||
|
||||
parsed_prompts = [
|
||||
(
|
||||
prompt
|
||||
if isinstance(prompt, bytes)
|
||||
else parse_model_prompt(self.model_config, prompt)
|
||||
)
|
||||
for prompt in prompts_seq
|
||||
]
|
||||
tok_params = self.renderer.default_cmpl_tok_params.with_kwargs(
|
||||
**(ctx.tokenization_kwargs or {})
|
||||
)
|
||||
|
||||
pooling_params: PoolingParams | Sequence[PoolingParams]
|
||||
if ctx.pooling_params is None:
|
||||
pooling_params = PoolingParams()
|
||||
else:
|
||||
pooling_params = ctx.pooling_params
|
||||
|
||||
# Clone to avoid modifying the caller's pooling parameters.
|
||||
params_seq = [
|
||||
param.clone() for param in self._params_to_seq(pooling_params, num_requests)
|
||||
]
|
||||
|
||||
for param in params_seq:
|
||||
if param.task is None:
|
||||
param.task = pooling_task
|
||||
elif pooling_task == "plugin":
|
||||
# `plugin` task uses io_processor.parse_data to verify inputs.
|
||||
# We actually allow plugin to overwrite pooling_task.
|
||||
pass
|
||||
elif param.task != pooling_task:
|
||||
msg = f"You cannot overwrite {param.task=!r} with {pooling_task=!r}!"
|
||||
raise VLLMValidationError(msg)
|
||||
|
||||
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
|
||||
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
|
||||
|
||||
def request_factory() -> RequestGenerator:
|
||||
for i in range(num_requests):
|
||||
yield EncodeCMPLRenderParams(
|
||||
prompts=parsed_prompts[i],
|
||||
tok_params=tok_params,
|
||||
prompt_extras=None,
|
||||
skip_mm_cache=False,
|
||||
params=params_seq[i],
|
||||
lora_requests=seq_lora_requests[i],
|
||||
priorities=seq_priority[i],
|
||||
)
|
||||
|
||||
return request_factory, num_requests
|
||||
|
||||
def post_process_offline(
|
||||
self,
|
||||
ctx: OfflineOutputsContext,
|
||||
) -> list[PoolingRequestOutput]:
|
||||
return ctx.outputs
|
||||
|
||||
#######################################
|
||||
# helpers
|
||||
|
||||
def render(
|
||||
self,
|
||||
render_params: EncodeCMPLRenderParams
|
||||
| EncodeChatRenderParams
|
||||
| ScoringRenderParams,
|
||||
) -> PoolingEngineInput:
|
||||
if "conversations" in render_params:
|
||||
render_params = cast(EncodeChatRenderParams, render_params)
|
||||
(_,), engine_input = self.renderer.render_chat(
|
||||
conversations=[render_params["conversations"]],
|
||||
chat_params=render_params["chat_params"],
|
||||
tok_params=render_params["tok_params"],
|
||||
prompt_extras=render_params["prompt_extras"],
|
||||
skip_mm_cache=render_params["skip_mm_cache"],
|
||||
)
|
||||
elif "prompts" in render_params:
|
||||
render_params = cast(EncodeCMPLRenderParams, render_params)
|
||||
engine_input = self.renderer.render_cmpl(
|
||||
prompts=[render_params["prompts"]],
|
||||
tok_params=render_params["tok_params"],
|
||||
prompt_extras=render_params["prompt_extras"],
|
||||
skip_mm_cache=render_params["skip_mm_cache"],
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unsupported render_params type {render_params.__class__.__name__}"
|
||||
)
|
||||
|
||||
return PoolingEngineInput(
|
||||
prompts=engine_input[0],
|
||||
params=render_params["params"],
|
||||
lora_requests=render_params["lora_requests"],
|
||||
priorities=render_params["priorities"],
|
||||
)
|
||||
|
||||
def _validate_chat_template(
|
||||
self,
|
||||
request_chat_template: str | None,
|
||||
chat_template_kwargs: dict[str, Any] | None,
|
||||
trust_request_chat_template: bool,
|
||||
):
|
||||
if not trust_request_chat_template and (
|
||||
request_chat_template is not None
|
||||
or (
|
||||
chat_template_kwargs
|
||||
and chat_template_kwargs.get("chat_template") is not None
|
||||
)
|
||||
):
|
||||
raise VLLMValidationError(
|
||||
"Chat template is passed with request, but "
|
||||
"--trust-request-chat-template is not set. "
|
||||
"Refused request with untrusted chat template."
|
||||
)
|
||||
return None
|
||||
|
||||
def _params_to_seq(
|
||||
self,
|
||||
params: PoolingParams | Sequence[PoolingParams],
|
||||
num_requests: int,
|
||||
) -> Sequence[PoolingParams]:
|
||||
if isinstance(params, Sequence):
|
||||
if len(params) != num_requests:
|
||||
raise VLLMValidationError(
|
||||
f"The lengths of prompts ({num_requests}) "
|
||||
f"and params ({len(params)}) must be the same."
|
||||
)
|
||||
|
||||
return params
|
||||
|
||||
return [params] * num_requests
|
||||
|
||||
def _lora_request_to_seq(
|
||||
self,
|
||||
lora_request: LoRARequest | None | Sequence[LoRARequest | None],
|
||||
num_requests: int,
|
||||
) -> Sequence[LoRARequest | None]:
|
||||
if isinstance(lora_request, Sequence):
|
||||
if len(lora_request) != num_requests:
|
||||
raise VLLMValidationError(
|
||||
f"The lengths of prompts ({num_requests}) "
|
||||
f"and lora_request ({len(lora_request)}) must be the same."
|
||||
)
|
||||
|
||||
return lora_request
|
||||
|
||||
return [lora_request] * num_requests
|
||||
|
||||
def _priority_to_seq(
|
||||
self,
|
||||
priority: int | Sequence[int] | None,
|
||||
num_requests: int,
|
||||
) -> Sequence[int]:
|
||||
if priority is not None:
|
||||
if isinstance(priority, int):
|
||||
return [priority] * num_requests
|
||||
|
||||
if len(priority) != num_requests:
|
||||
raise ValueError(
|
||||
f"The lengths of prompts ({num_requests}) "
|
||||
f"and priority ({len(priority)}) must be the same."
|
||||
)
|
||||
|
||||
return priority
|
||||
|
||||
return [0] * num_requests
|
||||
@@ -1,366 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
from pydantic import Field, model_validator
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.generate.base.protocol import validate_cache_salt
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
|
||||
from vllm.renderers.chat_utils import (
|
||||
ChatCompletionMessageParam,
|
||||
ChatTemplateContentFormatOption,
|
||||
)
|
||||
from vllm.tasks import check_removed_pooling_task
|
||||
from vllm.utils import random_uuid
|
||||
from vllm.utils.serial_utils import EmbedDType, EncodingFormat, Endianness
|
||||
|
||||
|
||||
def reject_removed_pooling_parameters(data):
|
||||
if not isinstance(data, dict):
|
||||
return data
|
||||
if "normalize" in data:
|
||||
raise VLLMValidationError(
|
||||
"Parameter `normalize` was removed; use `use_activation` instead.",
|
||||
parameter="normalize",
|
||||
)
|
||||
check_removed_pooling_task(data.get("task"))
|
||||
return data
|
||||
|
||||
|
||||
class PoolingBasicRequestMixin(OpenAIBaseModel):
|
||||
# --8<-- [start:pooling-common-params]
|
||||
model: str | None = None
|
||||
user: str | None = None
|
||||
# --8<-- [end:pooling-common-params]
|
||||
|
||||
# --8<-- [start:pooling-common-extra-params]
|
||||
truncate_prompt_tokens: Annotated[int, Field(ge=-1)] | None = None
|
||||
padding: Literal["max_length", "do_not_pad"] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Whether to pad the prompt, using the same names as the "
|
||||
"Transformers tokenizer. 'max_length' pads to the maximum input "
|
||||
"length, 'do_not_pad' leaves the prompt as is. Models trained "
|
||||
"with a fixed sequence length and no attention mask, such as "
|
||||
"SigLIP, need 'max_length' to match training, otherwise their "
|
||||
"embeddings are not comparable."
|
||||
),
|
||||
)
|
||||
truncation_side: Literal["left", "right"] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Which side to truncate from when truncate_prompt_tokens is active. "
|
||||
"'right' keeps the first N tokens. "
|
||||
"'left' keeps the last N tokens."
|
||||
),
|
||||
)
|
||||
request_id: str = Field(
|
||||
default_factory=random_uuid,
|
||||
description=(
|
||||
"The request_id related to this request. If the caller does "
|
||||
"not set it, a random_uuid will be generated. This id is used "
|
||||
"through out the inference process and return in response."
|
||||
),
|
||||
)
|
||||
priority: int = Field(
|
||||
default=0,
|
||||
ge=-(2**63),
|
||||
le=2**63 - 1,
|
||||
description=(
|
||||
"The priority of the request (lower means earlier handling; "
|
||||
"default: 0). Any priority other than 0 will raise an error "
|
||||
"if the served model does not use priority scheduling."
|
||||
),
|
||||
)
|
||||
mm_processor_kwargs: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description="Additional kwargs to pass to the HF processor.",
|
||||
)
|
||||
cache_salt: str | None = Field(
|
||||
default=None,
|
||||
min_length=1,
|
||||
max_length=1024,
|
||||
description=(
|
||||
"If specified, the prefix cache will be salted with the provided "
|
||||
"string to prevent an attacker to guess prompts in multi-user "
|
||||
"environments. The salt should be random, protected from "
|
||||
"access by 3rd parties, and long enough to be "
|
||||
"unpredictable (e.g., 43 characters base64-encoded, corresponding "
|
||||
"to 256 bit)."
|
||||
),
|
||||
)
|
||||
# --8<-- [end:pooling-common-extra-params]
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def check_cache_salt_support(cls, data: Any) -> Any:
|
||||
if not isinstance(data, dict):
|
||||
return data
|
||||
|
||||
validate_cache_salt(data.get("cache_salt"))
|
||||
return data
|
||||
|
||||
def _build_pooling_tok_params(
|
||||
self,
|
||||
model_config: ModelConfig,
|
||||
*,
|
||||
add_special_tokens: bool,
|
||||
max_total_tokens: int | None,
|
||||
max_output_tokens: int,
|
||||
max_total_tokens_param: str = "max_model_len",
|
||||
max_output_tokens_param: str | None = None,
|
||||
) -> TokenizeParams:
|
||||
encoder_config = model_config.encoder_config or {}
|
||||
if max_output_tokens_param is None:
|
||||
tok_params = TokenizeParams(
|
||||
max_total_tokens=max_total_tokens,
|
||||
max_output_tokens=max_output_tokens,
|
||||
truncate_prompt_tokens=self.truncate_prompt_tokens,
|
||||
truncation_side=self.truncation_side,
|
||||
do_lower_case=encoder_config.get("do_lower_case", False),
|
||||
add_special_tokens=add_special_tokens,
|
||||
max_total_tokens_param=max_total_tokens_param,
|
||||
)
|
||||
else:
|
||||
tok_params = TokenizeParams(
|
||||
max_total_tokens=max_total_tokens,
|
||||
max_output_tokens=max_output_tokens,
|
||||
truncate_prompt_tokens=self.truncate_prompt_tokens,
|
||||
truncation_side=self.truncation_side,
|
||||
do_lower_case=encoder_config.get("do_lower_case", False),
|
||||
add_special_tokens=add_special_tokens,
|
||||
max_total_tokens_param=max_total_tokens_param,
|
||||
max_output_tokens_param=max_output_tokens_param,
|
||||
)
|
||||
|
||||
if self.padding is None:
|
||||
return tok_params
|
||||
|
||||
return tok_params.with_kwargs(padding=self.padding)
|
||||
|
||||
|
||||
class PoolingTokenizeParamsMixin:
|
||||
add_special_tokens: bool
|
||||
|
||||
def _build_pooling_tok_params(
|
||||
self,
|
||||
model_config: ModelConfig,
|
||||
*,
|
||||
add_special_tokens: bool,
|
||||
max_total_tokens: int | None,
|
||||
max_output_tokens: int,
|
||||
max_total_tokens_param: str = "max_model_len",
|
||||
max_output_tokens_param: str | None = None,
|
||||
) -> TokenizeParams:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class FixedMaxLenTokenizeParamsMixin(PoolingTokenizeParamsMixin):
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
return self._build_pooling_tok_params(
|
||||
model_config,
|
||||
add_special_tokens=self.add_special_tokens,
|
||||
max_total_tokens=model_config.max_model_len,
|
||||
max_output_tokens=0,
|
||||
)
|
||||
|
||||
|
||||
class EmbeddingTokenizeParamsMixin(PoolingTokenizeParamsMixin):
|
||||
padding: Literal["max_length", "do_not_pad"] | None
|
||||
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
default_max_total_tokens = model_config.max_model_len
|
||||
max_total_tokens: int | None = default_max_total_tokens
|
||||
max_output_tokens = 0
|
||||
|
||||
pooler_config = model_config.pooler_config
|
||||
if pooler_config is not None:
|
||||
if pooler_config.enable_chunked_processing:
|
||||
max_total_tokens = pooler_config.max_embed_len
|
||||
else:
|
||||
max_total_tokens = (
|
||||
pooler_config.max_embed_len or default_max_total_tokens
|
||||
)
|
||||
|
||||
tok_params = self._build_pooling_tok_params(
|
||||
model_config,
|
||||
add_special_tokens=self.add_special_tokens,
|
||||
max_total_tokens=max_total_tokens,
|
||||
max_output_tokens=max_output_tokens,
|
||||
)
|
||||
|
||||
if (
|
||||
self.padding == "max_length"
|
||||
and pooler_config is not None
|
||||
and pooler_config.enable_chunked_processing
|
||||
):
|
||||
max_padding_len = pooler_config.max_embed_len
|
||||
if max_padding_len is None:
|
||||
max_padding_len = default_max_total_tokens
|
||||
return tok_params.with_kwargs(
|
||||
pad_prompt_tokens=min(default_max_total_tokens, max_padding_len)
|
||||
)
|
||||
|
||||
return tok_params
|
||||
|
||||
|
||||
class CompletionRequestMixin(OpenAIBaseModel):
|
||||
# --8<-- [start:completion-params]
|
||||
input: list[int] | list[list[int]] | str | list[str]
|
||||
# --8<-- [end:completion-params]
|
||||
|
||||
# --8<-- [start:completion-extra-params]
|
||||
add_special_tokens: bool = Field(
|
||||
default=True,
|
||||
description=(
|
||||
"If true (the default), special tokens (e.g. BOS) will be added to "
|
||||
"the prompt."
|
||||
),
|
||||
)
|
||||
# --8<-- [end:completion-extra-params]
|
||||
|
||||
|
||||
class ChatRequestOptionsMixin(OpenAIBaseModel):
|
||||
# --8<-- [start:chat-extra-params]
|
||||
add_generation_prompt: bool = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"If true, the generation prompt will be added to the chat template. "
|
||||
"This is a parameter used by chat template in tokenizer config of the "
|
||||
"model."
|
||||
),
|
||||
)
|
||||
continue_final_message: bool = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"If this is set, the chat will be formatted so that the final "
|
||||
"message in the chat is open-ended, without any EOS tokens. The "
|
||||
"model will continue this message rather than starting a new one. "
|
||||
'This allows you to "prefill" part of the model\'s response for it. '
|
||||
"Cannot be used at the same time as `add_generation_prompt`."
|
||||
),
|
||||
)
|
||||
add_special_tokens: bool = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"If true, special tokens (e.g. BOS) will be added to the prompt "
|
||||
"on top of what is added by the chat template. "
|
||||
"For most models, the chat template takes care of adding the "
|
||||
"special tokens so this should be set to false (as is the "
|
||||
"default)."
|
||||
),
|
||||
)
|
||||
chat_template: str | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"A Jinja template to use for this conversion. "
|
||||
"As of transformers v4.44, default chat template is no longer "
|
||||
"allowed, so you must provide a chat template if the tokenizer "
|
||||
"does not define one."
|
||||
),
|
||||
)
|
||||
chat_template_kwargs: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Additional keyword args to pass to the template renderer. "
|
||||
"Will be accessible by the chat template."
|
||||
),
|
||||
)
|
||||
media_io_kwargs: dict[str, dict[str, Any]] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Additional kwargs to pass to the media IO connectors, "
|
||||
"keyed by modality. Merged with engine-level media_io_kwargs."
|
||||
),
|
||||
)
|
||||
# --8<-- [end:chat-extra-params]
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def check_generation_prompt(cls, data):
|
||||
if not isinstance(data, dict):
|
||||
return data
|
||||
if data.get("continue_final_message") and data.get("add_generation_prompt"):
|
||||
raise VLLMValidationError(
|
||||
"Cannot set both `continue_final_message` and "
|
||||
"`add_generation_prompt` to True.",
|
||||
)
|
||||
return data
|
||||
|
||||
def build_chat_params(
|
||||
self,
|
||||
default_template: str | None,
|
||||
default_template_content_format: ChatTemplateContentFormatOption,
|
||||
) -> ChatParams:
|
||||
return ChatParams(
|
||||
chat_template=self.chat_template or default_template,
|
||||
chat_template_content_format=default_template_content_format,
|
||||
chat_template_kwargs=merge_kwargs(
|
||||
self.chat_template_kwargs,
|
||||
dict(
|
||||
add_generation_prompt=self.add_generation_prompt,
|
||||
continue_final_message=self.continue_final_message,
|
||||
),
|
||||
),
|
||||
media_io_kwargs=self.media_io_kwargs,
|
||||
)
|
||||
|
||||
|
||||
class ChatRequestMixin(ChatRequestOptionsMixin):
|
||||
# --8<-- [start:chat-params]
|
||||
messages: list[ChatCompletionMessageParam]
|
||||
# --8<-- [end:chat-params]
|
||||
|
||||
|
||||
class EncodingRequestMixin(OpenAIBaseModel):
|
||||
# --8<-- [start:encoding-params]
|
||||
encoding_format: EncodingFormat = "float"
|
||||
# --8<-- [end:encoding-params]
|
||||
|
||||
# --8<-- [start:encoding-extra-params]
|
||||
embed_dtype: EmbedDType = Field(
|
||||
default="float32",
|
||||
description=(
|
||||
"What dtype to use for encoding. Default to using float32 for base64 "
|
||||
"encoding to match the OpenAI python client behavior. "
|
||||
"This parameter will affect base64 and binary_response."
|
||||
),
|
||||
)
|
||||
endianness: Endianness = Field(
|
||||
default="native",
|
||||
description=(
|
||||
"What endianness to use for encoding. Default to using native for "
|
||||
"base64 encoding to match the OpenAI python client behavior."
|
||||
"This parameter will affect base64 and binary_response."
|
||||
),
|
||||
)
|
||||
# --8<-- [end:encoding-extra-params]
|
||||
|
||||
|
||||
class EmbedRequestMixin(EncodingRequestMixin):
|
||||
# --8<-- [start:embed-params]
|
||||
dimensions: int | None = None
|
||||
# --8<-- [end:embed-params]
|
||||
|
||||
# --8<-- [start:embed-extra-params]
|
||||
use_activation: bool | None = Field(
|
||||
default=None,
|
||||
description="Whether to use activation for the pooler outputs. "
|
||||
"`None` uses the pooler's default, which is `True` in most cases.",
|
||||
)
|
||||
# --8<-- [end:embed-extra-params]
|
||||
|
||||
|
||||
class ClassifyRequestMixin(OpenAIBaseModel):
|
||||
# --8<-- [start:classify-extra-params]
|
||||
use_activation: bool | None = Field(
|
||||
default=None,
|
||||
description="Whether to use activation for the pooler outputs. "
|
||||
"`None` uses the pooler's default, which is `True` in most cases.",
|
||||
)
|
||||
# --8<-- [end:classify-extra-params]
|
||||
@@ -1,280 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import AsyncGenerator
|
||||
from concurrent.futures import Executor
|
||||
from http import HTTPStatus
|
||||
from typing import ClassVar
|
||||
|
||||
import torch
|
||||
from fastapi import Request
|
||||
from fastapi.responses import Response
|
||||
|
||||
from vllm import PoolingRequestOutput, envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.generate.base.protocol import validate_request_mm_kwargs
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.engine.typing import AnyRequest
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.exceptions import GenerationError, RetryableRequestError
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.renderers.base import BaseRenderer
|
||||
from vllm.renderers.chat_utils import ChatTemplateConfig
|
||||
from vllm.utils.async_utils import make_async, merge_async_iterators
|
||||
|
||||
from ...serve.engine.protocol import ErrorResponse
|
||||
from ..typing import AnyPoolingRequest, PoolingServeContext
|
||||
from .io_processor import PoolingIOProcessor
|
||||
|
||||
|
||||
class PoolingBaseServing(ABC, BaseServing):
|
||||
request_id_prefix: ClassVar[str]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
engine_client: EngineClient,
|
||||
models: OpenAIServingModels,
|
||||
*,
|
||||
request_logger: RequestLogger | None,
|
||||
chat_template_config: ChatTemplateConfig,
|
||||
return_tokens_as_token_ids: bool = False,
|
||||
log_error_stack: bool = False,
|
||||
):
|
||||
super().__init__(
|
||||
models=models,
|
||||
model_config=models.model_config,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
|
||||
self.engine_client = engine_client
|
||||
self.renderer = engine_client.renderer
|
||||
self.vllm_config = engine_client.vllm_config
|
||||
self.max_model_len = self.model_config.max_model_len
|
||||
self.return_tokens_as_token_ids = return_tokens_as_token_ids
|
||||
self.log_error_stack = log_error_stack
|
||||
self.chat_template_config = chat_template_config
|
||||
self.trust_request_mm_kwargs = chat_template_config.trust_request_mm_kwargs
|
||||
|
||||
# Shared thread pool executor for preprocessing and postprocessing.
|
||||
self._executor: Executor = self.renderer._executor
|
||||
self._postprocessing_async = make_async(
|
||||
self._postprocessing, executor=self._executor
|
||||
)
|
||||
|
||||
async def __call__(
|
||||
self,
|
||||
request: AnyPoolingRequest,
|
||||
raw_request: Request | None = None,
|
||||
) -> Response:
|
||||
io_processor = self.get_io_processor(request)
|
||||
ctx = await self._init_ctx(io_processor, request, raw_request)
|
||||
await self._preprocessing(io_processor, ctx)
|
||||
await self._prepare_generators(ctx)
|
||||
await self._collect_batch(ctx)
|
||||
return await self._postprocessing_async(io_processor, ctx)
|
||||
|
||||
@abstractmethod
|
||||
def get_io_processor(self, request: AnyPoolingRequest) -> PoolingIOProcessor:
|
||||
raise NotImplementedError
|
||||
|
||||
async def _preprocessing(
|
||||
self, io_processor: PoolingIOProcessor, ctx: PoolingServeContext
|
||||
):
|
||||
requests = io_processor.get_request_factory_online(ctx)
|
||||
|
||||
if len(requests) == 0:
|
||||
raise ValueError("You must pass at least one prompt")
|
||||
|
||||
ctx.engine_inputs = await asyncio.gather(
|
||||
*[io_processor.render_async(request) for request in requests]
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
def _postprocessing(
|
||||
self, io_processor: PoolingIOProcessor, ctx: PoolingServeContext
|
||||
):
|
||||
io_processor.post_process_online(ctx)
|
||||
return self._build_response(ctx)
|
||||
|
||||
async def _init_ctx(
|
||||
self,
|
||||
io_processor: PoolingIOProcessor,
|
||||
request: AnyPoolingRequest,
|
||||
raw_request: Request | None = None,
|
||||
):
|
||||
validate_request_mm_kwargs(
|
||||
mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),
|
||||
media_io_kwargs=getattr(request, "media_io_kwargs", None),
|
||||
trust_request_mm_kwargs=self.trust_request_mm_kwargs,
|
||||
)
|
||||
base_request_id = self._base_request_id(
|
||||
raw_request, getattr(request, "request_id", None)
|
||||
)
|
||||
request_id = f"{self.request_id_prefix}-{base_request_id}"
|
||||
await self._check_model(request)
|
||||
|
||||
pooling_params = io_processor.create_pooling_params(request)
|
||||
lora_request = self._maybe_get_adapters(request)
|
||||
model_name = self.models.model_name(lora_request)
|
||||
priorities = getattr(request, "priority", 0)
|
||||
prompt_extras = {
|
||||
k: v
|
||||
for k in ("mm_processor_kwargs", "cache_salt", "chat_template_kwargs")
|
||||
if (v := getattr(request, k, None)) is not None
|
||||
}
|
||||
|
||||
ctx = PoolingServeContext(
|
||||
request=request,
|
||||
raw_request=raw_request,
|
||||
model_name=model_name,
|
||||
pooling_params=pooling_params,
|
||||
request_id=request_id,
|
||||
lora_request=lora_request,
|
||||
priorities=priorities,
|
||||
prompt_extras=prompt_extras,
|
||||
)
|
||||
|
||||
self._validate_request(ctx)
|
||||
return ctx
|
||||
|
||||
async def _prepare_generators(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
):
|
||||
if ctx.engine_inputs is None:
|
||||
raise ValueError("Engine inputs not available")
|
||||
|
||||
generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
|
||||
|
||||
trace_headers = (
|
||||
None
|
||||
if ctx.raw_request is None
|
||||
else await self._get_trace_headers(ctx.raw_request.headers)
|
||||
)
|
||||
|
||||
assert ctx.pooling_params is not None
|
||||
pooling_params = ctx.pooling_params
|
||||
pooling_params.verify(self.model_config)
|
||||
|
||||
for i, engine_input in enumerate(ctx.engine_inputs):
|
||||
prompt_request_id = (
|
||||
f"{ctx.request_id}-{i}"
|
||||
if ctx.prompt_request_ids is None
|
||||
else ctx.prompt_request_ids[i]
|
||||
)
|
||||
|
||||
self._log_inputs(
|
||||
prompt_request_id,
|
||||
engine_input["prompts"],
|
||||
params=engine_input["params"],
|
||||
lora_request=ctx.lora_request,
|
||||
)
|
||||
|
||||
generator = self.engine_client.encode(
|
||||
request_id=prompt_request_id,
|
||||
prompt=engine_input["prompts"],
|
||||
pooling_params=engine_input["params"],
|
||||
lora_request=engine_input["lora_requests"],
|
||||
priority=engine_input["priorities"],
|
||||
trace_headers=trace_headers,
|
||||
)
|
||||
|
||||
generators.append(generator)
|
||||
|
||||
ctx.result_generator = merge_async_iterators(*generators)
|
||||
|
||||
async def _collect_batch(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
):
|
||||
if ctx.engine_inputs is None:
|
||||
raise ValueError("Engine prompts not available")
|
||||
|
||||
if ctx.result_generator is None:
|
||||
raise ValueError("Result generator not available")
|
||||
|
||||
num_inputs = len(ctx.engine_inputs)
|
||||
final_res_batch: list[PoolingRequestOutput | None]
|
||||
final_res_batch = [None] * num_inputs
|
||||
|
||||
async for i, res in ctx.result_generator:
|
||||
if res.error is not None:
|
||||
if res.error.retryable:
|
||||
raise RetryableRequestError(res.error.message)
|
||||
raise GenerationError(res.error.message)
|
||||
final_res_batch[i] = res
|
||||
|
||||
if None in final_res_batch:
|
||||
raise ValueError("Failed to generate results for all prompts")
|
||||
|
||||
ctx.final_res_batch = [res for res in final_res_batch if res is not None]
|
||||
|
||||
@abstractmethod
|
||||
def _build_response(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
) -> Response:
|
||||
raise NotImplementedError
|
||||
|
||||
async def _check_model(
|
||||
self,
|
||||
request: AnyRequest | AnyPoolingRequest,
|
||||
) -> ErrorResponse | None:
|
||||
if self._is_model_supported(request.model):
|
||||
return None
|
||||
if request.model in self.models.lora_requests:
|
||||
return None
|
||||
if (
|
||||
envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING
|
||||
and request.model
|
||||
and (load_result := await self.models.resolve_lora(request.model))
|
||||
):
|
||||
if isinstance(load_result, LoRARequest):
|
||||
return None
|
||||
if (
|
||||
isinstance(load_result, ErrorResponse)
|
||||
and load_result.error.code == HTTPStatus.BAD_REQUEST.value
|
||||
):
|
||||
raise ValueError(load_result.error.message)
|
||||
return None
|
||||
|
||||
def _validate_request(self, ctx: PoolingServeContext) -> None:
|
||||
truncate_prompt_tokens = getattr(ctx.request, "truncate_prompt_tokens", None)
|
||||
|
||||
if (
|
||||
truncate_prompt_tokens is not None
|
||||
and truncate_prompt_tokens > self.max_model_len
|
||||
):
|
||||
raise ValueError(
|
||||
"truncate_prompt_tokens value is "
|
||||
"greater than max_model_len."
|
||||
" Please request a smaller truncation size."
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
class PoolingServing(PoolingBaseServing, ABC):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
self.io_processor = self.init_io_processor(
|
||||
vllm_config=self.vllm_config,
|
||||
renderer=self.renderer,
|
||||
chat_template_config=self.chat_template_config,
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
def init_io_processor(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
renderer: BaseRenderer,
|
||||
chat_template_config: ChatTemplateConfig,
|
||||
) -> PoolingIOProcessor:
|
||||
raise NotImplementedError
|
||||
|
||||
def get_io_processor(self, request: AnyPoolingRequest) -> PoolingIOProcessor:
|
||||
return self.io_processor
|
||||
@@ -1,33 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from fastapi.responses import Response
|
||||
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
with_cancellation,
|
||||
)
|
||||
|
||||
from .protocol import ClassificationRequest
|
||||
from .serving import ServingClassification
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def classify(request: Request) -> ServingClassification:
|
||||
handler = getattr(request.app.state, "serving_classification", None)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Classification API")
|
||||
return handler
|
||||
|
||||
|
||||
@router.post("/classify", dependencies=[Depends(validate_json_request)])
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def create_classify(
|
||||
request: ClassificationRequest, raw_request: Request
|
||||
) -> Response:
|
||||
handler = classify(raw_request)
|
||||
return await handler(request, raw_request)
|
||||
@@ -1,12 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from ..base.io_processor import PoolingIOProcessor
|
||||
|
||||
|
||||
class ClassifyIOProcessor(PoolingIOProcessor):
|
||||
name = "classify"
|
||||
|
||||
|
||||
class TokenClassifyIOProcessor(PoolingIOProcessor):
|
||||
name = "token_classify"
|
||||
@@ -1,71 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import time
|
||||
from typing import Annotated, TypeAlias
|
||||
|
||||
from pydantic import BeforeValidator, Field
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
from ..base.protocol import (
|
||||
ChatRequestMixin,
|
||||
ClassifyRequestMixin,
|
||||
CompletionRequestMixin,
|
||||
FixedMaxLenTokenizeParamsMixin,
|
||||
PoolingBasicRequestMixin,
|
||||
reject_removed_pooling_parameters,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class ClassificationCompletionRequest(
|
||||
PoolingBasicRequestMixin,
|
||||
CompletionRequestMixin,
|
||||
ClassifyRequestMixin,
|
||||
FixedMaxLenTokenizeParamsMixin,
|
||||
):
|
||||
def to_pooling_params(self):
|
||||
return PoolingParams(
|
||||
task="classify",
|
||||
use_activation=self.use_activation,
|
||||
)
|
||||
|
||||
|
||||
class ClassificationChatRequest(
|
||||
PoolingBasicRequestMixin,
|
||||
ChatRequestMixin,
|
||||
ClassifyRequestMixin,
|
||||
FixedMaxLenTokenizeParamsMixin,
|
||||
):
|
||||
def to_pooling_params(self):
|
||||
return PoolingParams(
|
||||
task="classify",
|
||||
use_activation=self.use_activation,
|
||||
)
|
||||
|
||||
|
||||
ClassificationRequest: TypeAlias = Annotated[
|
||||
ClassificationCompletionRequest | ClassificationChatRequest,
|
||||
BeforeValidator(reject_removed_pooling_parameters),
|
||||
]
|
||||
|
||||
|
||||
class ClassificationData(OpenAIBaseModel):
|
||||
index: int
|
||||
label: str | None
|
||||
probs: list[float]
|
||||
num_classes: int
|
||||
|
||||
|
||||
class ClassificationResponse(OpenAIBaseModel):
|
||||
id: str = Field(default_factory=lambda: f"classify-{random_uuid()}")
|
||||
object: str = "list"
|
||||
created: int = Field(default_factory=lambda: int(time.time()))
|
||||
model: str
|
||||
data: list[ClassificationData]
|
||||
usage: UsageInfo
|
||||
@@ -1,72 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TypeAlias
|
||||
|
||||
import numpy as np
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from vllm.entrypoints.serve.engine.protocol import UsageInfo
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import ClassificationOutput
|
||||
|
||||
from ..base.serving import PoolingServing
|
||||
from ..typing import PoolingServeContext
|
||||
from .io_processor import ClassifyIOProcessor
|
||||
from .protocol import (
|
||||
ClassificationData,
|
||||
ClassificationRequest,
|
||||
ClassificationResponse,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
ClassificationServeContext: TypeAlias = PoolingServeContext[ClassificationRequest]
|
||||
|
||||
|
||||
class ServingClassification(PoolingServing):
|
||||
request_id_prefix = "classify"
|
||||
|
||||
def init_io_processor(self, *args, **kwargs) -> ClassifyIOProcessor:
|
||||
return ClassifyIOProcessor(*args, **kwargs)
|
||||
|
||||
def _build_response(
|
||||
self,
|
||||
ctx: ClassificationServeContext,
|
||||
) -> JSONResponse:
|
||||
id2label = getattr(self.model_config.hf_config, "id2label", {})
|
||||
num_prompt_tokens = 0
|
||||
items: list[ClassificationData] = []
|
||||
for idx, final_res in enumerate(ctx.final_res_batch):
|
||||
classify_res = ClassificationOutput.from_base(final_res.outputs)
|
||||
|
||||
probs = classify_res.probs
|
||||
predicted_index = int(np.argmax(probs))
|
||||
label = id2label.get(predicted_index)
|
||||
|
||||
item = ClassificationData(
|
||||
index=idx,
|
||||
label=label,
|
||||
probs=probs,
|
||||
num_classes=len(probs),
|
||||
)
|
||||
|
||||
items.append(item)
|
||||
prompt_token_ids = final_res.prompt_token_ids
|
||||
num_prompt_tokens += len(prompt_token_ids)
|
||||
|
||||
usage = UsageInfo(
|
||||
prompt_tokens=num_prompt_tokens,
|
||||
total_tokens=num_prompt_tokens,
|
||||
)
|
||||
|
||||
response = ClassificationResponse(
|
||||
id=ctx.request_id,
|
||||
created=ctx.created_time,
|
||||
model=ctx.model_name,
|
||||
data=items,
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
return JSONResponse(content=response.model_dump())
|
||||
@@ -1,61 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from http import HTTPStatus
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
with_cancellation,
|
||||
)
|
||||
|
||||
from .protocol import CohereEmbedRequest, EmbeddingRequest
|
||||
from .serving import ServingEmbedding
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def embedding(request: Request) -> ServingEmbedding:
|
||||
handler = getattr(request.app.state, "serving_embedding", None)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Embeddings API")
|
||||
return handler
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/embeddings",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def create_embedding(
|
||||
request: EmbeddingRequest,
|
||||
raw_request: Request,
|
||||
):
|
||||
handler = embedding(raw_request)
|
||||
return await handler(request, raw_request)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v2/embed",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def create_cohere_embedding(
|
||||
request: CohereEmbedRequest,
|
||||
raw_request: Request,
|
||||
):
|
||||
handler = embedding(raw_request)
|
||||
return await handler(request, raw_request)
|
||||
@@ -1,777 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Literal, cast
|
||||
|
||||
import torch
|
||||
from openai.types.chat import (
|
||||
ChatCompletionContentPartImageParam,
|
||||
ChatCompletionContentPartTextParam,
|
||||
)
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.inputs import tokens_input
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import PoolingOutput, PoolingRequestOutput
|
||||
from vllm.renderers import merge_kwargs
|
||||
from vllm.renderers.chat_utils import (
|
||||
ChatCompletionContentPartParam,
|
||||
ChatCompletionMessageParam,
|
||||
CustomChatCompletionMessageParam,
|
||||
ImageURL,
|
||||
)
|
||||
from vllm.renderers.hf import resolve_chat_template
|
||||
from vllm.utils.collection_utils import chunk_list
|
||||
from vllm.utils.mistral import is_mistral_tokenizer
|
||||
|
||||
from ..base.io_processor import PoolingIOProcessor
|
||||
from ..late_chunking import (
|
||||
attach_late_chunking_metadata,
|
||||
build_late_chunking_metadata,
|
||||
prepare_late_chunking_input,
|
||||
)
|
||||
from ..scoring.io_processor import JinaRankingIOProcessorMixin
|
||||
from ..typing import (
|
||||
AnyOfflineInputsContext,
|
||||
AnyRenderParam,
|
||||
ChunkedEmbeddingMetadata,
|
||||
EncodeChatRenderParams,
|
||||
LateChunkingSlot,
|
||||
OfflineEncodeInputsContext,
|
||||
OfflineOutputsContext,
|
||||
PoolingChatLikeRequest,
|
||||
PoolingCompletionLikeRequest,
|
||||
PoolingEngineInput,
|
||||
PoolingServeContext,
|
||||
RequestFactory,
|
||||
RequestGenerator,
|
||||
)
|
||||
from .protocol import (
|
||||
CohereEmbedContent,
|
||||
CohereEmbedInput,
|
||||
CohereEmbedRequest,
|
||||
EmbeddingBatchChatInputRequest,
|
||||
EmbeddingBatchChatRequest,
|
||||
EmbeddingChatInputRequest,
|
||||
EmbeddingChatRequest,
|
||||
EmbeddingCompletionRequest,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _ChunkedPromptAggregator:
|
||||
weighted_sum: torch.Tensor | None = None
|
||||
total_weight: int = 0
|
||||
num_chunks: int = 0
|
||||
all_chunks_l2_normalized: bool = True
|
||||
|
||||
|
||||
class EmbedIOProcessor(PoolingIOProcessor):
|
||||
name = "embed"
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
assert self.model_config.pooler_config is not None
|
||||
|
||||
self.pooler_config = self.model_config.pooler_config
|
||||
self.enable_chunked_processing = self.pooler_config.enable_chunked_processing
|
||||
|
||||
# Load task instructions from HF config or sentence-transformers config
|
||||
self.task_instructions: dict[str, str] | None = self._load_task_instructions(
|
||||
self.model_config.hf_config
|
||||
) or self._load_st_prompts(self.model_config.model, self.model_config.revision)
|
||||
if self.task_instructions:
|
||||
logger.info(
|
||||
"Loaded prompt prefixes for input_type: %s",
|
||||
list(self.task_instructions.keys()),
|
||||
)
|
||||
|
||||
def get_request_factory_online(
|
||||
self, ctx: PoolingServeContext
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
if isinstance(ctx.request, CohereEmbedRequest):
|
||||
requests = self._get_request_factory_cohere_online(ctx)
|
||||
elif isinstance(
|
||||
ctx.request,
|
||||
(
|
||||
EmbeddingChatRequest,
|
||||
EmbeddingBatchChatRequest,
|
||||
EmbeddingChatInputRequest,
|
||||
EmbeddingBatchChatInputRequest,
|
||||
),
|
||||
):
|
||||
requests = self._get_request_factory_chat_input_online(ctx)
|
||||
else:
|
||||
requests = super().get_request_factory_online(ctx)
|
||||
|
||||
return requests
|
||||
|
||||
def post_process_online(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
):
|
||||
if ctx.final_res_batch is None:
|
||||
raise ValueError("Final response batch not available")
|
||||
|
||||
if not self.enable_chunked_processing:
|
||||
self._enforce_cohere_max_tokens(ctx)
|
||||
return super().post_process_online(ctx)
|
||||
|
||||
self._post_process_chunked(ctx)
|
||||
self._enforce_cohere_max_tokens(ctx)
|
||||
|
||||
#################################################################
|
||||
# Long Text Embedding with Chunked Processing
|
||||
# PTAL: examples/pooling/embed/openai_embedding_long_text
|
||||
#################################################################
|
||||
|
||||
def maybe_pre_process_chunked(self, ctx: PoolingServeContext) -> None:
|
||||
if not self.enable_chunked_processing:
|
||||
return None
|
||||
|
||||
if ctx.engine_inputs is None:
|
||||
raise ValueError("Engine prompts not available")
|
||||
|
||||
ctx.original_engine_inputs = ctx.engine_inputs
|
||||
request_id = ctx.request_id
|
||||
max_model_len = self.model_config.max_model_len
|
||||
chunked_engine_inputs: list[PoolingEngineInput] = []
|
||||
prompt_request_ids: list[str] = []
|
||||
chunked_embedding_metadata: list[ChunkedEmbeddingMetadata] = []
|
||||
for prompt_idx, engine_input in enumerate(ctx.engine_inputs):
|
||||
token_ids = engine_input["prompts"].get("prompt_token_ids", None)
|
||||
if token_ids is None:
|
||||
raise NotImplementedError(
|
||||
"Long Text Embedding with Chunked Processing does "
|
||||
"not support EmbedsPrompt and EncoderDecoderInput."
|
||||
)
|
||||
|
||||
prompt_token_ids = cast(list[int], token_ids)
|
||||
cache_salt = cast(str | None, engine_input["prompts"].get("cache_salt"))
|
||||
|
||||
for chunk_idx, chunk_tokens in enumerate(
|
||||
chunk_list(prompt_token_ids, max_model_len)
|
||||
):
|
||||
chunked_engine_inputs.append(
|
||||
PoolingEngineInput(
|
||||
prompts=tokens_input(
|
||||
prompt_token_ids=chunk_tokens, cache_salt=cache_salt
|
||||
),
|
||||
params=engine_input["params"],
|
||||
lora_requests=engine_input["lora_requests"],
|
||||
priorities=engine_input["priorities"],
|
||||
)
|
||||
)
|
||||
prompt_request_ids.append(
|
||||
f"{request_id}-prompt-{prompt_idx}-chunk-{chunk_idx}"
|
||||
)
|
||||
chunked_embedding_metadata.append(
|
||||
ChunkedEmbeddingMetadata(
|
||||
prompt_index=prompt_idx,
|
||||
chunk_index=chunk_idx,
|
||||
)
|
||||
)
|
||||
|
||||
ctx.engine_inputs = chunked_engine_inputs
|
||||
ctx.prompt_request_ids = prompt_request_ids
|
||||
ctx.chunked_embedding_metadata = chunked_embedding_metadata
|
||||
|
||||
return None
|
||||
|
||||
def _post_process_chunked(self, ctx: PoolingServeContext) -> None:
|
||||
# Online aggregation for chunked requests to
|
||||
# minimize memory usage
|
||||
# Track aggregation state for each prompt
|
||||
if ctx.chunked_embedding_metadata is None:
|
||||
raise ValueError("Chunked embedding metadata not available")
|
||||
if len(ctx.chunked_embedding_metadata) != len(ctx.final_res_batch):
|
||||
raise ValueError(
|
||||
"Chunked embedding metadata count does not match result count"
|
||||
)
|
||||
|
||||
prompt_aggregators: dict[int, _ChunkedPromptAggregator] = {}
|
||||
for result, chunk_metadata in zip(
|
||||
ctx.final_res_batch, ctx.chunked_embedding_metadata
|
||||
):
|
||||
prompt_idx = chunk_metadata.prompt_index
|
||||
aggregator = prompt_aggregators.setdefault(
|
||||
prompt_idx, _ChunkedPromptAggregator()
|
||||
)
|
||||
|
||||
# MEAN pooling with online weighted averaging
|
||||
# Ensure result is PoolingRequestOutput
|
||||
# for embedding processing
|
||||
if not isinstance(result, PoolingRequestOutput):
|
||||
raise ValueError(
|
||||
f"Expected PoolingRequestOutput for "
|
||||
f"chunked embedding, got "
|
||||
f"{type(result).__name__}"
|
||||
)
|
||||
if result.prompt_token_ids is None:
|
||||
raise ValueError(
|
||||
"prompt_token_ids cannot be None for chunked processing"
|
||||
)
|
||||
|
||||
weight = len(result.prompt_token_ids)
|
||||
embedding_data = result.outputs.data
|
||||
float_embedding = embedding_data.to(dtype=torch.float32)
|
||||
if ctx.pooling_params.use_activation:
|
||||
# Custom activations need not L2-normalize; allow BF16 rounding.
|
||||
embedding_norm = torch.linalg.vector_norm(float_embedding)
|
||||
aggregator.all_chunks_l2_normalized &= torch.allclose(
|
||||
embedding_norm,
|
||||
torch.ones_like(embedding_norm),
|
||||
rtol=1e-2,
|
||||
atol=1e-5,
|
||||
)
|
||||
weighted_embedding = float_embedding * weight
|
||||
|
||||
if aggregator.weighted_sum is None:
|
||||
# First chunk
|
||||
aggregator.weighted_sum = weighted_embedding
|
||||
else:
|
||||
# Accumulate
|
||||
aggregator.weighted_sum += weighted_embedding
|
||||
|
||||
aggregator.total_weight += weight
|
||||
aggregator.num_chunks += 1
|
||||
|
||||
if ctx.original_engine_inputs is None:
|
||||
raise ValueError("Original engine inputs not available")
|
||||
|
||||
original_engine_inputs = ctx.original_engine_inputs
|
||||
num_prompts = len(original_engine_inputs)
|
||||
|
||||
# Finalize aggregated results
|
||||
final_res_batch: list[PoolingRequestOutput] = []
|
||||
for prompt_idx in range(num_prompts):
|
||||
if prompt_idx in prompt_aggregators:
|
||||
# Finalize MEAN aggregation for this chunked prompt
|
||||
aggregator = prompt_aggregators[prompt_idx]
|
||||
|
||||
weighted_sum = aggregator.weighted_sum
|
||||
total_weight = aggregator.total_weight
|
||||
|
||||
if (
|
||||
weighted_sum is not None
|
||||
and isinstance(weighted_sum, torch.Tensor)
|
||||
and isinstance(total_weight, (int, float))
|
||||
and total_weight > 0
|
||||
):
|
||||
# Compute final mean embedding
|
||||
final_embedding = weighted_sum / total_weight
|
||||
if (
|
||||
ctx.pooling_params.use_activation
|
||||
and aggregator.num_chunks > 1
|
||||
and aggregator.all_chunks_l2_normalized
|
||||
):
|
||||
final_embedding = torch.nn.functional.normalize(
|
||||
final_embedding, dim=-1
|
||||
)
|
||||
|
||||
# Create a PoolingRequestOutput
|
||||
# for the aggregated result
|
||||
pooling_output_data = PoolingOutput(data=final_embedding)
|
||||
|
||||
# Get original prompt token IDs for this prompt
|
||||
original_prompt = original_engine_inputs[prompt_idx]
|
||||
token_ids = original_prompt["prompts"].get("prompt_token_ids", None)
|
||||
if token_ids is None:
|
||||
raise NotImplementedError(
|
||||
"Long Text Embedding with Chunked Processing does "
|
||||
"not support EmbedsPrompt and EncoderDecoderInput."
|
||||
)
|
||||
|
||||
original_token_ids = cast(list[int], token_ids)
|
||||
pooling_request_output = PoolingRequestOutput(
|
||||
request_id=f"{ctx.request_id}-prompt-{prompt_idx}",
|
||||
prompt_token_ids=original_token_ids,
|
||||
outputs=pooling_output_data,
|
||||
num_cached_tokens=0,
|
||||
finished=True,
|
||||
)
|
||||
|
||||
final_res_batch.append(pooling_request_output)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Failed to aggregate chunks for prompt {prompt_idx}"
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Result not found for prompt {prompt_idx}")
|
||||
|
||||
ctx.final_res_batch = final_res_batch
|
||||
|
||||
return None
|
||||
|
||||
#################################################################
|
||||
# Chat input Request Preprocessing & Postprocessing
|
||||
#################################################################
|
||||
|
||||
def _get_request_factory_chat_input_online(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
request = ctx.request
|
||||
renderer = self.renderer
|
||||
|
||||
self._validate_chat_template(
|
||||
request_chat_template=request.chat_template,
|
||||
chat_template_kwargs=request.chat_template_kwargs,
|
||||
trust_request_chat_template=self.trust_request_chat_template,
|
||||
)
|
||||
|
||||
if isinstance(
|
||||
request, (EmbeddingBatchChatRequest, EmbeddingBatchChatInputRequest)
|
||||
):
|
||||
all_messages = request.messages
|
||||
else:
|
||||
all_messages = [request.messages]
|
||||
num_requests = len(all_messages)
|
||||
|
||||
default_template_kwargs = merge_kwargs(
|
||||
self.template_kwargs,
|
||||
dict(
|
||||
tools=self.tool_dicts,
|
||||
tokenize=is_mistral_tokenizer(renderer.tokenizer),
|
||||
),
|
||||
)
|
||||
|
||||
mm_config = self.model_config.multimodal_config
|
||||
tok_params = request.build_tok_params(self.model_config)
|
||||
chat_params = request.build_chat_params(
|
||||
self.chat_template, self.chat_template_content_format
|
||||
).with_defaults(
|
||||
default_template_kwargs,
|
||||
default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
|
||||
)
|
||||
|
||||
params_seq = self._params_to_seq(ctx.pooling_params, num_requests)
|
||||
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
|
||||
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
|
||||
|
||||
requests = [
|
||||
EncodeChatRenderParams(
|
||||
conversations=all_messages[i],
|
||||
chat_params=chat_params,
|
||||
tok_params=tok_params,
|
||||
prompt_extras=ctx.prompt_extras,
|
||||
skip_mm_cache=False,
|
||||
params=params_seq[i],
|
||||
lora_requests=seq_lora_requests[i],
|
||||
priorities=seq_priority[i],
|
||||
)
|
||||
for i in range(num_requests)
|
||||
]
|
||||
|
||||
return requests
|
||||
|
||||
#################################################################
|
||||
# Cohere Request Preprocessing & Postprocessing
|
||||
#################################################################
|
||||
|
||||
@staticmethod
|
||||
def _load_task_instructions(hf_config: Any) -> dict[str, str] | None:
|
||||
"""Extract ``task_instructions`` from the HF model config."""
|
||||
ti = getattr(hf_config, "task_instructions", None)
|
||||
if not isinstance(ti, dict) or not ti:
|
||||
return None
|
||||
return {k: v for k, v in ti.items() if isinstance(v, str)}
|
||||
|
||||
@staticmethod
|
||||
def _load_st_prompts(
|
||||
model: str | Any,
|
||||
revision: str | None,
|
||||
) -> dict[str, str] | None:
|
||||
"""Load ``task_instructions`` from ``config_sentence_transformers.json``."""
|
||||
from vllm.transformers_utils.repo_utils import get_hf_file_to_dict
|
||||
|
||||
try:
|
||||
cfg = get_hf_file_to_dict(
|
||||
"config_sentence_transformers.json", str(model), revision
|
||||
)
|
||||
except (ValueError, OSError):
|
||||
return None
|
||||
|
||||
if cfg is None:
|
||||
return None
|
||||
prompts = cfg.get("prompts")
|
||||
if not isinstance(prompts, dict) or not prompts:
|
||||
return None
|
||||
return {k: v for k, v in prompts.items() if isinstance(v, str)}
|
||||
|
||||
@staticmethod
|
||||
def _mixed_input_to_messages(
|
||||
inp: CohereEmbedInput,
|
||||
*,
|
||||
task_prefix: str | None = None,
|
||||
) -> list[ChatCompletionMessageParam]:
|
||||
"""Build chat messages from a mixed text+image input.
|
||||
|
||||
When *task_prefix* is given, it is used as the system prompt.
|
||||
"""
|
||||
messages: list[ChatCompletionMessageParam] = []
|
||||
if task_prefix is not None:
|
||||
messages.append(
|
||||
CustomChatCompletionMessageParam(
|
||||
role="system",
|
||||
content=[
|
||||
ChatCompletionContentPartTextParam(
|
||||
type="text", text=task_prefix
|
||||
)
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
parts: list[ChatCompletionContentPartParam] = []
|
||||
for item in inp.content:
|
||||
if item.type == "text" and item.text is not None:
|
||||
parts.append(
|
||||
ChatCompletionContentPartTextParam(type="text", text=item.text)
|
||||
)
|
||||
elif item.type == "image_url" and item.image_url is not None:
|
||||
parts.append(
|
||||
ChatCompletionContentPartImageParam(
|
||||
type="image_url",
|
||||
image_url=ImageURL(url=item.image_url["url"]),
|
||||
)
|
||||
)
|
||||
messages.append(CustomChatCompletionMessageParam(role="user", content=parts))
|
||||
return messages
|
||||
|
||||
@staticmethod
|
||||
def _check_cohere_max_tokens(
|
||||
outputs: list[PoolingRequestOutput],
|
||||
max_tokens_check: int | None,
|
||||
) -> None:
|
||||
"""Raise if any output exceeds *max_tokens_check* tokens.
|
||||
|
||||
Used to enforce ``truncate=NONE`` with an explicit ``max_tokens``:
|
||||
the pipeline runs without truncation and we reject afterwards.
|
||||
"""
|
||||
if max_tokens_check is None:
|
||||
return
|
||||
for out in outputs:
|
||||
n = len(out.prompt_token_ids)
|
||||
if n > max_tokens_check:
|
||||
raise ValueError(
|
||||
f"Input of {n} tokens exceeds max_tokens={max_tokens_check} "
|
||||
"with truncate=NONE. Set truncate to END or START to "
|
||||
"allow truncation."
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _resolve_cohere_truncation(
|
||||
request: CohereEmbedRequest,
|
||||
) -> tuple[int | None, Literal["left", "right"] | None]:
|
||||
"""Return ``(truncate_prompt_tokens, truncation_side)``."""
|
||||
if request.truncate == "NONE":
|
||||
return None, None
|
||||
if request.truncate == "START":
|
||||
tokens = request.max_tokens if request.max_tokens is not None else -1
|
||||
return tokens, "left"
|
||||
if request.max_tokens is not None:
|
||||
return request.max_tokens, None
|
||||
return -1, None
|
||||
|
||||
def create_pooling_params(self, request):
|
||||
if isinstance(request, CohereEmbedRequest):
|
||||
return PoolingParams(
|
||||
task="embed",
|
||||
dimensions=request.output_dimension,
|
||||
)
|
||||
return super().create_pooling_params(request)
|
||||
|
||||
def _get_request_factory_cohere_online(
|
||||
self, ctx: PoolingServeContext
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
"""Convert a ``CohereEmbedRequest`` into engine prompts.
|
||||
|
||||
If a model has a chat template the task instruction are rendered
|
||||
as a system prompt. Otherwise they are just prepended to the input text.
|
||||
|
||||
Images and mixed inputs are always batch-rendered through the chat
|
||||
template in one ``render_chat`` call.
|
||||
"""
|
||||
request = ctx.request
|
||||
assert isinstance(request, CohereEmbedRequest)
|
||||
|
||||
if request.texts is None and request.images is None and request.inputs is None:
|
||||
raise ValueError("One of texts, images, or inputs must be provided")
|
||||
|
||||
truncate_prompt_tokens, truncation_side = self._resolve_cohere_truncation(
|
||||
request
|
||||
)
|
||||
input_type = request.input_type
|
||||
self._validate_input_type(input_type)
|
||||
|
||||
if request.images is not None:
|
||||
input: list[CohereEmbedInput] = [
|
||||
CohereEmbedInput(
|
||||
content=[
|
||||
CohereEmbedContent(type="image_url", image_url={"url": uri})
|
||||
]
|
||||
)
|
||||
for uri in request.images
|
||||
]
|
||||
elif request.inputs is not None:
|
||||
input = request.inputs
|
||||
else:
|
||||
texts = request.texts or []
|
||||
task_prefix = self._get_task_instruction_prefix(input_type)
|
||||
|
||||
if task_prefix is None:
|
||||
return self._get_request_factory_cohere_text_completion(
|
||||
ctx,
|
||||
texts,
|
||||
truncate_prompt_tokens,
|
||||
truncation_side,
|
||||
)
|
||||
|
||||
all_messages = [
|
||||
self._mixed_input_to_messages(
|
||||
CohereEmbedInput(
|
||||
content=[CohereEmbedContent(type="text", text=text)]
|
||||
),
|
||||
task_prefix=task_prefix,
|
||||
)
|
||||
for text in texts
|
||||
]
|
||||
if self._has_chat_template():
|
||||
return self._batch_render_chat(
|
||||
ctx,
|
||||
all_messages,
|
||||
truncate_prompt_tokens,
|
||||
truncation_side,
|
||||
)
|
||||
else:
|
||||
return self._get_request_factory_cohere_text_completion(
|
||||
ctx,
|
||||
self._apply_task_instruction(texts, input_type),
|
||||
truncate_prompt_tokens,
|
||||
truncation_side,
|
||||
)
|
||||
|
||||
task_prefix = self._get_task_instruction_prefix(input_type)
|
||||
all_messages = [
|
||||
self._mixed_input_to_messages(inp, task_prefix=task_prefix) for inp in input
|
||||
]
|
||||
|
||||
return self._batch_render_chat(
|
||||
ctx,
|
||||
all_messages,
|
||||
truncate_prompt_tokens,
|
||||
truncation_side,
|
||||
)
|
||||
|
||||
def _has_chat_template(self) -> bool:
|
||||
return (
|
||||
resolve_chat_template(
|
||||
self.renderer.tokenizer,
|
||||
chat_template=self.chat_template,
|
||||
tools=None,
|
||||
model_config=self.model_config,
|
||||
)
|
||||
is not None
|
||||
)
|
||||
|
||||
def _get_request_factory_cohere_text_completion(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
texts: list[str],
|
||||
truncate_prompt_tokens: int | None,
|
||||
truncation_side: Literal["left", "right"] | None,
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
request = ctx.request
|
||||
proxy = EmbeddingCompletionRequest(
|
||||
model=request.model,
|
||||
input=texts,
|
||||
dimensions=request.output_dimension,
|
||||
encoding_format="float",
|
||||
truncate_prompt_tokens=truncate_prompt_tokens,
|
||||
truncation_side=truncation_side,
|
||||
)
|
||||
ctx.request = proxy
|
||||
requests = super().get_request_factory_online(ctx)
|
||||
ctx.request = request
|
||||
return requests
|
||||
|
||||
def _batch_render_chat(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
all_messages: Sequence[list[ChatCompletionMessageParam]],
|
||||
truncate_prompt_tokens: int | None,
|
||||
truncation_side: Literal["left", "right"] | None,
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
"""Batch-render multiple conversations through the chat template."""
|
||||
request = ctx.request
|
||||
proxy = EmbeddingBatchChatRequest(
|
||||
model=request.model,
|
||||
messages=all_messages,
|
||||
dimensions=request.output_dimension,
|
||||
encoding_format="float",
|
||||
truncate_prompt_tokens=truncate_prompt_tokens,
|
||||
truncation_side=truncation_side,
|
||||
)
|
||||
ctx.request = proxy
|
||||
requests = self._get_request_factory_chat_input_online(ctx)
|
||||
ctx.request = request
|
||||
return requests
|
||||
|
||||
def _validate_input_type(self, input_type: str | None) -> None:
|
||||
"""Raise if *input_type* is not supported by this model."""
|
||||
if input_type is None:
|
||||
return
|
||||
if self.task_instructions is None:
|
||||
raise ValueError(
|
||||
f"Unsupported input_type {input_type!r}. "
|
||||
"This model does not define any input_type task instructions."
|
||||
)
|
||||
if input_type not in self.task_instructions:
|
||||
supported = ", ".join(sorted(self.task_instructions))
|
||||
raise ValueError(
|
||||
f"Unsupported input_type {input_type!r}. Supported values: {supported}"
|
||||
)
|
||||
|
||||
def _apply_task_instruction(
|
||||
self,
|
||||
texts: list[str],
|
||||
input_type: str | None,
|
||||
) -> list[str]:
|
||||
"""Prepend the task-instruction prefix for *input_type*.
|
||||
|
||||
Returns *texts* unchanged when no matching prefix is configured.
|
||||
"""
|
||||
prefix = self._get_task_instruction_prefix(input_type)
|
||||
if not prefix:
|
||||
return texts
|
||||
return [prefix + t for t in texts]
|
||||
|
||||
def _get_task_instruction_prefix(self, input_type: str | None) -> str | None:
|
||||
"""Return the task-instruction prefix for *input_type*, or ``None``."""
|
||||
if not self.task_instructions or input_type is None:
|
||||
return None
|
||||
return self.task_instructions.get(input_type) or None
|
||||
|
||||
def _enforce_cohere_max_tokens(self, ctx: PoolingServeContext) -> None:
|
||||
if isinstance(ctx.request, CohereEmbedRequest):
|
||||
request = ctx.request
|
||||
if request.truncate == "NONE" and request.max_tokens is not None:
|
||||
self._check_cohere_max_tokens(ctx.final_res_batch, request.max_tokens)
|
||||
|
||||
|
||||
class TokenEmbedIOProcessor(PoolingIOProcessor):
|
||||
name = "token_embed"
|
||||
|
||||
def get_request_factory_offline(
|
||||
self, ctx: AnyOfflineInputsContext
|
||||
) -> tuple[RequestFactory, int]:
|
||||
factory, num_requests = super().get_request_factory_offline(ctx)
|
||||
assert isinstance(ctx, OfflineEncodeInputsContext)
|
||||
if ctx.pooling_params is None or not any(
|
||||
p.late_chunking_params is not None
|
||||
for p in self._params_to_seq(ctx.pooling_params, num_requests)
|
||||
):
|
||||
return factory, num_requests
|
||||
|
||||
if (ctx.tokenization_kwargs or {}).get("padding") not in (
|
||||
None,
|
||||
False,
|
||||
"do_not_pad",
|
||||
):
|
||||
raise VLLMValidationError("Late chunking does not support input padding")
|
||||
|
||||
def request_factory() -> RequestGenerator:
|
||||
for request in factory():
|
||||
slot = None
|
||||
if request["params"].late_chunking_params is not None:
|
||||
slot = LateChunkingSlot()
|
||||
request["late_chunking"] = slot
|
||||
ctx.late_chunking.append(slot)
|
||||
yield request
|
||||
|
||||
return request_factory, num_requests
|
||||
|
||||
def render(self, render_params: AnyRenderParam) -> PoolingEngineInput:
|
||||
late_chunking = render_params["params"].late_chunking_params
|
||||
if late_chunking is None:
|
||||
return super().render(render_params)
|
||||
|
||||
text, tok_params = prepare_late_chunking_input(self.vllm_config, render_params)
|
||||
render_params = render_params.copy()
|
||||
render_params["tok_params"] = tok_params
|
||||
result = super().render(render_params)
|
||||
prompt = result["prompts"]
|
||||
if prompt["type"] != "token":
|
||||
raise VLLMValidationError("Late chunking requires tokenized text")
|
||||
render_params["late_chunking"].metadata = build_late_chunking_metadata(
|
||||
text,
|
||||
len(prompt["prompt_token_ids"]),
|
||||
prompt.pop("prompt_token_offsets", None),
|
||||
late_chunking.chunk_size,
|
||||
)
|
||||
return result
|
||||
|
||||
def post_process_offline(self, ctx: OfflineOutputsContext):
|
||||
if ctx.late_chunking:
|
||||
for output, slot in zip(ctx.outputs, ctx.late_chunking, strict=True):
|
||||
if slot is not None:
|
||||
assert slot.metadata is not None
|
||||
attach_late_chunking_metadata(output, slot.metadata)
|
||||
return ctx.outputs
|
||||
|
||||
|
||||
class JinaRankingTokenEmbedIOProcessor(
|
||||
TokenEmbedIOProcessor, JinaRankingIOProcessorMixin
|
||||
):
|
||||
def render(self, render_params: AnyRenderParam) -> PoolingEngineInput:
|
||||
if render_params["params"].late_chunking_params is not None:
|
||||
raise VLLMValidationError(
|
||||
"JinaForRanking returns document/query vectors and does not support "
|
||||
"late chunking"
|
||||
)
|
||||
return super().render(render_params)
|
||||
|
||||
def get_request_factory_online(
|
||||
self, ctx: PoolingServeContext
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
request = ctx.request
|
||||
if isinstance(request, PoolingCompletionLikeRequest):
|
||||
prompts = request.input
|
||||
if not isinstance(prompts, Sequence) or len(prompts) < 2:
|
||||
raise ValueError("The JinaForRanking model requires at least 2 inputs.")
|
||||
|
||||
text_prompts = self.ensure_str(prompts)
|
||||
|
||||
# The JinaForRanking model concatenates docs first, then query.
|
||||
# Let's stay consistent with this novel design.
|
||||
prompt_input = self.format_docs_prompts_func(
|
||||
query=text_prompts[-1], docs=text_prompts[:-1]
|
||||
)
|
||||
|
||||
request.input = prompt_input
|
||||
return super().get_request_factory_online(ctx)
|
||||
elif isinstance(request, PoolingChatLikeRequest):
|
||||
raise ValueError("The JinaForRanking does not support chat Request.")
|
||||
else:
|
||||
raise ValueError(f"Invalid {self.name} request type")
|
||||
|
||||
def get_request_factory_offline(
|
||||
self, ctx: AnyOfflineInputsContext
|
||||
) -> tuple[RequestFactory, int]:
|
||||
assert isinstance(ctx, OfflineEncodeInputsContext)
|
||||
if not isinstance(ctx.prompts, Sequence) or len(ctx.prompts) < 2:
|
||||
raise ValueError("The JinaForRanking model requires at least 2 inputs.")
|
||||
|
||||
text_prompts = self.ensure_str(ctx.prompts)
|
||||
|
||||
# The JinaForRanking model concatenates docs first, then query.
|
||||
# Let's stay consistent with this novel design.
|
||||
ctx.prompts = self.format_docs_prompts_func(
|
||||
query=text_prompts[-1], docs=text_prompts[:-1]
|
||||
)
|
||||
|
||||
return super().get_request_factory_offline(ctx)
|
||||
@@ -1,354 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Embedding API protocol models for OpenAI and Cohere formats.
|
||||
|
||||
OpenAI: https://platform.openai.com/docs/api-reference/embeddings
|
||||
Cohere: https://docs.cohere.com/reference/embed
|
||||
"""
|
||||
|
||||
import builtins
|
||||
import struct
|
||||
import time
|
||||
from collections.abc import Sequence
|
||||
from typing import Annotated, Any, Literal, TypeAlias
|
||||
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
from pydantic import BaseModel, BeforeValidator, Field, model_validator
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||
from vllm.renderers.chat_utils import ChatCompletionMessageParam
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
from ..base.protocol import (
|
||||
ChatRequestMixin,
|
||||
ChatRequestOptionsMixin,
|
||||
CompletionRequestMixin,
|
||||
EmbeddingTokenizeParamsMixin,
|
||||
EmbedRequestMixin,
|
||||
PoolingBasicRequestMixin,
|
||||
reject_removed_pooling_parameters,
|
||||
)
|
||||
|
||||
|
||||
class EmbeddingCompletionRequest(
|
||||
PoolingBasicRequestMixin,
|
||||
CompletionRequestMixin,
|
||||
EmbedRequestMixin,
|
||||
EmbeddingTokenizeParamsMixin,
|
||||
):
|
||||
def to_pooling_params(self):
|
||||
return PoolingParams(
|
||||
task="embed",
|
||||
dimensions=self.dimensions,
|
||||
use_activation=self.use_activation,
|
||||
)
|
||||
|
||||
|
||||
def _is_chat_message(value: Any) -> bool:
|
||||
return isinstance(value, dict) and isinstance(value.get("role"), str)
|
||||
|
||||
|
||||
def _is_chat_messages(value: Any) -> bool:
|
||||
return (
|
||||
isinstance(value, list)
|
||||
and bool(value)
|
||||
and all(_is_chat_message(item) for item in value)
|
||||
)
|
||||
|
||||
|
||||
def _is_batched_chat_messages(value: Any) -> bool:
|
||||
return (
|
||||
isinstance(value, list)
|
||||
and bool(value)
|
||||
and all(_is_chat_messages(item) for item in value)
|
||||
)
|
||||
|
||||
|
||||
class EmbeddingChatRequest(
|
||||
PoolingBasicRequestMixin,
|
||||
ChatRequestMixin,
|
||||
EmbedRequestMixin,
|
||||
EmbeddingTokenizeParamsMixin,
|
||||
):
|
||||
"""OpenAI embeddings request with one top-level chat conversation."""
|
||||
|
||||
def to_pooling_params(self):
|
||||
return PoolingParams(
|
||||
task="embed",
|
||||
dimensions=self.dimensions,
|
||||
use_activation=self.use_activation,
|
||||
)
|
||||
|
||||
|
||||
class EmbeddingBatchChatRequest(
|
||||
PoolingBasicRequestMixin,
|
||||
ChatRequestOptionsMixin,
|
||||
EmbedRequestMixin,
|
||||
EmbeddingTokenizeParamsMixin,
|
||||
):
|
||||
"""OpenAI embeddings request with batched top-level chat conversations.
|
||||
|
||||
Mirrors ``BatchChatCompletionRequest`` by keeping batched conversations in
|
||||
``messages`` instead of introducing a separate batch-specific field.
|
||||
"""
|
||||
|
||||
messages: Sequence[
|
||||
Annotated[list[ChatCompletionMessageParam], Field(min_length=1)]
|
||||
] = Field(..., min_length=1)
|
||||
|
||||
def to_pooling_params(self):
|
||||
return PoolingParams(
|
||||
task="embed",
|
||||
dimensions=self.dimensions,
|
||||
use_activation=self.use_activation,
|
||||
)
|
||||
|
||||
|
||||
class EmbeddingChatInputRequest(
|
||||
EmbeddingChatRequest,
|
||||
):
|
||||
"""OpenAI embeddings request with one chat conversation in ``input``."""
|
||||
|
||||
input: list[ChatCompletionMessageParam]
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def normalize_input_messages(cls, data):
|
||||
if not isinstance(data, dict):
|
||||
return data
|
||||
|
||||
if "messages" in data or "input" not in data:
|
||||
return data
|
||||
|
||||
input_data = data["input"]
|
||||
if not _is_chat_messages(input_data):
|
||||
return data
|
||||
|
||||
normalized = dict(data)
|
||||
normalized["messages"] = input_data
|
||||
return normalized
|
||||
|
||||
|
||||
class EmbeddingBatchChatInputRequest(EmbeddingBatchChatRequest):
|
||||
"""OpenAI embeddings request with batched chat conversations in ``input``."""
|
||||
|
||||
input: Sequence[
|
||||
Annotated[list[ChatCompletionMessageParam], Field(min_length=1)]
|
||||
] = Field(..., min_length=1)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def normalize_input_messages(cls, data):
|
||||
if not isinstance(data, dict):
|
||||
return data
|
||||
|
||||
if "messages" in data or "input" not in data:
|
||||
return data
|
||||
|
||||
input_data = data["input"]
|
||||
if not _is_batched_chat_messages(input_data):
|
||||
return data
|
||||
|
||||
normalized = dict(data)
|
||||
normalized["messages"] = input_data
|
||||
return normalized
|
||||
|
||||
|
||||
EmbeddingRequest: TypeAlias = Annotated[
|
||||
EmbeddingCompletionRequest
|
||||
| EmbeddingChatRequest
|
||||
| EmbeddingBatchChatRequest
|
||||
| EmbeddingChatInputRequest
|
||||
| EmbeddingBatchChatInputRequest,
|
||||
BeforeValidator(reject_removed_pooling_parameters),
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# OpenAI /v1/embeddings — response models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class EmbeddingResponseData(OpenAIBaseModel):
|
||||
index: int
|
||||
object: str = "embedding"
|
||||
embedding: list[float] | str
|
||||
|
||||
|
||||
class EmbeddingResponse(OpenAIBaseModel):
|
||||
id: str = Field(default_factory=lambda: f"embd-{random_uuid()}")
|
||||
object: str = "list"
|
||||
created: int = Field(default_factory=lambda: int(time.time()))
|
||||
model: str | None = None
|
||||
data: list[EmbeddingResponseData]
|
||||
usage: UsageInfo
|
||||
|
||||
|
||||
class EmbeddingBytesResponse(OpenAIBaseModel):
|
||||
content: list[bytes]
|
||||
headers: dict[str, str] | None = None
|
||||
media_type: str = "application/octet-stream"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cohere /v2/embed — request models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
CohereEmbeddingType = Literal[
|
||||
"float",
|
||||
"binary",
|
||||
"ubinary",
|
||||
"base64",
|
||||
]
|
||||
CohereTruncate = Literal["NONE", "START", "END"]
|
||||
|
||||
|
||||
class CohereEmbedContent(BaseModel):
|
||||
type: Literal["text", "image_url"]
|
||||
text: str | None = None
|
||||
image_url: dict[str, str] | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_content_payload(self):
|
||||
if self.type == "text":
|
||||
if self.text is None:
|
||||
raise ValueError("CohereEmbedContent with type='text' requires text")
|
||||
elif not self.image_url or not self.image_url.get("url"):
|
||||
raise ValueError(
|
||||
"CohereEmbedContent with type='image_url' requires image_url.url"
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
class CohereEmbedInput(BaseModel):
|
||||
content: list[CohereEmbedContent]
|
||||
|
||||
|
||||
class CohereEmbedRequest(BaseModel):
|
||||
model: str | None = None
|
||||
input_type: str | None = None
|
||||
texts: list[str] | None = None
|
||||
images: list[str] | None = None
|
||||
inputs: list[CohereEmbedInput] | None = None
|
||||
output_dimension: int | None = None
|
||||
embedding_types: list[CohereEmbeddingType] | None = None
|
||||
truncate: CohereTruncate = "END"
|
||||
max_tokens: int | None = None
|
||||
priority: int = Field(default=0, ge=-(2**63), le=2**63 - 1)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_input_fields(self):
|
||||
input_fields = (self.texts, self.images, self.inputs)
|
||||
provided_fields = [field for field in input_fields if field is not None]
|
||||
if len(provided_fields) != 1 or not provided_fields[0]:
|
||||
raise ValueError(
|
||||
"Exactly one of texts, images, or inputs must be provided, "
|
||||
"and it must be non-empty"
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cohere /v2/embed — response models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class CohereApiVersion(BaseModel):
|
||||
version: str = "2"
|
||||
|
||||
|
||||
class CohereBilledUnits(BaseModel):
|
||||
input_tokens: int | None = None
|
||||
image_tokens: int | None = None
|
||||
|
||||
|
||||
class CohereMeta(BaseModel):
|
||||
api_version: CohereApiVersion = Field(default_factory=CohereApiVersion)
|
||||
billed_units: CohereBilledUnits | None = None
|
||||
|
||||
|
||||
class CohereEmbedByTypeEmbeddings(BaseModel):
|
||||
# The field name ``float`` shadows the builtin type, so the annotation
|
||||
# must use ``builtins.float`` to avoid a self-referential type error.
|
||||
float: list[list[builtins.float]] | None = None
|
||||
binary: list[list[int]] | None = None
|
||||
ubinary: list[list[int]] | None = None
|
||||
base64: list[str] | None = None
|
||||
|
||||
|
||||
class CohereEmbedResponse(BaseModel):
|
||||
id: str = Field(default_factory=lambda: f"embd-{random_uuid()}")
|
||||
embeddings: CohereEmbedByTypeEmbeddings
|
||||
texts: list[str] | None = None
|
||||
meta: CohereMeta | None = None
|
||||
response_type: Literal["embeddings_by_type"] = "embeddings_by_type"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cohere embedding type conversion helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_UNSIGNED_TO_SIGNED_DIFF = 1 << 7 # 128
|
||||
|
||||
|
||||
def _pack_binary_embeddings(
|
||||
float_embeddings: list[list[float]],
|
||||
signed: bool,
|
||||
) -> list[list[int]]:
|
||||
"""Bit-pack float embeddings: positive -> 1, negative -> 0.
|
||||
|
||||
Bits are packed MSB-first, eight per byte.
|
||||
"""
|
||||
if not float_embeddings:
|
||||
return []
|
||||
|
||||
array = np.asarray(float_embeddings, dtype=np.float64)
|
||||
if array.ndim != 2:
|
||||
raise ValueError(
|
||||
f"Expected a 2D batch of embeddings, but got {array.ndim}D input."
|
||||
)
|
||||
|
||||
dim = array.shape[1]
|
||||
if dim % 8 != 0:
|
||||
raise ValueError(
|
||||
"Embedding dimension must be a multiple of 8 for binary "
|
||||
f"embedding types, but got {dim}."
|
||||
)
|
||||
|
||||
packed = np.packbits(array >= 0, axis=-1)
|
||||
if signed:
|
||||
packed = packed.astype(np.int16) - _UNSIGNED_TO_SIGNED_DIFF
|
||||
return packed.tolist()
|
||||
|
||||
|
||||
def _encode_base64_embeddings(
|
||||
float_embeddings: list[list[float]],
|
||||
) -> list[str]:
|
||||
"""Encode float embeddings as base64 (little-endian float32)."""
|
||||
result: list[str] = []
|
||||
for embedding in float_embeddings:
|
||||
buf = struct.pack(f"<{len(embedding)}f", *embedding)
|
||||
result.append(base64.b64encode(buf).decode("utf-8"))
|
||||
return result
|
||||
|
||||
|
||||
def build_typed_embeddings(
|
||||
float_embeddings: list[list[float]],
|
||||
embedding_types: Sequence[str],
|
||||
) -> CohereEmbedByTypeEmbeddings:
|
||||
"""Convert float embeddings to all requested Cohere embedding types."""
|
||||
result = CohereEmbedByTypeEmbeddings()
|
||||
|
||||
for emb_type in embedding_types:
|
||||
if emb_type == "float":
|
||||
result.float = float_embeddings
|
||||
elif emb_type == "binary":
|
||||
result.binary = _pack_binary_embeddings(float_embeddings, signed=True)
|
||||
elif emb_type == "ubinary":
|
||||
result.ubinary = _pack_binary_embeddings(float_embeddings, signed=False)
|
||||
elif emb_type == "base64":
|
||||
result.base64 = _encode_base64_embeddings(float_embeddings)
|
||||
|
||||
return result
|
||||
@@ -1,220 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import TypeAlias, assert_never, cast
|
||||
|
||||
from fastapi.responses import JSONResponse, Response, StreamingResponse
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import PoolingRequestOutput
|
||||
from vllm.utils.serial_utils import EmbedDType, Endianness
|
||||
|
||||
from ..base.io_processor import PoolingIOProcessor
|
||||
from ..base.serving import PoolingServing
|
||||
from ..typing import PoolingServeContext
|
||||
from ..utils import (
|
||||
BytesEncodingFormat,
|
||||
JsonEncodingFormat,
|
||||
build_pooling_bytes_streaming_response,
|
||||
encode_pooling_output_float,
|
||||
encode_pooling_output_float_or_ndarray,
|
||||
get_json_response_cls,
|
||||
get_pooling_output_encoder,
|
||||
get_pooling_usage,
|
||||
)
|
||||
from .io_processor import EmbedIOProcessor
|
||||
from .protocol import (
|
||||
CohereBilledUnits,
|
||||
CohereEmbedRequest,
|
||||
CohereEmbedResponse,
|
||||
CohereMeta,
|
||||
EmbeddingRequest,
|
||||
EmbeddingResponse,
|
||||
EmbeddingResponseData,
|
||||
build_typed_embeddings,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
EmbeddingServeContext: TypeAlias = PoolingServeContext[EmbeddingRequest]
|
||||
|
||||
|
||||
class ServingEmbedding(PoolingServing):
|
||||
"""Embedding API supporting both OpenAI and Cohere formats."""
|
||||
|
||||
request_id_prefix = "embd"
|
||||
io_processor: EmbedIOProcessor
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
self.json_response_cls = get_json_response_cls()
|
||||
|
||||
def init_io_processor(self, *args, **kwargs) -> EmbedIOProcessor:
|
||||
return EmbedIOProcessor(*args, **kwargs)
|
||||
|
||||
async def _preprocessing(
|
||||
self, io_processor: PoolingIOProcessor, ctx: PoolingServeContext
|
||||
):
|
||||
await super()._preprocessing(io_processor, ctx)
|
||||
self.io_processor.maybe_pre_process_chunked(ctx)
|
||||
|
||||
def _build_response(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
) -> Response:
|
||||
if isinstance(ctx.request, CohereEmbedRequest):
|
||||
return self._build_cohere_response_from_ctx(ctx)
|
||||
return self._build_openai_response(ctx)
|
||||
|
||||
def _build_openai_response(
|
||||
self,
|
||||
ctx: EmbeddingServeContext,
|
||||
) -> JSONResponse | StreamingResponse:
|
||||
encoding_format = ctx.request.encoding_format
|
||||
embed_dtype = ctx.request.embed_dtype
|
||||
endianness = ctx.request.endianness
|
||||
|
||||
if encoding_format == "float" or encoding_format == "base64":
|
||||
return self._openai_json_response(
|
||||
ctx.final_res_batch,
|
||||
ctx.request_id,
|
||||
ctx.created_time,
|
||||
ctx.model_name,
|
||||
encoding_format,
|
||||
embed_dtype,
|
||||
endianness,
|
||||
)
|
||||
|
||||
if encoding_format == "bytes" or encoding_format == "bytes_only":
|
||||
return self._openai_bytes_response(
|
||||
ctx.final_res_batch,
|
||||
ctx.request_id,
|
||||
ctx.created_time,
|
||||
ctx.model_name,
|
||||
encoding_format,
|
||||
embed_dtype,
|
||||
endianness,
|
||||
)
|
||||
|
||||
assert_never(encoding_format)
|
||||
|
||||
def _openai_json_response(
|
||||
self,
|
||||
final_res_batch: list[PoolingRequestOutput],
|
||||
request_id: str,
|
||||
created_time: int,
|
||||
model_name: str,
|
||||
encoding_format: JsonEncodingFormat,
|
||||
embed_dtype: EmbedDType,
|
||||
endianness: Endianness,
|
||||
) -> JSONResponse:
|
||||
use_ndarray_response = (
|
||||
encoding_format == "float"
|
||||
and self.json_response_cls.__name__ == "ORJSONResponse"
|
||||
)
|
||||
if use_ndarray_response:
|
||||
ndarray_items: list[dict[str, object]] = []
|
||||
|
||||
for idx, final_res in enumerate(final_res_batch):
|
||||
item_dict = EmbeddingResponseData(
|
||||
index=idx,
|
||||
embedding=[],
|
||||
).model_dump()
|
||||
item_dict["embedding"] = encode_pooling_output_float_or_ndarray(
|
||||
final_res
|
||||
)
|
||||
ndarray_items.append(item_dict)
|
||||
ndarray_response = EmbeddingResponse(
|
||||
id=request_id,
|
||||
created=created_time,
|
||||
model=model_name,
|
||||
data=[], # type: ignore[arg-type]
|
||||
usage=get_pooling_usage(final_res_batch),
|
||||
).model_dump()
|
||||
ndarray_response["data"] = ndarray_items
|
||||
|
||||
return self.json_response_cls(content=ndarray_response)
|
||||
|
||||
encode_fn = get_pooling_output_encoder(
|
||||
encoding_format=encoding_format,
|
||||
embed_dtype=embed_dtype,
|
||||
endianness=endianness,
|
||||
)
|
||||
|
||||
items: list[EmbeddingResponseData] = []
|
||||
|
||||
for idx, final_res in enumerate(final_res_batch):
|
||||
item = EmbeddingResponseData(
|
||||
index=idx,
|
||||
embedding=cast(list[float] | str, encode_fn(final_res)),
|
||||
)
|
||||
|
||||
items.append(item)
|
||||
|
||||
response = EmbeddingResponse(
|
||||
id=request_id,
|
||||
created=created_time,
|
||||
model=model_name,
|
||||
data=items,
|
||||
usage=get_pooling_usage(final_res_batch),
|
||||
)
|
||||
return self.json_response_cls(content=response.model_dump())
|
||||
|
||||
def _openai_bytes_response(
|
||||
self,
|
||||
final_res_batch: list[PoolingRequestOutput],
|
||||
request_id: str,
|
||||
created_time: int,
|
||||
model_name: str,
|
||||
encoding_format: BytesEncodingFormat,
|
||||
embed_dtype: EmbedDType,
|
||||
endianness: Endianness,
|
||||
) -> StreamingResponse:
|
||||
return build_pooling_bytes_streaming_response(
|
||||
pooling_outputs=final_res_batch,
|
||||
request_id=request_id,
|
||||
created_time=created_time,
|
||||
model_name=model_name,
|
||||
encoding_format=encoding_format,
|
||||
embed_dtype=embed_dtype,
|
||||
endianness=endianness,
|
||||
)
|
||||
|
||||
def _build_cohere_response_from_ctx(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
) -> JSONResponse:
|
||||
request = ctx.request
|
||||
assert isinstance(request, CohereEmbedRequest)
|
||||
|
||||
all_floats = [
|
||||
cast(list[float], encode_pooling_output_float(out))
|
||||
for out in ctx.final_res_batch
|
||||
]
|
||||
total_tokens = get_pooling_usage(ctx.final_res_batch).prompt_tokens
|
||||
|
||||
has_image_input = request.images is not None or any(
|
||||
content.type == "image_url"
|
||||
for input_item in request.inputs or []
|
||||
for content in input_item.content
|
||||
)
|
||||
image_tokens = total_tokens if has_image_input else 0
|
||||
texts_echo = request.texts
|
||||
|
||||
embedding_types = request.embedding_types or ["float"]
|
||||
embeddings_obj = build_typed_embeddings(all_floats, embedding_types)
|
||||
|
||||
input_tokens = total_tokens - image_tokens
|
||||
response = CohereEmbedResponse(
|
||||
id=ctx.request_id,
|
||||
embeddings=embeddings_obj,
|
||||
texts=texts_echo,
|
||||
meta=CohereMeta(
|
||||
billed_units=CohereBilledUnits(
|
||||
input_tokens=input_tokens,
|
||||
image_tokens=image_tokens,
|
||||
),
|
||||
),
|
||||
)
|
||||
return self.json_response_cls(content=response.model_dump(exclude_none=True))
|
||||
@@ -1,272 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from fastapi import FastAPI
|
||||
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.plugins.io_processors import has_io_processor
|
||||
from vllm.renderers import BaseRenderer
|
||||
from vllm.renderers.chat_utils import ChatTemplateConfig
|
||||
from vllm.tasks import POOLING_TASKS, SCORE_TYPE_MAP, SupportedTask
|
||||
|
||||
from .base.io_processor import PoolingIOProcessor
|
||||
from .utils import enable_scoring_api
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from argparse import Namespace
|
||||
|
||||
from starlette.datastructures import State
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.serve.sagemaker.api_router import (
|
||||
EndpointFn,
|
||||
GetHandlerFn,
|
||||
RequestType,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
|
||||
else:
|
||||
RequestLogger = object
|
||||
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def init_pooling_io_processors(
|
||||
supported_tasks: tuple[SupportedTask, ...],
|
||||
vllm_config: VllmConfig,
|
||||
renderer: BaseRenderer,
|
||||
chat_template_config: ChatTemplateConfig,
|
||||
) -> dict[str, PoolingIOProcessor]:
|
||||
model_config = vllm_config.model_config
|
||||
processors: dict[str, type[PoolingIOProcessor]] = {}
|
||||
pooling_task = model_config.get_pooling_task(supported_tasks)
|
||||
|
||||
if pooling_task == "classify":
|
||||
from .classify.io_processor import ClassifyIOProcessor
|
||||
|
||||
processors["classify"] = ClassifyIOProcessor
|
||||
|
||||
if pooling_task == "token_classify":
|
||||
from .classify.io_processor import TokenClassifyIOProcessor
|
||||
|
||||
processors["token_classify"] = TokenClassifyIOProcessor
|
||||
|
||||
if pooling_task == "embed":
|
||||
from .embed.io_processor import EmbedIOProcessor
|
||||
|
||||
processors["embed"] = EmbedIOProcessor
|
||||
|
||||
if pooling_task == "token_embed":
|
||||
from .embed.io_processor import TokenEmbedIOProcessor
|
||||
|
||||
processors["token_embed"] = TokenEmbedIOProcessor
|
||||
|
||||
if pooling_task == "embed&token_classify":
|
||||
from .pooling.io_processor import UnsupportedCombinedTaskIOProcessor
|
||||
|
||||
processors[pooling_task] = UnsupportedCombinedTaskIOProcessor
|
||||
|
||||
has_plugin = has_io_processor(
|
||||
vllm_config,
|
||||
model_config.io_processor_plugin,
|
||||
)
|
||||
if has_plugin:
|
||||
from .pooling.io_processor import PluginWithIOProcessorPlugins
|
||||
|
||||
processors["plugin"] = PluginWithIOProcessorPlugins
|
||||
elif pooling_task == "plugin":
|
||||
from .pooling.io_processor import PluginWithoutIOProcessorPlugins
|
||||
|
||||
processors["plugin"] = PluginWithoutIOProcessorPlugins
|
||||
|
||||
if enable_scoring_api(supported_tasks, model_config):
|
||||
from .scoring.io_processor import ScoringIOProcessors
|
||||
|
||||
score_type: str | None = SCORE_TYPE_MAP.get(pooling_task, None) # type: ignore[arg-type]
|
||||
if score_type is not None and score_type in ScoringIOProcessors:
|
||||
processors[score_type] = ScoringIOProcessors[score_type]
|
||||
|
||||
if model_config.architecture == "JinaForRanking":
|
||||
from .embed.io_processor import JinaRankingTokenEmbedIOProcessor
|
||||
from .scoring.io_processor import ScoringIOProcessors
|
||||
|
||||
processors["token_embed"] = JinaRankingTokenEmbedIOProcessor
|
||||
processors["late-interaction"] = ScoringIOProcessors["jina-reranking-scoring"]
|
||||
|
||||
return {
|
||||
task: processor_cls(
|
||||
vllm_config=vllm_config,
|
||||
renderer=renderer,
|
||||
chat_template_config=chat_template_config,
|
||||
)
|
||||
for task, processor_cls in processors.items()
|
||||
}
|
||||
|
||||
|
||||
def register_pooling_api_routers(
|
||||
app: FastAPI,
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
model_config: ModelConfig | None = None,
|
||||
):
|
||||
if model_config is None:
|
||||
return
|
||||
|
||||
pooling_task = model_config.get_pooling_task(supported_tasks)
|
||||
|
||||
if pooling_task is not None:
|
||||
from .pooling.api_router import router as pooling_router
|
||||
|
||||
app.include_router(pooling_router)
|
||||
|
||||
if "classify" in supported_tasks:
|
||||
from .classify.api_router import (
|
||||
router as classify_router,
|
||||
)
|
||||
|
||||
app.include_router(classify_router)
|
||||
|
||||
if "embed" in supported_tasks:
|
||||
from .embed.api_router import router as embed_router
|
||||
|
||||
app.include_router(embed_router)
|
||||
|
||||
if enable_scoring_api(supported_tasks, model_config):
|
||||
from .scoring.api_router import router as score_router
|
||||
|
||||
app.include_router(score_router)
|
||||
|
||||
|
||||
def init_pooling_state(
|
||||
engine_client: "EngineClient",
|
||||
state: "State",
|
||||
args: "Namespace",
|
||||
request_logger: RequestLogger | None,
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
):
|
||||
model_config = engine_client.model_config
|
||||
if model_config is None:
|
||||
return
|
||||
|
||||
from vllm.renderers.chat_utils import load_chat_template
|
||||
from vllm.tasks import POOLING_TASKS
|
||||
|
||||
from .classify.serving import ServingClassification
|
||||
from .embed.serving import ServingEmbedding
|
||||
from .pooling.serving import ServingPooling
|
||||
from .scoring.serving import ServingScores
|
||||
|
||||
resolved_chat_template = load_chat_template(args.chat_template)
|
||||
pooling_task = model_config.get_pooling_task(supported_tasks)
|
||||
|
||||
chat_template_config = ChatTemplateConfig(
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
trust_request_mm_kwargs=args.trust_request_mm_kwargs,
|
||||
)
|
||||
|
||||
state.serving_pooling = (
|
||||
(
|
||||
ServingPooling(
|
||||
engine_client,
|
||||
state.openai_serving_models,
|
||||
supported_tasks=supported_tasks,
|
||||
request_logger=request_logger,
|
||||
chat_template_config=chat_template_config,
|
||||
)
|
||||
)
|
||||
if any(t in supported_tasks for t in POOLING_TASKS)
|
||||
else None
|
||||
)
|
||||
state.serving_embedding = (
|
||||
ServingEmbedding(
|
||||
engine_client,
|
||||
state.openai_serving_models,
|
||||
request_logger=request_logger,
|
||||
chat_template_config=chat_template_config,
|
||||
)
|
||||
if pooling_task == "embed"
|
||||
else None
|
||||
)
|
||||
state.serving_classification = (
|
||||
ServingClassification(
|
||||
engine_client,
|
||||
state.openai_serving_models,
|
||||
request_logger=request_logger,
|
||||
chat_template_config=chat_template_config,
|
||||
)
|
||||
if pooling_task == "classify"
|
||||
else None
|
||||
)
|
||||
state.serving_scores = (
|
||||
ServingScores(
|
||||
engine_client,
|
||||
state.openai_serving_models,
|
||||
supported_tasks=supported_tasks,
|
||||
request_logger=request_logger,
|
||||
chat_template_config=chat_template_config,
|
||||
enable_flash_late_interaction=getattr(
|
||||
args, "enable_flash_late_interaction", True
|
||||
),
|
||||
)
|
||||
if enable_scoring_api(supported_tasks, model_config)
|
||||
else None
|
||||
)
|
||||
|
||||
|
||||
def get_pooling_invocation_types(
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
model_config: ModelConfig | None = None,
|
||||
):
|
||||
# NOTE: Items defined earlier take higher priority
|
||||
invocation_types: list[tuple[RequestType, tuple[GetHandlerFn, EndpointFn]]] = []
|
||||
|
||||
if model_config is None:
|
||||
return invocation_types
|
||||
|
||||
pooling_task = model_config.get_pooling_task(supported_tasks)
|
||||
|
||||
if pooling_task == "embed":
|
||||
from .embed.api_router import create_embedding, embedding
|
||||
from .embed.protocol import EmbeddingRequest
|
||||
|
||||
invocation_types += [
|
||||
(EmbeddingRequest, (embedding, create_embedding)),
|
||||
]
|
||||
|
||||
if pooling_task == "classify":
|
||||
from .classify.api_router import classify, create_classify
|
||||
from .classify.protocol import ClassificationRequest
|
||||
|
||||
invocation_types += [
|
||||
(ClassificationRequest, (classify, create_classify)),
|
||||
]
|
||||
|
||||
if enable_scoring_api(supported_tasks, model_config):
|
||||
from .scoring.api_router import do_rerank, rerank
|
||||
from .scoring.protocol import RerankRequest
|
||||
|
||||
invocation_types += [
|
||||
(RerankRequest, (rerank, do_rerank)),
|
||||
]
|
||||
|
||||
from .scoring.api_router import create_score, score
|
||||
from .scoring.protocol import ScoreRequest
|
||||
|
||||
invocation_types += [
|
||||
(ScoreRequest, (score, create_score)),
|
||||
]
|
||||
|
||||
if any(task in POOLING_TASKS for task in supported_tasks):
|
||||
from .pooling.api_router import create_pooling, pooling
|
||||
from .pooling.protocol import PoolingRequest
|
||||
|
||||
invocation_types += [
|
||||
(PoolingRequest, (pooling, create_pooling)),
|
||||
]
|
||||
|
||||
return invocation_types
|
||||
@@ -1,92 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import replace
|
||||
from typing import cast
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.outputs import LateChunk, LateChunkingMetadata, PoolingRequestOutput
|
||||
from vllm.renderers import TokenizeParams
|
||||
|
||||
from .typing import AnyRenderParam, EncodeCMPLRenderParams
|
||||
|
||||
|
||||
def prepare_late_chunking_input(
|
||||
config: VllmConfig, render_params: AnyRenderParam
|
||||
) -> tuple[str, TokenizeParams]:
|
||||
"""Validate full-document text input before requesting tokenizer offsets."""
|
||||
params = render_params["params"]
|
||||
params.verify(config.model_config)
|
||||
if "prompts" not in render_params:
|
||||
raise VLLMValidationError("Late chunking requires plain-text input")
|
||||
prompt = cast(EncodeCMPLRenderParams, render_params)["prompts"]
|
||||
if (
|
||||
not isinstance(prompt, dict)
|
||||
or set(prompt) - {"prompt", "cache_salt"}
|
||||
or not isinstance(text := prompt.get("prompt"), str)
|
||||
or not text
|
||||
):
|
||||
raise VLLMValidationError("Late chunking requires nonempty plain-text input")
|
||||
tok_params = render_params["tok_params"]
|
||||
if (
|
||||
tok_params.pad_prompt_tokens is not None
|
||||
or tok_params.truncate_prompt_tokens is not None
|
||||
or tok_params.do_lower_case
|
||||
):
|
||||
raise VLLMValidationError(
|
||||
"Late chunking does not support input padding, truncation or "
|
||||
"renderer text normalization"
|
||||
)
|
||||
return text, replace(tok_params, return_token_offsets=True)
|
||||
|
||||
|
||||
def build_late_chunking_metadata(
|
||||
text: str,
|
||||
num_tokens: int,
|
||||
offsets: Sequence[tuple[int, int]] | None,
|
||||
chunk_size: int,
|
||||
) -> LateChunkingMetadata:
|
||||
"""Map chunks to original text, keeping special-token-only chunks."""
|
||||
if num_tokens <= 0 or offsets is None or len(offsets) != num_tokens:
|
||||
raise VLLMValidationError(
|
||||
"Late chunking requires a fast tokenizer with offsets aligned to "
|
||||
"all input tokens"
|
||||
)
|
||||
previous_start = previous_end = 0
|
||||
for start, end in offsets:
|
||||
if not 0 <= start <= end <= len(text):
|
||||
raise VLLMValidationError("Token offsets are not aligned to the input text")
|
||||
if start < end:
|
||||
if start < previous_start or end < previous_end:
|
||||
raise VLLMValidationError("Token offsets are not in source-text order")
|
||||
previous_start, previous_end = start, end
|
||||
|
||||
chunks = []
|
||||
for start in range(0, num_tokens, chunk_size):
|
||||
end = min(start + chunk_size, num_tokens)
|
||||
source = [(a, b) for a, b in offsets[start:end] if a < b]
|
||||
char_range = (
|
||||
(min(a for a, _ in source), max(b for _, b in source)) if source else None
|
||||
)
|
||||
chunks.append(LateChunk(token_range=(start, end), char_range=char_range))
|
||||
return LateChunkingMetadata(
|
||||
chunk_size=chunk_size, input_tokens=num_tokens, chunks=chunks
|
||||
)
|
||||
|
||||
|
||||
def attach_late_chunking_metadata(
|
||||
output: PoolingRequestOutput, metadata: LateChunkingMetadata
|
||||
) -> None:
|
||||
"""Attach ranges only to successful, complete chunk-vector results."""
|
||||
if not output.finished or output.error is not None:
|
||||
return
|
||||
data = output.outputs.data
|
||||
if (
|
||||
data.ndim != 2
|
||||
or data.shape[0] != len(metadata.chunks)
|
||||
or len(output.prompt_token_ids) != metadata.input_tokens
|
||||
):
|
||||
raise ValueError("Late-chunking output does not match its token/source ranges")
|
||||
output.late_chunking = metadata
|
||||
@@ -1,479 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable, Sequence
|
||||
from typing import Any
|
||||
|
||||
from tqdm.auto import tqdm
|
||||
|
||||
from vllm.entrypoints.common.offline import OfflineInferenceMixin
|
||||
from vllm.inputs import DataPrompt, PromptType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.outputs import (
|
||||
ClassificationRequestOutput,
|
||||
EmbeddingRequestOutput,
|
||||
PoolingRequestOutput,
|
||||
ScoringRequestOutput,
|
||||
)
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.renderers.chat_utils import ChatTemplateConfig
|
||||
from vllm.tasks import SCORE_TYPE_MAP, PoolingTask, SupportedTask
|
||||
|
||||
from .base.io_processor import PoolingIOProcessor
|
||||
from .factories import init_pooling_io_processors
|
||||
from .scoring.io_processor import ScoringIOProcessor
|
||||
from .scoring.typing import ScoreInput
|
||||
from .typing import (
|
||||
AnyOfflineInputsContext,
|
||||
OfflineEncodeInputsContext,
|
||||
OfflineOutputsContext,
|
||||
OfflinePluginInputsContext,
|
||||
OfflineScoringInputsContext,
|
||||
RequestFactory,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class PoolingOfflineMixin(OfflineInferenceMixin):
|
||||
"""Offline inference for pooling models."""
|
||||
|
||||
runner_type: str
|
||||
chat_template: str | None
|
||||
supported_tasks: tuple[SupportedTask, ...]
|
||||
|
||||
def __init__(self):
|
||||
self.pooling_task = self.model_config.get_pooling_task(self.supported_tasks)
|
||||
if self.pooling_task is not None:
|
||||
logger.info("Supported pooling task: %s", self.pooling_task)
|
||||
|
||||
self.chat_template_config = ChatTemplateConfig(chat_template=self.chat_template)
|
||||
self.pooling_io_processors = init_pooling_io_processors(
|
||||
supported_tasks=self.supported_tasks,
|
||||
vllm_config=self.llm_engine.vllm_config,
|
||||
renderer=self.renderer,
|
||||
chat_template_config=self.chat_template_config,
|
||||
)
|
||||
|
||||
# Use thread pool executor to accelerate preprocessing.
|
||||
self._executor = self.renderer._executor
|
||||
|
||||
def encode(
|
||||
self,
|
||||
prompts: PromptType | Sequence[PromptType] | DataPrompt,
|
||||
pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,
|
||||
*,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
lora_request: list[LoRARequest] | LoRARequest | None = None,
|
||||
pooling_task: PoolingTask | None = None,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
) -> list[PoolingRequestOutput]:
|
||||
"""Apply pooling to the hidden states corresponding to the input
|
||||
prompts.
|
||||
|
||||
This class automatically batches the given prompts, considering
|
||||
the memory constraint. For the best performance, put all of your prompts
|
||||
into a single list and pass it to this method.
|
||||
|
||||
Args:
|
||||
prompts: The prompts to the LLM. You may pass a sequence of prompts
|
||||
for batch inference. See [PromptType][vllm.inputs.PromptType]
|
||||
for more details about the format of each prompt.
|
||||
pooling_params: The pooling parameters for pooling. If None, we
|
||||
use the default pooling parameters.
|
||||
use_tqdm: If `True`, shows a tqdm progress bar.
|
||||
If a callable (e.g., `functools.partial(tqdm, leave=False)`),
|
||||
it is used to create the progress bar.
|
||||
If `False`, no progress bar is created.
|
||||
lora_request: LoRA request to use for generation, if any.
|
||||
pooling_task: Override the pooling task to use.
|
||||
tokenization_kwargs: Overrides for `tokenizer.encode`.
|
||||
|
||||
Returns:
|
||||
A list of `PoolingRequestOutput` objects containing the
|
||||
pooled hidden states in the same order as the input prompts.
|
||||
|
||||
"""
|
||||
if isinstance(prompts, dict) and "data" in prompts and pooling_task != "plugin":
|
||||
raise ValueError(
|
||||
"The 'data' field is only supported for the 'plugin' pooling task."
|
||||
)
|
||||
self._verify_pooling_task(pooling_task)
|
||||
assert pooling_task is not None and pooling_task in self.pooling_io_processors
|
||||
|
||||
io_processor = self.pooling_io_processors[pooling_task]
|
||||
|
||||
ctx: AnyOfflineInputsContext
|
||||
if isinstance(prompts, dict) and "data" in prompts:
|
||||
ctx = OfflinePluginInputsContext(
|
||||
pooling_task=pooling_task,
|
||||
prompts=prompts, # type: ignore[arg-type]
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
pooling_params=pooling_params,
|
||||
lora_request=lora_request,
|
||||
priorities=None,
|
||||
)
|
||||
else:
|
||||
ctx = OfflineEncodeInputsContext(
|
||||
pooling_task=pooling_task,
|
||||
prompts=prompts,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
pooling_params=pooling_params,
|
||||
lora_request=lora_request,
|
||||
priorities=None,
|
||||
)
|
||||
|
||||
request_factory, num_requests = io_processor.get_request_factory_offline(ctx)
|
||||
outputs = self._run_tiling_engine(
|
||||
io_processor, request_factory, num_requests, use_tqdm=use_tqdm
|
||||
)
|
||||
outputs = io_processor.post_process_offline(
|
||||
ctx=OfflineOutputsContext(outputs=outputs, late_chunking=ctx.late_chunking)
|
||||
)
|
||||
return outputs
|
||||
|
||||
def _verify_pooling_task(self, pooling_task: PoolingTask | None):
|
||||
if self.runner_type != "pooling":
|
||||
raise ValueError(
|
||||
"LLM.encode() is only supported for pooling models. "
|
||||
"Try passing `--runner pooling` to use the model as a "
|
||||
"pooling model."
|
||||
)
|
||||
|
||||
if pooling_task is None:
|
||||
raise ValueError(
|
||||
"""
|
||||
pooling_task required for `LLM.encode`.
|
||||
Please use one of the more specific methods or set the pooling_task when using `LLM.encode`:
|
||||
- For embeddings, use `LLM.embed(...)` or `pooling_task="embed"`.
|
||||
- For classification logits, use `LLM.classify(...)` or `pooling_task="classify"`.
|
||||
- For similarity scores, use `LLM.score(...)`.
|
||||
- For rewards, `pooling_task="classify"` or `pooling_task="token_classify"`.
|
||||
- For token classification, use `pooling_task="token_classify"`.
|
||||
- For multi-vector retrieval, use `pooling_task="token_embed"`.
|
||||
""" # noqa: E501
|
||||
)
|
||||
|
||||
if (
|
||||
pooling_task in ("embed", "token_embed")
|
||||
and pooling_task not in self.supported_tasks
|
||||
):
|
||||
raise ValueError(
|
||||
"Embedding API is not supported by this model. "
|
||||
"Try converting the model using `--convert embed`."
|
||||
)
|
||||
|
||||
if (
|
||||
pooling_task in ("classify", "token_classify")
|
||||
and pooling_task not in self.supported_tasks
|
||||
):
|
||||
raise ValueError(
|
||||
"Classification API is not supported by this model. "
|
||||
"Try converting the model using `--convert classify`."
|
||||
)
|
||||
|
||||
# plugin task uses io_processor.parse_data to verify inputs
|
||||
if pooling_task != "plugin" and pooling_task != self.pooling_task:
|
||||
if pooling_task not in self.supported_tasks:
|
||||
raise ValueError(
|
||||
f"Unsupported task: {pooling_task!r} "
|
||||
f"Supported tasks: {self.supported_tasks}"
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Try switching the model's pooling_task "
|
||||
f'via `PoolerConfig(task="{pooling_task}")`'
|
||||
)
|
||||
|
||||
if pooling_task == "plugin" and "plugin" not in self.pooling_io_processors:
|
||||
raise ValueError(
|
||||
"No IOProcessor plugin installed. Please refer "
|
||||
"to the documentation and to the "
|
||||
"'prithvi_geospatial_mae_io_processor' "
|
||||
"offline inference example for more details."
|
||||
)
|
||||
|
||||
def embed(
|
||||
self,
|
||||
prompts: PromptType | Sequence[PromptType],
|
||||
*,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,
|
||||
lora_request: list[LoRARequest] | LoRARequest | None = None,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
) -> list[EmbeddingRequestOutput]:
|
||||
"""Generate an embedding vector for each prompt.
|
||||
|
||||
This class automatically batches the given prompts, considering
|
||||
the memory constraint. For the best performance, put all of your prompts
|
||||
into a single list and pass it to this method.
|
||||
|
||||
Args:
|
||||
prompts: The prompts to the LLM. You may pass a sequence of prompts
|
||||
for batch inference. See [PromptType][vllm.inputs.PromptType]
|
||||
for more details about the format of each prompt.
|
||||
pooling_params: The pooling parameters for pooling. If None, we
|
||||
use the default pooling parameters.
|
||||
use_tqdm: If `True`, shows a tqdm progress bar.
|
||||
If a callable (e.g., `functools.partial(tqdm, leave=False)`),
|
||||
it is used to create the progress bar.
|
||||
If `False`, no progress bar is created.
|
||||
lora_request: LoRA request to use for generation, if any.
|
||||
tokenization_kwargs: Overrides for `tokenizer.encode`.
|
||||
|
||||
Returns:
|
||||
A list of `EmbeddingRequestOutput` objects containing the
|
||||
embedding vectors in the same order as the input prompts.
|
||||
|
||||
"""
|
||||
items = self.encode(
|
||||
prompts,
|
||||
use_tqdm=use_tqdm,
|
||||
pooling_params=pooling_params,
|
||||
lora_request=lora_request,
|
||||
pooling_task="embed",
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
|
||||
return [EmbeddingRequestOutput.from_base(item) for item in items]
|
||||
|
||||
def classify(
|
||||
self,
|
||||
prompts: PromptType | Sequence[PromptType],
|
||||
*,
|
||||
pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
lora_request: list[LoRARequest] | LoRARequest | None = None,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
) -> list[ClassificationRequestOutput]:
|
||||
"""Generate class logits for each prompt.
|
||||
|
||||
This class automatically batches the given prompts, considering
|
||||
the memory constraint. For the best performance, put all of your prompts
|
||||
into a single list and pass it to this method.
|
||||
|
||||
Args:
|
||||
prompts: The prompts to the LLM. You may pass a sequence of prompts
|
||||
for batch inference. See [PromptType][vllm.inputs.PromptType]
|
||||
for more details about the format of each prompt.
|
||||
pooling_params: The pooling parameters for pooling. If None, we
|
||||
use the default pooling parameters.
|
||||
use_tqdm: If `True`, shows a tqdm progress bar.
|
||||
If a callable (e.g., `functools.partial(tqdm, leave=False)`),
|
||||
it is used to create the progress bar.
|
||||
If `False`, no progress bar is created.
|
||||
lora_request: LoRA request to use for generation, if any.
|
||||
tokenization_kwargs: Overrides for `tokenizer.encode`.
|
||||
|
||||
Returns:
|
||||
A list of `ClassificationRequestOutput` objects containing the
|
||||
embedding vectors in the same order as the input prompts.
|
||||
|
||||
"""
|
||||
items = self.encode(
|
||||
prompts,
|
||||
use_tqdm=use_tqdm,
|
||||
pooling_params=pooling_params,
|
||||
lora_request=lora_request,
|
||||
pooling_task="classify",
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
|
||||
return [ClassificationRequestOutput.from_base(item) for item in items]
|
||||
|
||||
def score(
|
||||
self,
|
||||
data_1: ScoreInput | list[ScoreInput],
|
||||
data_2: ScoreInput | list[ScoreInput],
|
||||
/,
|
||||
*,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
pooling_params: PoolingParams | None = None,
|
||||
lora_request: list[LoRARequest] | LoRARequest | None = None,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
chat_template: str | None = None,
|
||||
) -> list[ScoringRequestOutput]:
|
||||
"""Generate similarity scores for all pairs `<text,text_pair>` or
|
||||
`<multi-modal data, multi-modal data pair>`.
|
||||
|
||||
The inputs can be `1 -> 1`, `1 -> N` or `N -> N`.
|
||||
In the `1 - N` case the `data_1` input will be replicated `N`
|
||||
times to pair with the `data_2` inputs.
|
||||
The input pairs are used to build a list of prompts for the
|
||||
cross encoder model. This class automatically batches the prompts,
|
||||
considering the memory constraint. For the best performance, put all
|
||||
of your inputs into a single list and pass it to this method.
|
||||
|
||||
Supports both text and multi-modal data (images, etc.) when used with
|
||||
appropriate multi-modal models. For multi-modal inputs, ensure the
|
||||
prompt structure matches the model's expected input format.
|
||||
|
||||
Args:
|
||||
data_1: Can be a single prompt, a list of prompts or
|
||||
`ScoreMultiModalParam`, which can contain either text or
|
||||
multi-modal data. When a list, it must have the same length as
|
||||
the `data_2` list.
|
||||
data_2: The data to pair with the query to form the input to
|
||||
the LLM. Can be text or multi-modal data. See [PromptType]
|
||||
[vllm.inputs.PromptType] for more details about the format of
|
||||
each prompt.
|
||||
pooling_params: The pooling parameters for pooling. If None, we
|
||||
use the default pooling parameters.
|
||||
use_tqdm: If `True`, shows a tqdm progress bar.
|
||||
If a callable (e.g., `functools.partial(tqdm, leave=False)`),
|
||||
it is used to create the progress bar.
|
||||
If `False`, no progress bar is created.
|
||||
lora_request: LoRA request to use for generation, if any.
|
||||
chat_template: The chat template to use for scoring. If None, use
|
||||
the chat template passed to the LLM constructor.
|
||||
tokenization_kwargs: Overrides for `tokenizer.encode`.
|
||||
|
||||
Returns:
|
||||
A list of `ScoringRequestOutput` objects containing the
|
||||
generated scores in the same order as the input prompts.
|
||||
|
||||
"""
|
||||
if self.runner_type != "pooling":
|
||||
raise ValueError(
|
||||
"LLM.score() is only supported for pooling models. "
|
||||
"Try passing `--runner pooling` to use the model as a "
|
||||
"pooling model."
|
||||
)
|
||||
|
||||
score_type: str | None = SCORE_TYPE_MAP.get(self.pooling_task, None) # type: ignore[arg-type]
|
||||
if (
|
||||
score_type == "cross-encoder"
|
||||
and getattr(self.model_config.hf_config, "num_labels", 0) != 1
|
||||
):
|
||||
raise ValueError("Scoring API is only enabled for num_labels == 1.")
|
||||
|
||||
if score_type is None or score_type not in self.pooling_io_processors:
|
||||
raise ValueError("This model does not support the Scoring API.")
|
||||
|
||||
io_processor = self.pooling_io_processors[score_type]
|
||||
assert isinstance(io_processor, ScoringIOProcessor)
|
||||
|
||||
scoring_data = io_processor.valid_inputs(data_1, data_2)
|
||||
n_queries = len(scoring_data.data_1)
|
||||
|
||||
if pooling_params is None:
|
||||
pooling_params = PoolingParams()
|
||||
|
||||
assert isinstance(pooling_params, PoolingParams)
|
||||
pooling_task = io_processor.pooling_task
|
||||
# Clone to avoid modifying the caller's pooling parameters.
|
||||
pooling_params = pooling_params.clone()
|
||||
if pooling_params.task is None:
|
||||
pooling_params.task = pooling_task
|
||||
elif pooling_params.task != pooling_task:
|
||||
msg = (
|
||||
f"You cannot overwrite {pooling_params.task=!r} with {pooling_task=!r}!"
|
||||
)
|
||||
raise ValueError(msg)
|
||||
|
||||
ctx = OfflineScoringInputsContext(
|
||||
pooling_task=pooling_task,
|
||||
scoring_data=scoring_data,
|
||||
pooling_params=pooling_params,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
lora_request=lora_request,
|
||||
chat_template=chat_template,
|
||||
priorities=None,
|
||||
)
|
||||
|
||||
request_factory, num_requests = io_processor.get_request_factory_offline(ctx)
|
||||
outputs = self._run_tiling_engine(
|
||||
io_processor, request_factory, num_requests, use_tqdm=use_tqdm
|
||||
)
|
||||
|
||||
outputs = io_processor.post_process_offline(
|
||||
ctx=OfflineOutputsContext(outputs=outputs, n_queries=n_queries),
|
||||
)
|
||||
|
||||
return [ScoringRequestOutput.from_base(item) for item in outputs]
|
||||
|
||||
def _run_tiling_engine(
|
||||
self,
|
||||
io_processor: PoolingIOProcessor,
|
||||
request_factory: RequestFactory,
|
||||
num_requests: int,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
):
|
||||
if num_requests == 0:
|
||||
raise ValueError("You must pass at least one prompt")
|
||||
|
||||
# Keeping max_num_seqs * 2 requests in the core can already saturate the core.
|
||||
# Therefore, keep most requests waiting outside the core.
|
||||
max_requests_in_core = (
|
||||
self.llm_engine.vllm_config.scheduler_config.max_num_seqs * 2
|
||||
)
|
||||
num_requests_in_core = 0
|
||||
num_waited_requests = num_requests
|
||||
|
||||
if use_tqdm:
|
||||
tqdm_func = use_tqdm if callable(use_tqdm) else tqdm
|
||||
pbar = tqdm_func(
|
||||
total=num_requests,
|
||||
desc="Processed prompts",
|
||||
dynamic_ncols=True,
|
||||
postfix=f"est. speed input: {0:.2f} toks/s, output: {0:.2f} toks/s",
|
||||
)
|
||||
|
||||
outputs: list[PoolingRequestOutput] = []
|
||||
added_request_ids: set[str] = set()
|
||||
|
||||
it = self._executor.map(io_processor.render, request_factory())
|
||||
|
||||
try:
|
||||
while num_waited_requests or self.llm_engine.has_unfinished_requests():
|
||||
requests = []
|
||||
for _ in range(max_requests_in_core - num_requests_in_core):
|
||||
if num_waited_requests == 0:
|
||||
break
|
||||
try:
|
||||
request = next(it)
|
||||
requests.append(request)
|
||||
except StopIteration:
|
||||
num_waited_requests = 0
|
||||
break
|
||||
|
||||
num_waited_requests -= 1
|
||||
num_requests_in_core += 1
|
||||
|
||||
if requests:
|
||||
request_ids = self._render_and_add_requests(
|
||||
prompts=[x["prompts"] for x in requests],
|
||||
params=[x["params"] for x in requests],
|
||||
lora_requests=[x["lora_requests"] for x in requests],
|
||||
priorities=[x["priorities"] for x in requests],
|
||||
)
|
||||
|
||||
for request_id in request_ids:
|
||||
# undo assign_request_id
|
||||
request_id = request_id.split("-", 1)[0]
|
||||
added_request_ids.add(request_id)
|
||||
|
||||
step_outputs = self.llm_engine.step()
|
||||
for output in step_outputs:
|
||||
assert isinstance(output, PoolingRequestOutput)
|
||||
assert output.finished
|
||||
outputs.append(output)
|
||||
added_request_ids.discard(output.request_id)
|
||||
num_requests_in_core -= 1
|
||||
|
||||
if use_tqdm:
|
||||
pbar.update(1)
|
||||
if pbar.n == num_requests:
|
||||
pbar.refresh()
|
||||
|
||||
except Exception:
|
||||
if added_request_ids:
|
||||
self.llm_engine.abort_request(list(added_request_ids))
|
||||
raise
|
||||
|
||||
finally:
|
||||
if use_tqdm:
|
||||
pbar.close()
|
||||
|
||||
return sorted(outputs, key=lambda x: int(x.request_id))
|
||||
@@ -1,39 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from http import HTTPStatus
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
with_cancellation,
|
||||
)
|
||||
|
||||
from .protocol import PoolingRequest
|
||||
from .serving import ServingPooling
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def pooling(request: Request) -> ServingPooling:
|
||||
handler = getattr(request.app.state, "serving_pooling", None)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Pooling API")
|
||||
return handler
|
||||
|
||||
|
||||
@router.post(
|
||||
"/pooling",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def create_pooling(request: PoolingRequest, raw_request: Request):
|
||||
handler = pooling(raw_request)
|
||||
return await handler(request, raw_request)
|
||||
@@ -1,187 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Sequence
|
||||
from typing import Any
|
||||
|
||||
from vllm import PoolingParams, PoolingRequestOutput
|
||||
from vllm.plugins.io_processors import get_io_processor
|
||||
from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq
|
||||
|
||||
from ..base.io_processor import PoolingIOProcessor
|
||||
from ..typing import (
|
||||
AnyOfflineInputsContext,
|
||||
AnyRenderParam,
|
||||
EncodeCMPLRenderParams,
|
||||
OfflineEncodeInputsContext,
|
||||
OfflineOutputsContext,
|
||||
OfflinePluginInputsContext,
|
||||
PoolingServeContext,
|
||||
RequestFactory,
|
||||
)
|
||||
from .protocol import IOProcessorRequest, IOProcessorResponse
|
||||
|
||||
|
||||
class UnsupportedCombinedTaskIOProcessor(PoolingIOProcessor):
|
||||
name = "embed&token_classify"
|
||||
|
||||
def create_pooling_params(self, request):
|
||||
raise ValueError(
|
||||
"The 'embed&token_classify' pooling task is only available "
|
||||
"through an IO processor plugin. Send a plugin request with "
|
||||
"a 'data' field, "
|
||||
"or select a concrete task with --pooler-config.task."
|
||||
)
|
||||
|
||||
|
||||
class PluginWithoutIOProcessorPlugins(PoolingIOProcessor):
|
||||
# Some models, such as Terratorch (tests/models/test_terratorch.py),
|
||||
# use plugin tasks in the pooler but do not use IO Processor plugins.
|
||||
name = "plugin"
|
||||
|
||||
|
||||
class PluginWithIOProcessorPlugins(PoolingIOProcessor):
|
||||
"""IO Processor plugins are a feature that allows pre- and post-processing
|
||||
of the model input and output for pooling models."""
|
||||
|
||||
name = "plugin"
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
io_processor = get_io_processor(
|
||||
self.vllm_config,
|
||||
self.renderer,
|
||||
self.model_config.io_processor_plugin,
|
||||
)
|
||||
|
||||
assert io_processor is not None
|
||||
self.io_processor = io_processor
|
||||
|
||||
#######################################
|
||||
# online APIs
|
||||
|
||||
def get_request_factory_online(
|
||||
self, ctx: PoolingServeContext
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
assert isinstance(ctx.request, IOProcessorRequest)
|
||||
|
||||
validated_prompt = self.io_processor.parse_data(ctx.request.data)
|
||||
|
||||
raw_prompts = self.io_processor.pre_process(
|
||||
prompt=validated_prompt, request_id=ctx.request_id
|
||||
)
|
||||
|
||||
parsed_prompts = [
|
||||
(
|
||||
prompt
|
||||
if isinstance(prompt, bytes)
|
||||
else parse_model_prompt(self.model_config, prompt)
|
||||
)
|
||||
for prompt in prompt_to_seq(raw_prompts)
|
||||
]
|
||||
num_requests = len(parsed_prompts)
|
||||
|
||||
tok_params = ctx.request.build_tok_params(self.model_config)
|
||||
|
||||
pooling_params = self.io_processor.merge_pooling_params()
|
||||
if pooling_params.task is None:
|
||||
pooling_params.task = "plugin"
|
||||
ctx.pooling_params = pooling_params
|
||||
|
||||
params_seq = self._params_to_seq(ctx.pooling_params, num_requests)
|
||||
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
|
||||
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
|
||||
|
||||
requests = [
|
||||
EncodeCMPLRenderParams(
|
||||
prompts=parsed_prompts[i],
|
||||
tok_params=tok_params,
|
||||
prompt_extras=ctx.prompt_extras,
|
||||
skip_mm_cache=False,
|
||||
params=params_seq[i],
|
||||
lora_requests=seq_lora_requests[i],
|
||||
priorities=seq_priority[i],
|
||||
)
|
||||
for i in range(num_requests)
|
||||
]
|
||||
return requests
|
||||
|
||||
def post_process_online(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
):
|
||||
output = self.io_processor.post_process(
|
||||
ctx.final_res_batch,
|
||||
request_id=ctx.request_id,
|
||||
)
|
||||
ctx.response = IOProcessorResponse(request_id=ctx.request_id, data=output)
|
||||
|
||||
#######################################
|
||||
# offline APIs
|
||||
|
||||
def get_request_factory_offline(
|
||||
self, ctx: AnyOfflineInputsContext
|
||||
) -> tuple[RequestFactory, int]:
|
||||
assert isinstance(ctx, OfflinePluginInputsContext)
|
||||
assert isinstance(ctx.prompts, dict) and "data" in ctx.prompts
|
||||
|
||||
# Validate the request data is valid for the loaded plugin
|
||||
prompt_data = ctx.prompts.get("data")
|
||||
if prompt_data is None:
|
||||
raise ValueError(
|
||||
"The 'data' field of the prompt is expected to contain "
|
||||
"the prompt data and it cannot be None. "
|
||||
"Refer to the documentation of the IOProcessor "
|
||||
"in use for more details."
|
||||
)
|
||||
validated_prompt = self.io_processor.parse_data(prompt_data)
|
||||
|
||||
# obtain the actual model prompts from the pre-processor
|
||||
prompts = self.io_processor.pre_process(prompt=validated_prompt)
|
||||
prompts_seq = prompt_to_seq(prompts)
|
||||
|
||||
num_requests = len(prompts_seq)
|
||||
|
||||
pooling_params: PoolingParams | Sequence[PoolingParams]
|
||||
if ctx.pooling_params is None:
|
||||
pooling_params = PoolingParams()
|
||||
else:
|
||||
pooling_params = ctx.pooling_params
|
||||
|
||||
params_seq: list[PoolingParams] = [
|
||||
self.io_processor.merge_pooling_params(param.clone())
|
||||
for param in self._params_to_seq(
|
||||
pooling_params,
|
||||
num_requests,
|
||||
)
|
||||
]
|
||||
for p in params_seq:
|
||||
if p.task is None:
|
||||
p.task = "plugin"
|
||||
|
||||
return super().get_request_factory_offline(
|
||||
OfflineEncodeInputsContext(
|
||||
prompts=prompts_seq,
|
||||
pooling_params=params_seq,
|
||||
pooling_task="plugin",
|
||||
tokenization_kwargs=ctx.tokenization_kwargs,
|
||||
lora_request=ctx.lora_request,
|
||||
priorities=ctx.priorities,
|
||||
)
|
||||
)
|
||||
|
||||
def post_process_offline(
|
||||
self,
|
||||
ctx: OfflineOutputsContext,
|
||||
) -> list[PoolingRequestOutput]:
|
||||
processed_outputs = self.io_processor.post_process(ctx.outputs)
|
||||
|
||||
return [
|
||||
PoolingRequestOutput[Any](
|
||||
request_id="",
|
||||
outputs=processed_outputs,
|
||||
num_cached_tokens=getattr(processed_outputs, "num_cached_tokens", 0),
|
||||
prompt_token_ids=[],
|
||||
finished=True,
|
||||
)
|
||||
]
|
||||
@@ -1,120 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import time
|
||||
from typing import Annotated, Generic, TypeAlias, TypeVar
|
||||
|
||||
from pydantic import BeforeValidator, Field
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
from ..base.protocol import (
|
||||
ChatRequestMixin,
|
||||
ClassifyRequestMixin,
|
||||
CompletionRequestMixin,
|
||||
EmbedRequestMixin,
|
||||
EncodingRequestMixin,
|
||||
FixedMaxLenTokenizeParamsMixin,
|
||||
PoolingBasicRequestMixin,
|
||||
reject_removed_pooling_parameters,
|
||||
)
|
||||
|
||||
|
||||
class PoolingCompletionRequest(
|
||||
PoolingBasicRequestMixin,
|
||||
CompletionRequestMixin,
|
||||
EmbedRequestMixin,
|
||||
ClassifyRequestMixin,
|
||||
FixedMaxLenTokenizeParamsMixin,
|
||||
):
|
||||
task: PoolingTask | None = None
|
||||
|
||||
def to_pooling_params(self):
|
||||
return PoolingParams(
|
||||
task=self.task,
|
||||
use_activation=self.use_activation,
|
||||
dimensions=self.dimensions,
|
||||
)
|
||||
|
||||
|
||||
class PoolingChatRequest(
|
||||
PoolingBasicRequestMixin,
|
||||
ChatRequestMixin,
|
||||
EmbedRequestMixin,
|
||||
ClassifyRequestMixin,
|
||||
FixedMaxLenTokenizeParamsMixin,
|
||||
):
|
||||
task: PoolingTask | None = None
|
||||
|
||||
def to_pooling_params(self):
|
||||
return PoolingParams(
|
||||
task=self.task,
|
||||
use_activation=self.use_activation,
|
||||
dimensions=self.dimensions,
|
||||
)
|
||||
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
class IOProcessorRequest(PoolingBasicRequestMixin, EncodingRequestMixin, Generic[T]):
|
||||
data: T
|
||||
task: PoolingTask = "plugin"
|
||||
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
return self._build_pooling_tok_params(
|
||||
model_config,
|
||||
add_special_tokens=not model_config.is_encoder_decoder,
|
||||
max_total_tokens=model_config.max_model_len,
|
||||
max_output_tokens=0,
|
||||
)
|
||||
|
||||
def to_pooling_params(self):
|
||||
return PoolingParams(
|
||||
task=self.task,
|
||||
)
|
||||
|
||||
|
||||
class IOProcessorResponse(OpenAIBaseModel, Generic[T]):
|
||||
request_id: str | None = None
|
||||
"""
|
||||
The request_id associated with this response
|
||||
"""
|
||||
created_at: int = Field(default_factory=lambda: int(time.time()))
|
||||
|
||||
data: T
|
||||
"""
|
||||
When using plugins IOProcessor plugins, the actual output is generated
|
||||
by the plugin itself. Hence, we use a generic type for the response data
|
||||
"""
|
||||
|
||||
|
||||
PoolingRequest: TypeAlias = Annotated[
|
||||
PoolingCompletionRequest | PoolingChatRequest | IOProcessorRequest,
|
||||
BeforeValidator(reject_removed_pooling_parameters),
|
||||
]
|
||||
|
||||
|
||||
class PoolingResponseData(OpenAIBaseModel):
|
||||
index: int
|
||||
object: str = "pooling"
|
||||
data: list[list[float]] | list[float] | str
|
||||
|
||||
|
||||
class PoolingResponse(OpenAIBaseModel):
|
||||
id: str = Field(default_factory=lambda: f"pool-{random_uuid()}")
|
||||
object: str = "list"
|
||||
created: int = Field(default_factory=lambda: int(time.time()))
|
||||
model: str
|
||||
data: list[PoolingResponseData]
|
||||
usage: UsageInfo
|
||||
|
||||
|
||||
class PoolingBytesResponse(OpenAIBaseModel):
|
||||
content: list[bytes]
|
||||
headers: dict[str, str] | None = None
|
||||
media_type: str = "application/octet-stream"
|
||||
@@ -1,191 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import assert_never, cast
|
||||
|
||||
from fastapi.responses import JSONResponse, Response, StreamingResponse
|
||||
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import PoolingRequestOutput
|
||||
from vllm.tasks import SupportedTask
|
||||
from vllm.utils.serial_utils import EmbedDType, Endianness
|
||||
|
||||
from ..base.io_processor import PoolingIOProcessor
|
||||
from ..base.serving import PoolingBaseServing
|
||||
from ..factories import init_pooling_io_processors
|
||||
from ..typing import AnyPoolingRequest, PoolingServeContext
|
||||
from ..utils import (
|
||||
BytesEncodingFormat,
|
||||
JsonEncodingFormat,
|
||||
build_pooling_bytes_streaming_response,
|
||||
get_json_response_cls,
|
||||
get_pooling_output_encoder,
|
||||
get_pooling_usage,
|
||||
)
|
||||
from .protocol import (
|
||||
IOProcessorRequest,
|
||||
PoolingRequest,
|
||||
PoolingResponse,
|
||||
PoolingResponseData,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class ServingPooling(PoolingBaseServing):
|
||||
request_id_prefix = "pooling"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*args,
|
||||
supported_tasks: tuple[SupportedTask, ...],
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
self.supported_tasks = supported_tasks
|
||||
self.pooling_task = self.model_config.get_pooling_task(supported_tasks)
|
||||
self.io_processors = init_pooling_io_processors(
|
||||
supported_tasks=supported_tasks,
|
||||
vllm_config=self.vllm_config,
|
||||
renderer=self.renderer,
|
||||
chat_template_config=self.chat_template_config,
|
||||
)
|
||||
self.json_response_cls = get_json_response_cls()
|
||||
|
||||
def get_io_processor(self, request: AnyPoolingRequest) -> PoolingIOProcessor:
|
||||
request = cast(PoolingRequest, request)
|
||||
pooling_task = self._verify_pooling_task(request)
|
||||
return self.io_processors[pooling_task]
|
||||
|
||||
def _verify_pooling_task(self, request: PoolingRequest) -> str:
|
||||
if getattr(request, "dimensions", None) is not None:
|
||||
raise VLLMValidationError(
|
||||
"dimensions is currently not supported", parameter="dimensions"
|
||||
)
|
||||
|
||||
if request.task is None:
|
||||
request.task = self.pooling_task
|
||||
|
||||
if isinstance(request, IOProcessorRequest):
|
||||
request.task = "plugin"
|
||||
|
||||
assert request.task is not None
|
||||
pooling_task = request.task
|
||||
|
||||
# plugin task uses io_processor.parse_data to verify inputs
|
||||
if pooling_task != "plugin" and pooling_task != self.pooling_task:
|
||||
if pooling_task not in self.supported_tasks:
|
||||
raise VLLMValidationError(
|
||||
f"Unsupported task: {pooling_task!r} "
|
||||
f"Supported tasks: {self.supported_tasks}",
|
||||
parameter="task",
|
||||
)
|
||||
else:
|
||||
raise VLLMValidationError(
|
||||
"Try switching the model's pooling_task "
|
||||
f"via --pooler-config.task {request.task}.",
|
||||
parameter="task",
|
||||
)
|
||||
|
||||
if pooling_task == "plugin" and "plugin" not in self.io_processors:
|
||||
raise VLLMValidationError(
|
||||
"No IOProcessor plugin installed. Please refer "
|
||||
"to the documentation and to the "
|
||||
"'prithvi_geospatial_mae_io_processor' "
|
||||
"offline inference example for more details.",
|
||||
parameter="task",
|
||||
)
|
||||
|
||||
return pooling_task
|
||||
|
||||
def _build_response(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
) -> Response:
|
||||
if ctx.response is not None:
|
||||
# for IOProcessorResponse
|
||||
return self.json_response_cls(content=ctx.response.model_dump())
|
||||
|
||||
encoding_format = ctx.request.encoding_format
|
||||
embed_dtype = ctx.request.embed_dtype
|
||||
endianness = ctx.request.endianness
|
||||
|
||||
if encoding_format == "float" or encoding_format == "base64":
|
||||
return self.request_output_to_pooling_json_response(
|
||||
ctx.final_res_batch,
|
||||
ctx.request_id,
|
||||
ctx.created_time,
|
||||
ctx.model_name,
|
||||
encoding_format,
|
||||
embed_dtype,
|
||||
endianness,
|
||||
)
|
||||
|
||||
if encoding_format == "bytes" or encoding_format == "bytes_only":
|
||||
return self.request_output_to_pooling_bytes_response(
|
||||
ctx.final_res_batch,
|
||||
ctx.request_id,
|
||||
ctx.created_time,
|
||||
ctx.model_name,
|
||||
encoding_format,
|
||||
embed_dtype,
|
||||
endianness,
|
||||
)
|
||||
|
||||
assert_never(encoding_format)
|
||||
|
||||
def request_output_to_pooling_json_response(
|
||||
self,
|
||||
final_res_batch: list[PoolingRequestOutput],
|
||||
request_id: str,
|
||||
created_time: int,
|
||||
model_name: str,
|
||||
encoding_format: JsonEncodingFormat,
|
||||
embed_dtype: EmbedDType,
|
||||
endianness: Endianness,
|
||||
) -> JSONResponse:
|
||||
encode_fn = get_pooling_output_encoder(
|
||||
encoding_format=encoding_format,
|
||||
embed_dtype=embed_dtype,
|
||||
endianness=endianness,
|
||||
)
|
||||
|
||||
items: list[PoolingResponseData] = []
|
||||
|
||||
for idx, final_res in enumerate(final_res_batch):
|
||||
item = PoolingResponseData(
|
||||
index=idx,
|
||||
data=encode_fn(final_res),
|
||||
)
|
||||
|
||||
items.append(item)
|
||||
|
||||
response = PoolingResponse(
|
||||
id=request_id,
|
||||
created=created_time,
|
||||
model=model_name,
|
||||
data=items,
|
||||
usage=get_pooling_usage(final_res_batch),
|
||||
)
|
||||
return self.json_response_cls(content=response.model_dump())
|
||||
|
||||
def request_output_to_pooling_bytes_response(
|
||||
self,
|
||||
final_res_batch: list[PoolingRequestOutput],
|
||||
request_id: str,
|
||||
created_time: int,
|
||||
model_name: str,
|
||||
encoding_format: BytesEncodingFormat,
|
||||
embed_dtype: EmbedDType,
|
||||
endianness: Endianness,
|
||||
) -> StreamingResponse:
|
||||
return build_pooling_bytes_streaming_response(
|
||||
pooling_outputs=final_res_batch,
|
||||
request_id=request_id,
|
||||
created_time=created_time,
|
||||
model_name=model_name,
|
||||
encoding_format=encoding_format,
|
||||
embed_dtype=embed_dtype,
|
||||
endianness=endianness,
|
||||
)
|
||||
@@ -1,111 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from http import HTTPStatus
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
with_cancellation,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from .protocol import RerankRequest, ScoreRequest
|
||||
from .serving import ServingScores
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def score(request: Request) -> ServingScores:
|
||||
handler = getattr(request.app.state, "serving_scores", None)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Score API")
|
||||
return handler
|
||||
|
||||
|
||||
def rerank(request: Request) -> ServingScores:
|
||||
handler = getattr(request.app.state, "serving_scores", None)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Rerank (Score) API")
|
||||
return handler
|
||||
|
||||
|
||||
@router.post(
|
||||
"/score",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def create_score(request: ScoreRequest, raw_request: Request):
|
||||
handler = score(raw_request)
|
||||
return await handler(request, raw_request)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/score",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
async def create_score_v1(request: ScoreRequest, raw_request: Request):
|
||||
logger.warning(
|
||||
"To indicate that Score API is not part of standard OpenAI API, we "
|
||||
"have moved it to `/score`. Please update your client accordingly."
|
||||
)
|
||||
|
||||
return await create_score(request, raw_request)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/rerank",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def do_rerank(request: RerankRequest, raw_request: Request):
|
||||
handler = rerank(raw_request)
|
||||
return await handler(request, raw_request)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/rerank",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
async def do_rerank_v1(request: RerankRequest, raw_request: Request):
|
||||
logger.warning_once(
|
||||
"To indicate that the rerank API is not part of the standard OpenAI"
|
||||
" API, we have located it at `/rerank`. Please update your client "
|
||||
"accordingly. (Note: Conforms to JinaAI rerank API)"
|
||||
)
|
||||
|
||||
return await do_rerank(request, raw_request)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v2/rerank",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
async def do_rerank_v2(request: RerankRequest, raw_request: Request):
|
||||
return await do_rerank(request, raw_request)
|
||||
@@ -1,978 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import time
|
||||
from collections.abc import Sequence
|
||||
from typing import Any, TypeAlias, cast
|
||||
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm import PoolingParams, PoolingRequestOutput, TokensPrompt
|
||||
from vllm.exceptions import ChatTemplateResolutionError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.renderers.hf import safe_apply_chat_template
|
||||
from vllm.renderers.inputs.preprocess import (
|
||||
extract_target_prompt,
|
||||
parse_model_prompt,
|
||||
prompt_to_seq,
|
||||
)
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.utils.mistral import is_mistral_tokenizer
|
||||
|
||||
from ..base.io_processor import PoolingIOProcessor
|
||||
from ..pooling.protocol import PoolingCompletionRequest
|
||||
from ..typing import (
|
||||
AnyOfflineInputsContext,
|
||||
AnyPoolingRequest,
|
||||
AnyRenderParam,
|
||||
EncodeChatRenderParams,
|
||||
EncodeCMPLRenderParams,
|
||||
OfflineEncodeInputsContext,
|
||||
OfflineOutputsContext,
|
||||
OfflineScoringInputsContext,
|
||||
PoolingEngineInput,
|
||||
PoolingServeContext,
|
||||
RequestFactory,
|
||||
RequestGenerator,
|
||||
ScoringRenderParams,
|
||||
)
|
||||
from .protocol import RerankRequest, ScoreRequest, ScoringRequest
|
||||
from .typing import ScoreData, ScoreInput, ScoringData
|
||||
from .utils import (
|
||||
compress_token_type_ids,
|
||||
compute_maxsim_score,
|
||||
get_num_special_tokens_for_pair,
|
||||
parse_score_data,
|
||||
score_data_to_prompts,
|
||||
truncate_text_to_tokens,
|
||||
validate_score_input,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
ScoringServeContext: TypeAlias = PoolingServeContext[ScoringRequest]
|
||||
|
||||
|
||||
def _apply_post_tokenization_to_token_type_ids(
|
||||
tokenizer: Any,
|
||||
tok_params: TokenizeParams,
|
||||
token_type_ids: list[int],
|
||||
) -> list[int]:
|
||||
# Must stay in the same order as `TokenizeParams._validate_tokens`, which
|
||||
# truncates before padding. These ids are parallel to the prompt tokens
|
||||
# and locate the query/document boundary for the cross-encoder; applying
|
||||
# the two steps in a different order silently misplaces that boundary.
|
||||
max_length = tok_params.truncate_prompt_tokens
|
||||
if max_length is not None and max_length < 0:
|
||||
max_length = tok_params.max_input_tokens
|
||||
|
||||
if max_length is not None and max_length < len(token_type_ids):
|
||||
if max_length == 0:
|
||||
token_type_ids = token_type_ids[:0]
|
||||
else:
|
||||
side = tok_params.truncation_side or (
|
||||
tokenizer.truncation_side if tokenizer is not None else None
|
||||
)
|
||||
token_type_ids = (
|
||||
token_type_ids[-max_length:]
|
||||
if side == "left"
|
||||
else token_type_ids[:max_length]
|
||||
)
|
||||
|
||||
pad_length = tok_params.pad_prompt_tokens
|
||||
if pad_length is not None and pad_length < 0:
|
||||
pad_length = tok_params.max_input_tokens
|
||||
|
||||
if pad_length is not None and pad_length > len(token_type_ids):
|
||||
pad_token_type_id = getattr(tokenizer, "pad_token_type_id", 0) or 0
|
||||
token_type_ids = token_type_ids + [pad_token_type_id] * (
|
||||
pad_length - len(token_type_ids)
|
||||
)
|
||||
|
||||
return token_type_ids
|
||||
|
||||
|
||||
class ScoringIOProcessor(PoolingIOProcessor):
|
||||
name: str
|
||||
pooling_task: PoolingTask
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
self.tokenizer = self.renderer.get_tokenizer()
|
||||
self.architecture = self.model_config.architecture
|
||||
self.is_multimodal_model = self.model_config.is_multimodal_model
|
||||
self.pad_token_id = self.tokenizer.pad_token_id
|
||||
|
||||
def create_pooling_params(self, request):
|
||||
return request.to_pooling_params(self.pooling_task)
|
||||
|
||||
def _validate_token_limit(self, value: int, name: str) -> None:
|
||||
if value < 0:
|
||||
raise ValueError(f"{name} must be a non-negative integer")
|
||||
if value >= self.model_config.max_model_len:
|
||||
raise ValueError(
|
||||
f"{name} ({value}) must be less "
|
||||
f"than max_model_len ({self.model_config.max_model_len})."
|
||||
)
|
||||
|
||||
def _get_token_limits(
|
||||
self,
|
||||
request: ScoringRequest | None = None,
|
||||
pooling_params: PoolingParams | None = None,
|
||||
) -> tuple[int, int]:
|
||||
"""Extract and validate token limits from request or pooling_params."""
|
||||
if request is not None:
|
||||
max_tokens_per_query = getattr(request, "max_tokens_per_query", 0)
|
||||
max_tokens_per_doc = getattr(request, "max_tokens_per_doc", 0)
|
||||
else:
|
||||
extra = (
|
||||
(pooling_params.extra_kwargs or {})
|
||||
if pooling_params is not None
|
||||
else {}
|
||||
)
|
||||
max_tokens_per_query = extra.get("max_tokens_per_query", 0)
|
||||
max_tokens_per_doc = extra.get("max_tokens_per_doc", 0)
|
||||
|
||||
if max_tokens_per_query != 0:
|
||||
self._validate_token_limit(max_tokens_per_query, "max_tokens_per_query")
|
||||
if max_tokens_per_doc != 0:
|
||||
self._validate_token_limit(max_tokens_per_doc, "max_tokens_per_doc")
|
||||
return max_tokens_per_query, max_tokens_per_doc
|
||||
|
||||
def _truncate_scoring_data(
|
||||
self,
|
||||
scoring_data: ScoringData,
|
||||
max_tokens_per_query: int = 0,
|
||||
max_tokens_per_doc: int = 0,
|
||||
) -> ScoringData:
|
||||
"""Truncate query/document texts to token limits."""
|
||||
data_1 = scoring_data.data_1
|
||||
data_2 = scoring_data.data_2
|
||||
if max_tokens_per_query > 0:
|
||||
data_1 = [
|
||||
truncate_text_to_tokens(d, self.tokenizer, max_tokens_per_query)
|
||||
if isinstance(d, str)
|
||||
else d
|
||||
for d in data_1
|
||||
]
|
||||
if max_tokens_per_doc > 0:
|
||||
data_2 = [
|
||||
truncate_text_to_tokens(d, self.tokenizer, max_tokens_per_doc)
|
||||
if isinstance(d, str)
|
||||
else d
|
||||
for d in data_2
|
||||
]
|
||||
return ScoringData(data_1=data_1, data_2=data_2)
|
||||
|
||||
def valid_inputs(
|
||||
self,
|
||||
data_1: ScoreInput | list[ScoreInput],
|
||||
data_2: ScoreInput | list[ScoreInput],
|
||||
) -> ScoringData:
|
||||
scoring_data = validate_score_input(
|
||||
data_1,
|
||||
data_2,
|
||||
is_multimodal_model=self.is_multimodal_model,
|
||||
architecture=self.architecture,
|
||||
)
|
||||
return scoring_data
|
||||
|
||||
def valid_inputs_online(self, request: AnyPoolingRequest):
|
||||
if isinstance(request, ScoreRequest):
|
||||
data_1 = request.data_1
|
||||
data_2 = request.data_2
|
||||
elif isinstance(request, RerankRequest):
|
||||
data_1 = request.query
|
||||
data_2 = request.documents
|
||||
else:
|
||||
raise ValueError(f"Invalid {request.__class__.__name__} request type")
|
||||
|
||||
scoring_data = self.valid_inputs(data_1, data_2)
|
||||
return scoring_data
|
||||
|
||||
def _pair_error_output(
|
||||
self,
|
||||
output_1: PoolingRequestOutput,
|
||||
output_2: PoolingRequestOutput,
|
||||
) -> PoolingRequestOutput | None:
|
||||
failed = output_1 if output_1.error is not None else output_2
|
||||
if failed.error is None:
|
||||
return None
|
||||
|
||||
padding = [self.pad_token_id] if self.pad_token_id is not None else []
|
||||
return PoolingRequestOutput(
|
||||
request_id=f"{output_1.request_id}_{output_2.request_id}",
|
||||
outputs=failed.outputs,
|
||||
prompt_token_ids=(
|
||||
output_1.prompt_token_ids + padding + output_2.prompt_token_ids
|
||||
),
|
||||
num_cached_tokens=(output_1.num_cached_tokens + output_2.num_cached_tokens),
|
||||
finished=failed.finished,
|
||||
error=failed.error,
|
||||
)
|
||||
|
||||
|
||||
class BiEncoderIOProcessor(ScoringIOProcessor):
|
||||
name = "bi-encoder"
|
||||
pooling_task: PoolingTask = "embed"
|
||||
|
||||
#######################################
|
||||
# online APIs
|
||||
|
||||
def get_request_factory_online(
|
||||
self, ctx: PoolingServeContext
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
request = ctx.request
|
||||
scoring_data = self.valid_inputs_online(request)
|
||||
|
||||
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
|
||||
request=request
|
||||
)
|
||||
if max_tokens_per_query > 0 or max_tokens_per_doc > 0:
|
||||
scoring_data = self._truncate_scoring_data(
|
||||
scoring_data, max_tokens_per_query, max_tokens_per_doc
|
||||
)
|
||||
|
||||
data_1 = score_data_to_prompts(scoring_data.data_1, "query", self.model_config)
|
||||
data_2 = score_data_to_prompts(
|
||||
scoring_data.data_2, "document", self.model_config
|
||||
)
|
||||
prompts = data_1 + data_2
|
||||
ctx.n_queries = len(data_1)
|
||||
|
||||
prompts_seq = prompt_to_seq(prompts)
|
||||
parsed_prompts = [
|
||||
parse_model_prompt(self.model_config, prompt) for prompt in prompts_seq
|
||||
]
|
||||
num_requests = len(parsed_prompts)
|
||||
|
||||
tok_params = request.build_tok_params(self.model_config)
|
||||
params_seq = self._params_to_seq(ctx.pooling_params, num_requests)
|
||||
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
|
||||
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
|
||||
|
||||
requests = [
|
||||
EncodeCMPLRenderParams(
|
||||
prompts=parsed_prompts[i],
|
||||
tok_params=tok_params,
|
||||
prompt_extras=ctx.prompt_extras,
|
||||
skip_mm_cache=False,
|
||||
params=params_seq[i],
|
||||
lora_requests=seq_lora_requests[i],
|
||||
priorities=seq_priority[i],
|
||||
)
|
||||
for i in range(num_requests)
|
||||
]
|
||||
return requests
|
||||
|
||||
def post_process_online(
|
||||
self,
|
||||
ctx: ScoringServeContext,
|
||||
):
|
||||
assert ctx.final_res_batch is not None
|
||||
assert isinstance(ctx.n_queries, int)
|
||||
|
||||
ctx.final_res_batch = self._post_process(
|
||||
outputs=ctx.final_res_batch, n_queries=ctx.n_queries
|
||||
)
|
||||
|
||||
#######################################
|
||||
# offline APIs
|
||||
|
||||
def get_request_factory_offline(
|
||||
self, ctx: AnyOfflineInputsContext
|
||||
) -> tuple[RequestFactory, int]:
|
||||
assert isinstance(ctx, OfflineScoringInputsContext)
|
||||
|
||||
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
|
||||
pooling_params=ctx.pooling_params
|
||||
)
|
||||
|
||||
scoring_data = ctx.scoring_data
|
||||
if max_tokens_per_query > 0 or max_tokens_per_doc > 0:
|
||||
scoring_data = self._truncate_scoring_data(
|
||||
scoring_data, max_tokens_per_query, max_tokens_per_doc
|
||||
)
|
||||
|
||||
data_1 = score_data_to_prompts(scoring_data.data_1, "query", self.model_config)
|
||||
data_2 = score_data_to_prompts(
|
||||
scoring_data.data_2, "document", self.model_config
|
||||
)
|
||||
prompts = data_1 + data_2
|
||||
|
||||
return super().get_request_factory_offline(
|
||||
OfflineEncodeInputsContext(
|
||||
pooling_task=self.pooling_task,
|
||||
prompts=prompts,
|
||||
tokenization_kwargs=ctx.tokenization_kwargs,
|
||||
pooling_params=ctx.pooling_params,
|
||||
lora_request=ctx.lora_request,
|
||||
priorities=ctx.priorities,
|
||||
)
|
||||
)
|
||||
|
||||
def post_process_offline(
|
||||
self,
|
||||
ctx: OfflineOutputsContext,
|
||||
) -> list[PoolingRequestOutput]:
|
||||
assert ctx.n_queries is not None
|
||||
return self._post_process(outputs=ctx.outputs, n_queries=ctx.n_queries)
|
||||
|
||||
#######################################
|
||||
# helpers
|
||||
|
||||
def _post_process(self, outputs: list[PoolingRequestOutput], n_queries: int):
|
||||
emb_data_1 = outputs[:n_queries]
|
||||
emb_data_2 = outputs[n_queries:]
|
||||
|
||||
if len(emb_data_1) == 1:
|
||||
emb_data_1 = emb_data_1 * len(emb_data_2)
|
||||
|
||||
final_res_batch: list[PoolingRequestOutput] = []
|
||||
for emb_1, emb_2 in zip(emb_data_1, emb_data_2):
|
||||
if error_output := self._pair_error_output(emb_1, emb_2):
|
||||
final_res_batch.append(error_output)
|
||||
continue
|
||||
|
||||
pair_score = F.cosine_similarity(
|
||||
emb_1.outputs.data.float(), emb_2.outputs.data.float(), dim=0
|
||||
)
|
||||
|
||||
padding: list[int] = []
|
||||
if self.pad_token_id is not None:
|
||||
padding = [self.pad_token_id]
|
||||
|
||||
tokens = emb_1.prompt_token_ids + padding + emb_2.prompt_token_ids
|
||||
|
||||
final_res_batch.append(
|
||||
PoolingRequestOutput(
|
||||
request_id=f"{emb_1.request_id}_{emb_2.request_id}",
|
||||
outputs=pair_score,
|
||||
prompt_token_ids=tokens,
|
||||
num_cached_tokens=emb_1.num_cached_tokens + emb_2.num_cached_tokens,
|
||||
finished=True,
|
||||
)
|
||||
)
|
||||
return final_res_batch
|
||||
|
||||
|
||||
class LateInteractionIOProcessor(BiEncoderIOProcessor):
|
||||
name = "late-interaction"
|
||||
pooling_task: PoolingTask = "token_embed"
|
||||
|
||||
def _post_process(self, outputs: list[PoolingRequestOutput], n_queries: int):
|
||||
# Split into query and document embeddings
|
||||
emb_data_1 = outputs[:n_queries]
|
||||
emb_data_2 = outputs[n_queries:]
|
||||
|
||||
# Expand queries if 1:N scoring
|
||||
if len(emb_data_1) == 1:
|
||||
emb_data_1 = emb_data_1 * len(emb_data_2)
|
||||
|
||||
final_res_batch: list[PoolingRequestOutput] = []
|
||||
padding: list[int] = []
|
||||
if (pad_token_id := self.pad_token_id) is not None:
|
||||
padding = [pad_token_id]
|
||||
|
||||
# Compute MaxSim scores
|
||||
for emb_1, emb_2 in zip(emb_data_1, emb_data_2):
|
||||
if error_output := self._pair_error_output(emb_1, emb_2):
|
||||
final_res_batch.append(error_output)
|
||||
continue
|
||||
|
||||
# emb_1.outputs.data: [query_len, dim]
|
||||
# emb_2.outputs.data: [doc_len, dim]
|
||||
q_emb = emb_1.outputs.data
|
||||
d_emb = emb_2.outputs.data
|
||||
|
||||
maxsim_score = compute_maxsim_score(q_emb, d_emb)
|
||||
|
||||
tokens = emb_1.prompt_token_ids + padding + emb_2.prompt_token_ids
|
||||
|
||||
final_res_batch.append(
|
||||
PoolingRequestOutput(
|
||||
request_id=f"{emb_1.request_id}_{emb_2.request_id}",
|
||||
outputs=maxsim_score,
|
||||
prompt_token_ids=tokens,
|
||||
num_cached_tokens=emb_1.num_cached_tokens + emb_2.num_cached_tokens,
|
||||
finished=True,
|
||||
)
|
||||
)
|
||||
return final_res_batch
|
||||
|
||||
|
||||
class FlashLateInteractionIOProcessor(LateInteractionIOProcessor):
|
||||
name = "flash-late-interaction"
|
||||
|
||||
def post_process_online(
|
||||
self,
|
||||
ctx: ScoringServeContext,
|
||||
):
|
||||
assert ctx.query_final_res_batch is not None
|
||||
assert ctx.final_res_batch is not None
|
||||
assert isinstance(ctx.n_queries, int)
|
||||
|
||||
# Expand queries if 1:N scoring
|
||||
if len(ctx.query_final_res_batch) == 1:
|
||||
ctx.query_final_res_batch = ctx.query_final_res_batch * len(
|
||||
ctx.final_res_batch
|
||||
)
|
||||
|
||||
final_res_batch: list[PoolingRequestOutput] = []
|
||||
for d1, d2 in zip(ctx.query_final_res_batch, ctx.final_res_batch):
|
||||
padding: list[int] = []
|
||||
if (pad_token_id := self.pad_token_id) is not None:
|
||||
padding = [pad_token_id]
|
||||
|
||||
tokens = d1.prompt_token_ids + padding + d2.prompt_token_ids
|
||||
|
||||
final_res_batch.append(
|
||||
PoolingRequestOutput(
|
||||
request_id=f"{d1.request_id}_{d2.request_id}",
|
||||
outputs=d2.outputs,
|
||||
prompt_token_ids=tokens,
|
||||
num_cached_tokens=d1.num_cached_tokens + d2.num_cached_tokens,
|
||||
finished=True,
|
||||
)
|
||||
)
|
||||
ctx.final_res_batch = final_res_batch
|
||||
|
||||
|
||||
class CrossEncoderIOProcessor(ScoringIOProcessor):
|
||||
name = "cross-encoder"
|
||||
pooling_task: PoolingTask = "classify"
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
if is_mistral_tokenizer(self.tokenizer):
|
||||
raise ValueError("MistralTokenizer not supported for cross-encoding")
|
||||
|
||||
from vllm.model_executor.model_loader import get_model_cls
|
||||
from vllm.model_executor.models.interfaces import supports_score_template
|
||||
|
||||
model = get_model_cls(self.model_config)
|
||||
self.supports_score_template = supports_score_template(model)
|
||||
self.model = model if self.supports_score_template else None
|
||||
self.use_sep_token = self.model_config.use_sep_token
|
||||
|
||||
if (
|
||||
getattr(self.model_config.hf_config, "is_original_qwen3_reranker", False)
|
||||
and self.chat_template is None
|
||||
):
|
||||
suggested_template = (
|
||||
"examples/pooling/score/template/qwen3_vl_reranker.jinja"
|
||||
if self.model_config.is_multimodal_model
|
||||
else "examples/pooling/score/template/qwen3_reranker.jinja"
|
||||
)
|
||||
logger.warning(
|
||||
"Serving an original Qwen3 reranker (%s) without a "
|
||||
"--chat-template. The model was trained with a specific prompt "
|
||||
"template; running without it may produce inaccurate relevance "
|
||||
"scores. Consider specifying `--chat-template %s`.",
|
||||
self.model_config.model,
|
||||
suggested_template,
|
||||
)
|
||||
|
||||
#######################################
|
||||
# online APIs
|
||||
|
||||
def get_request_factory_online(
|
||||
self, ctx: PoolingServeContext
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
request = ctx.request
|
||||
scoring_data = self.valid_inputs_online(request)
|
||||
data_1 = scoring_data.data_1
|
||||
data_2 = scoring_data.data_2
|
||||
num_requests = len(data_2)
|
||||
|
||||
if len(data_1) == 1:
|
||||
data_1 = data_1 * num_requests
|
||||
|
||||
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
|
||||
request=request
|
||||
)
|
||||
|
||||
tok_params = request.build_tok_params(self.model_config)
|
||||
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
|
||||
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
|
||||
|
||||
requests = [
|
||||
ScoringRenderParams(
|
||||
data_1=data_1[i],
|
||||
data_2=data_2[i],
|
||||
chat_template=self.chat_template,
|
||||
max_tokens_per_query=max_tokens_per_query,
|
||||
max_tokens_per_doc=max_tokens_per_doc,
|
||||
tok_params=tok_params,
|
||||
prompt_extras=ctx.prompt_extras,
|
||||
skip_mm_cache=False,
|
||||
params=ctx.pooling_params,
|
||||
lora_requests=seq_lora_requests[i],
|
||||
priorities=seq_priority[i],
|
||||
)
|
||||
for i in range(num_requests)
|
||||
]
|
||||
|
||||
return requests
|
||||
|
||||
#######################################
|
||||
# offline APIs
|
||||
|
||||
def get_request_factory_offline(
|
||||
self, ctx: AnyOfflineInputsContext
|
||||
) -> tuple[RequestFactory, int]:
|
||||
assert isinstance(ctx, OfflineScoringInputsContext)
|
||||
|
||||
data_1 = ctx.scoring_data.data_1
|
||||
data_2 = ctx.scoring_data.data_2
|
||||
num_requests = len(data_2)
|
||||
|
||||
if len(data_1) == 1:
|
||||
data_1 = data_1 * num_requests
|
||||
|
||||
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
|
||||
pooling_params=ctx.pooling_params
|
||||
)
|
||||
tok_params = self.renderer.default_cmpl_tok_params.with_kwargs(
|
||||
**(ctx.tokenization_kwargs or {})
|
||||
)
|
||||
prompt_extras = ctx.pooling_params.extra_kwargs
|
||||
|
||||
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
|
||||
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
|
||||
|
||||
def request_factory() -> RequestGenerator:
|
||||
for i in range(num_requests):
|
||||
yield ScoringRenderParams(
|
||||
data_1=data_1[i],
|
||||
data_2=data_2[i],
|
||||
chat_template=(
|
||||
self.chat_template
|
||||
if ctx.chat_template is None
|
||||
else ctx.chat_template
|
||||
),
|
||||
max_tokens_per_query=max_tokens_per_query,
|
||||
max_tokens_per_doc=max_tokens_per_doc,
|
||||
tok_params=tok_params,
|
||||
prompt_extras=prompt_extras,
|
||||
skip_mm_cache=False,
|
||||
params=ctx.pooling_params,
|
||||
lora_requests=seq_lora_requests[i],
|
||||
priorities=seq_priority[i],
|
||||
)
|
||||
|
||||
return request_factory, num_requests
|
||||
|
||||
#######################################
|
||||
# helpers
|
||||
|
||||
def render(
|
||||
self,
|
||||
render_params: EncodeCMPLRenderParams
|
||||
| EncodeChatRenderParams
|
||||
| ScoringRenderParams,
|
||||
) -> PoolingEngineInput:
|
||||
if "data_1" not in render_params:
|
||||
raise ValueError(
|
||||
f"Unsupported render_params type {render_params.__class__.__name__}"
|
||||
)
|
||||
render_params = cast(ScoringRenderParams, render_params)
|
||||
|
||||
arrival_time = time.time()
|
||||
|
||||
tok_params = render_params["tok_params"]
|
||||
params = render_params["params"]
|
||||
prompt_extras = render_params["prompt_extras"]
|
||||
|
||||
_, engine_prompt = self.get_score_prompt(
|
||||
data_1=render_params["data_1"],
|
||||
data_2=render_params["data_2"],
|
||||
encode_kwargs=tok_params.get_encode_kwargs(),
|
||||
chat_template=render_params["chat_template"],
|
||||
max_tokens_per_query=render_params["max_tokens_per_query"],
|
||||
max_tokens_per_doc=render_params["max_tokens_per_doc"],
|
||||
chat_template_kwargs=prompt_extras.get("chat_template_kwargs")
|
||||
if prompt_extras
|
||||
else None,
|
||||
)
|
||||
|
||||
tok_params.apply_post_tokenization(self.tokenizer, engine_prompt)
|
||||
|
||||
if token_type_ids := engine_prompt.pop("token_type_ids", None):
|
||||
params = params.clone()
|
||||
compressed = compress_token_type_ids(
|
||||
_apply_post_tokenization_to_token_type_ids(
|
||||
self.tokenizer, tok_params, token_type_ids
|
||||
)
|
||||
)
|
||||
params.extra_kwargs = {
|
||||
**(params.extra_kwargs or {}),
|
||||
"compressed_token_type_ids": compressed,
|
||||
}
|
||||
|
||||
engine_prompt_extras = (
|
||||
{
|
||||
k: v
|
||||
for k in ("mm_processor_kwargs", "cache_salt")
|
||||
if (v := prompt_extras.get(k)) is not None
|
||||
}
|
||||
if prompt_extras
|
||||
else None
|
||||
)
|
||||
|
||||
if engine_prompt_extras:
|
||||
target_prompt = extract_target_prompt(self.model_config, engine_prompt)
|
||||
target_prompt.update(engine_prompt_extras)
|
||||
|
||||
engine_input = self.renderer.process_for_engine(engine_prompt, arrival_time)
|
||||
|
||||
return PoolingEngineInput(
|
||||
prompts=engine_input,
|
||||
params=params,
|
||||
lora_requests=render_params["lora_requests"],
|
||||
priorities=render_params["priorities"],
|
||||
)
|
||||
|
||||
def get_score_prompt(
|
||||
self,
|
||||
data_1: ScoreData,
|
||||
data_2: ScoreData,
|
||||
encode_kwargs: dict[str, Any],
|
||||
chat_template: str | None = None,
|
||||
max_tokens_per_query: int = 0,
|
||||
max_tokens_per_doc: int = 0,
|
||||
chat_template_kwargs: dict[str, Any] | None = None,
|
||||
):
|
||||
model_config = self.model_config
|
||||
tokenizer = self.tokenizer
|
||||
|
||||
prompt_1, prompt_2, mm_data, mm_uuids = parse_score_data(
|
||||
data_1,
|
||||
data_2,
|
||||
model_config,
|
||||
)
|
||||
|
||||
# Limit the query and document separately before composing them.
|
||||
if max_tokens_per_query > 0 and isinstance(prompt_1, str):
|
||||
prompt_1 = truncate_text_to_tokens(
|
||||
prompt_1, tokenizer, max_tokens_per_query
|
||||
)
|
||||
if max_tokens_per_doc > 0 and isinstance(prompt_2, str):
|
||||
prompt_2 = truncate_text_to_tokens(prompt_2, tokenizer, max_tokens_per_doc)
|
||||
|
||||
def default_tokenizer_encode():
|
||||
local_kwargs = encode_kwargs.copy()
|
||||
|
||||
if self.supports_score_template:
|
||||
assert self.model is not None
|
||||
full_prompt = self.model.get_score_template(prompt_1, prompt_2)
|
||||
if full_prompt is None:
|
||||
raise ValueError("Get empty score template from model")
|
||||
|
||||
prompt_inputs = tokenizer(full_prompt, **local_kwargs)
|
||||
else:
|
||||
if self.use_sep_token:
|
||||
# cross_encoder models defaults to using separating token.
|
||||
if max_tokens_per_doc > 0 and isinstance(prompt_2, str):
|
||||
query_tokens = tokenizer.encode(
|
||||
prompt_1, add_special_tokens=False
|
||||
)
|
||||
num_special = get_num_special_tokens_for_pair(tokenizer)
|
||||
doc_limit_max_length = (
|
||||
len(query_tokens) + max_tokens_per_doc + num_special
|
||||
)
|
||||
existing_max_length = local_kwargs.get("max_length")
|
||||
if existing_max_length is not None:
|
||||
effective_max_length = min(
|
||||
doc_limit_max_length, existing_max_length
|
||||
)
|
||||
else:
|
||||
effective_max_length = doc_limit_max_length
|
||||
local_kwargs["truncation"] = "only_second"
|
||||
local_kwargs["max_length"] = effective_max_length
|
||||
|
||||
prompt_inputs = tokenizer(
|
||||
text=prompt_1, text_pair=prompt_2, **local_kwargs
|
||||
)
|
||||
full_prompt = tokenizer.decode(prompt_inputs["input_ids"])
|
||||
else:
|
||||
# `llm as reranker` defaults to not using separating token.
|
||||
full_prompt = prompt_1 + prompt_2
|
||||
prompt_inputs = tokenizer(text=full_prompt, **local_kwargs)
|
||||
return full_prompt, prompt_inputs
|
||||
|
||||
# FIXME: For now, we only apply a template when one is explicitly provided.
|
||||
# We cannot rely on the tokenizer's chat template because many models
|
||||
# inherit junk templates from their base LLM, which breaks both the models
|
||||
# and the tests that use them.
|
||||
if chat_template is None:
|
||||
full_prompt, prompt_inputs = default_tokenizer_encode()
|
||||
else:
|
||||
# FIXME:
|
||||
# Try applying a score template from the CLI arg or tokenizer_config.json
|
||||
# If that fails because there is no such template,
|
||||
# fall back to the default implementation.
|
||||
try:
|
||||
_safe_kwargs = chat_template_kwargs or {}
|
||||
_reserved = {"chat_template", "tools", "tokenize"}
|
||||
_unexpected = _reserved & _safe_kwargs.keys()
|
||||
if _unexpected:
|
||||
raise ValueError(
|
||||
"chat_template_kwargs contains reserved keys that "
|
||||
f"conflict with fixed scorer arguments: {_unexpected}"
|
||||
)
|
||||
full_prompt = safe_apply_chat_template(
|
||||
model_config,
|
||||
tokenizer,
|
||||
[
|
||||
{"role": "query", "content": prompt_1},
|
||||
{"role": "document", "content": prompt_2},
|
||||
],
|
||||
chat_template=chat_template,
|
||||
tools=None,
|
||||
tokenize=False,
|
||||
**_safe_kwargs,
|
||||
)
|
||||
prompt_inputs = tokenizer(full_prompt, **encode_kwargs)
|
||||
except ChatTemplateResolutionError:
|
||||
full_prompt, prompt_inputs = default_tokenizer_encode()
|
||||
|
||||
engine_prompt = TokensPrompt(prompt_token_ids=prompt_inputs["input_ids"])
|
||||
|
||||
if (token_type_ids := prompt_inputs.get("token_type_ids")) is not None:
|
||||
engine_prompt["token_type_ids"] = token_type_ids
|
||||
|
||||
if self.model is not None:
|
||||
self.model.post_process_tokens(engine_prompt)
|
||||
|
||||
if mm_data is not None:
|
||||
engine_prompt["multi_modal_data"] = mm_data
|
||||
if mm_uuids is not None:
|
||||
engine_prompt["multi_modal_uuids"] = mm_uuids
|
||||
|
||||
return full_prompt, engine_prompt
|
||||
|
||||
|
||||
class JinaRankingIOProcessorMixin:
|
||||
@staticmethod
|
||||
def format_docs_prompts_func(
|
||||
query: str,
|
||||
docs: list[str],
|
||||
special_tokens: dict[str, str] | None = None,
|
||||
instruction: str | None = None,
|
||||
no_thinking: bool = True,
|
||||
) -> str:
|
||||
# TODO: Try converting the code below into a chat template.
|
||||
|
||||
default_special_tokens = {
|
||||
"query_embed_token": "<|rerank_token|>",
|
||||
"doc_embed_token": "<|embed_token|>",
|
||||
}
|
||||
if special_tokens is None:
|
||||
special_tokens = default_special_tokens
|
||||
|
||||
def sanitize_input(text: str) -> str:
|
||||
for token in special_tokens.values():
|
||||
text = text.replace(token, "")
|
||||
return text
|
||||
|
||||
query = sanitize_input(query)
|
||||
docs = [sanitize_input(doc) for doc in docs]
|
||||
|
||||
prefix = (
|
||||
"<|im_start|>system\n"
|
||||
"You are a search relevance expert who can determine a ranking of the passages based on how relevant they are to the query. " # noqa: E501
|
||||
"If the query is a question, how relevant a passage is depends on how well it answers the question. " # noqa: E501
|
||||
"If not, try to analyze the intent of the query and assess how well each passage satisfies the intent. " # noqa: E501
|
||||
"If an instruction is provided, you should follow the instruction when determining the ranking." # noqa: E501
|
||||
"<|im_end|>\n<|im_start|>user\n"
|
||||
)
|
||||
suffix = "<|im_end|>\n<|im_start|>assistant\n"
|
||||
if no_thinking:
|
||||
suffix += "<think>\n\n</think>\n\n"
|
||||
|
||||
doc_emb_token = special_tokens["doc_embed_token"]
|
||||
query_emb_token = special_tokens["query_embed_token"]
|
||||
|
||||
prompt = (
|
||||
f"I will provide you with {len(docs)} passages, each indicated by a numerical identifier. " # noqa: E501
|
||||
f"Rank the passages based on their relevance to query: {query}\n"
|
||||
)
|
||||
|
||||
if instruction:
|
||||
instruction = sanitize_input(instruction)
|
||||
prompt += f"<instruct>\n{instruction}\n</instruct>\n"
|
||||
|
||||
doc_prompts = [
|
||||
f'<passage id="{i}">\n{doc}{doc_emb_token}\n</passage>'
|
||||
for i, doc in enumerate(docs)
|
||||
]
|
||||
prompt += "\n".join(doc_prompts) + "\n"
|
||||
prompt += f"<query>\n{query}{query_emb_token}\n</query>"
|
||||
|
||||
return prefix + prompt + suffix
|
||||
|
||||
@staticmethod
|
||||
def ensure_str(data: Sequence[Any]) -> list[str]:
|
||||
text: list[str] = []
|
||||
for prompt in data:
|
||||
if not isinstance(prompt, str):
|
||||
raise ValueError(
|
||||
"The JinaForRanking model only supports text as input."
|
||||
)
|
||||
text.append(prompt)
|
||||
return text
|
||||
|
||||
|
||||
class JinaRankingIOProcessor(LateInteractionIOProcessor, JinaRankingIOProcessorMixin):
|
||||
name = "jina-reranking-scoring"
|
||||
pooling_task: PoolingTask = "token_embed"
|
||||
|
||||
def get_request_factory_online(
|
||||
self, ctx: PoolingServeContext
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
request = ctx.request
|
||||
ctx.n_queries = 1
|
||||
|
||||
prompt_extras = ctx.prompt_extras
|
||||
scoring_data = self.valid_inputs_online(request)
|
||||
|
||||
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
|
||||
request=request
|
||||
)
|
||||
|
||||
if max_tokens_per_query > 0 or max_tokens_per_doc > 0:
|
||||
scoring_data = self._truncate_scoring_data(
|
||||
scoring_data, max_tokens_per_query, max_tokens_per_doc
|
||||
)
|
||||
|
||||
queries = self.ensure_str(scoring_data.data_1)
|
||||
docs = self.ensure_str(scoring_data.data_2)
|
||||
|
||||
chat_template_kwargs = (
|
||||
prompt_extras.get("chat_template_kwargs") if prompt_extras else None
|
||||
)
|
||||
instruction = (
|
||||
chat_template_kwargs.get("instruction") if chat_template_kwargs else None
|
||||
)
|
||||
|
||||
if len(queries) == 1:
|
||||
prompts = [
|
||||
self.format_docs_prompts_func(
|
||||
query=queries[0], docs=docs, instruction=instruction
|
||||
)
|
||||
]
|
||||
else:
|
||||
prompts = [
|
||||
self.format_docs_prompts_func(
|
||||
query=q, docs=[d], instruction=instruction
|
||||
)
|
||||
for q, d in zip(queries, docs)
|
||||
]
|
||||
|
||||
# Forward truncation from the real request: the base factory reads
|
||||
# these off ctx.request, so omitting them here silently drops
|
||||
# truncate_prompt_tokens for Jina rerank/score (unlike the embed and
|
||||
# bi/cross-encoder paths, which read them from the real request).
|
||||
ctx.request = PoolingCompletionRequest(
|
||||
task="token_embed",
|
||||
input=prompts,
|
||||
truncate_prompt_tokens=request.truncate_prompt_tokens,
|
||||
truncation_side=request.truncation_side,
|
||||
)
|
||||
requests = PoolingIOProcessor.get_request_factory_online(self, ctx)
|
||||
ctx.request = request
|
||||
return requests
|
||||
|
||||
def get_request_factory_offline(
|
||||
self, ctx: AnyOfflineInputsContext
|
||||
) -> tuple[RequestFactory, int]:
|
||||
assert isinstance(ctx, OfflineScoringInputsContext)
|
||||
|
||||
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
|
||||
pooling_params=ctx.pooling_params
|
||||
)
|
||||
scoring_data = ctx.scoring_data
|
||||
if max_tokens_per_query > 0 or max_tokens_per_doc > 0:
|
||||
scoring_data = self._truncate_scoring_data(
|
||||
scoring_data, max_tokens_per_query, max_tokens_per_doc
|
||||
)
|
||||
prompt_extras = ctx.pooling_params.extra_kwargs
|
||||
|
||||
queries = self.ensure_str(scoring_data.data_1)
|
||||
docs = self.ensure_str(scoring_data.data_2)
|
||||
chat_template_kwargs = (
|
||||
prompt_extras.get("chat_template_kwargs") if prompt_extras else None
|
||||
)
|
||||
instruction = (
|
||||
chat_template_kwargs.get("instruction") if chat_template_kwargs else None
|
||||
)
|
||||
|
||||
if len(queries) == 1:
|
||||
prompts = [
|
||||
self.format_docs_prompts_func(
|
||||
query=queries[0], docs=docs, instruction=instruction
|
||||
)
|
||||
]
|
||||
else:
|
||||
prompts = [
|
||||
self.format_docs_prompts_func(
|
||||
query=q, docs=[d], instruction=instruction
|
||||
)
|
||||
for q, d in zip(queries, docs)
|
||||
]
|
||||
|
||||
return PoolingIOProcessor.get_request_factory_offline(
|
||||
self,
|
||||
OfflineEncodeInputsContext(
|
||||
pooling_task=self.pooling_task,
|
||||
prompts=prompts,
|
||||
tokenization_kwargs=ctx.tokenization_kwargs,
|
||||
pooling_params=ctx.pooling_params,
|
||||
lora_request=ctx.lora_request,
|
||||
priorities=ctx.priorities,
|
||||
),
|
||||
)
|
||||
|
||||
def _post_process(self, outputs: list[PoolingRequestOutput], n_queries: int):
|
||||
final_res_batch: list[PoolingRequestOutput] = []
|
||||
|
||||
for i in range(len(outputs)):
|
||||
if outputs[i].error is not None:
|
||||
final_res_batch.append(outputs[i])
|
||||
continue
|
||||
|
||||
embeds = outputs[i].outputs.data.float()
|
||||
|
||||
# The JinaForRanking model concatenates docs first, then query.
|
||||
# Let's stay consistent with this novel design.
|
||||
query_embeds = embeds[-1]
|
||||
doc_embeds = embeds[:-1]
|
||||
|
||||
scores = F.cosine_similarity(query_embeds, doc_embeds)
|
||||
|
||||
for score in scores:
|
||||
final_res_batch.append(
|
||||
PoolingRequestOutput(
|
||||
request_id=outputs[i].request_id,
|
||||
outputs=score,
|
||||
prompt_token_ids=outputs[i].prompt_token_ids,
|
||||
num_cached_tokens=outputs[i].num_cached_tokens,
|
||||
finished=True,
|
||||
)
|
||||
)
|
||||
return final_res_batch
|
||||
|
||||
|
||||
ScoringIOProcessors: dict[str, type[ScoringIOProcessor]] = {
|
||||
p.name: p
|
||||
for p in [
|
||||
BiEncoderIOProcessor,
|
||||
LateInteractionIOProcessor,
|
||||
JinaRankingIOProcessor,
|
||||
FlashLateInteractionIOProcessor,
|
||||
CrossEncoderIOProcessor,
|
||||
]
|
||||
}
|
||||
@@ -1,187 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import time
|
||||
from typing import Any, TypeAlias
|
||||
|
||||
from pydantic import BaseModel, Field, model_validator
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
from ..base.protocol import ClassifyRequestMixin, PoolingBasicRequestMixin
|
||||
from .typing import ScoreContentPartParam, ScoreInput
|
||||
|
||||
|
||||
class ScoringRequestMixin(PoolingBasicRequestMixin, ClassifyRequestMixin):
|
||||
# --8<-- [start:scoring-common-params]
|
||||
max_tokens_per_query: int = Field(
|
||||
default=0,
|
||||
description=(
|
||||
"Maximum number of tokens per query. Queries longer than "
|
||||
"this will be truncated to this length. 0 means no "
|
||||
"query-level truncation is applied."
|
||||
),
|
||||
)
|
||||
max_tokens_per_doc: int = Field(
|
||||
default=0,
|
||||
description=(
|
||||
"Maximum number of tokens per document. Documents longer than "
|
||||
"this will be truncated to this length. 0 means no "
|
||||
"document-level truncation is applied (only truncate_prompt_tokens "
|
||||
"applies to the combined query+document)."
|
||||
),
|
||||
)
|
||||
instruction: str | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Task instruction prepended to each scored pair via the chat "
|
||||
"template. Equivalent to passing "
|
||||
"chat_template_kwargs={'instruction': ...}."
|
||||
),
|
||||
)
|
||||
chat_template_kwargs: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Additional keyword args to pass to the chat template renderer. "
|
||||
"Will be accessible by the score/rerank chat template."
|
||||
),
|
||||
)
|
||||
# --8<-- [end:scoring-common-params]
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _merge_instruction_into_kwargs(self) -> "ScoringRequestMixin":
|
||||
"""Fold the top-level `instruction` field into `chat_template_kwargs`.
|
||||
|
||||
This allows callers to use either the convenience field or the generic
|
||||
dict. Explicit keys inside `chat_template_kwargs` take precedence over
|
||||
the top-level `instruction` field.
|
||||
"""
|
||||
if self.instruction is not None:
|
||||
merged = dict(self.chat_template_kwargs or {})
|
||||
merged.setdefault("instruction", self.instruction)
|
||||
self.chat_template_kwargs = merged
|
||||
return self
|
||||
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
return self._build_pooling_tok_params(
|
||||
model_config,
|
||||
add_special_tokens=True,
|
||||
max_total_tokens=model_config.max_model_len,
|
||||
max_output_tokens=0,
|
||||
)
|
||||
|
||||
def to_pooling_params(self, task: PoolingTask = "classify"):
|
||||
return PoolingParams(
|
||||
task=task,
|
||||
use_activation=self.use_activation,
|
||||
)
|
||||
|
||||
|
||||
class ScoreDataRequest(ScoringRequestMixin):
|
||||
data_1: ScoreInput | list[ScoreInput]
|
||||
data_2: ScoreInput | list[ScoreInput]
|
||||
|
||||
|
||||
class ScoreQueriesDocumentsRequest(ScoringRequestMixin):
|
||||
# --8<-- [start:score-request-params]
|
||||
queries: ScoreInput | list[ScoreInput]
|
||||
documents: ScoreInput | list[ScoreInput]
|
||||
# --8<-- [end:score-request-params]
|
||||
|
||||
@property
|
||||
def data_1(self):
|
||||
return self.queries
|
||||
|
||||
@property
|
||||
def data_2(self):
|
||||
return self.documents
|
||||
|
||||
|
||||
class ScoreQueriesItemsRequest(ScoringRequestMixin):
|
||||
queries: ScoreInput | list[ScoreInput]
|
||||
items: ScoreInput | list[ScoreInput]
|
||||
|
||||
@property
|
||||
def data_1(self):
|
||||
return self.queries
|
||||
|
||||
@property
|
||||
def data_2(self):
|
||||
return self.items
|
||||
|
||||
|
||||
class ScoreTextRequest(ScoringRequestMixin):
|
||||
text_1: ScoreInput | list[ScoreInput]
|
||||
text_2: ScoreInput | list[ScoreInput]
|
||||
|
||||
@property
|
||||
def data_1(self):
|
||||
return self.text_1
|
||||
|
||||
@property
|
||||
def data_2(self):
|
||||
return self.text_2
|
||||
|
||||
|
||||
ScoreRequest: TypeAlias = (
|
||||
ScoreQueriesDocumentsRequest
|
||||
| ScoreQueriesItemsRequest
|
||||
| ScoreDataRequest
|
||||
| ScoreTextRequest
|
||||
)
|
||||
|
||||
|
||||
class RerankRequest(ScoringRequestMixin):
|
||||
# --8<-- [start:rerank-request-params]
|
||||
query: ScoreInput
|
||||
documents: ScoreInput | list[ScoreInput]
|
||||
top_n: int = Field(default=0, ge=0)
|
||||
# --8<-- [end:rerank-request-params]
|
||||
|
||||
|
||||
ScoringRequest: TypeAlias = ScoreRequest | RerankRequest
|
||||
|
||||
|
||||
class RerankDocument(BaseModel):
|
||||
text: str | None = None
|
||||
multi_modal: list[ScoreContentPartParam] | None = None
|
||||
|
||||
|
||||
class RerankResult(BaseModel):
|
||||
index: int
|
||||
document: RerankDocument
|
||||
relevance_score: float
|
||||
|
||||
|
||||
class RerankUsage(BaseModel):
|
||||
prompt_tokens: int
|
||||
total_tokens: int
|
||||
|
||||
|
||||
class RerankResponse(OpenAIBaseModel):
|
||||
id: str
|
||||
model: str
|
||||
usage: RerankUsage
|
||||
results: list[RerankResult]
|
||||
|
||||
|
||||
class ScoreResponseData(OpenAIBaseModel):
|
||||
index: int
|
||||
object: str = "score"
|
||||
score: float
|
||||
|
||||
|
||||
class ScoreResponse(OpenAIBaseModel):
|
||||
id: str = Field(default_factory=lambda: f"embd-{random_uuid()}")
|
||||
object: str = "list"
|
||||
created: int = Field(default_factory=lambda: int(time.time()))
|
||||
model: str
|
||||
data: list[ScoreResponseData]
|
||||
usage: UsageInfo
|
||||
|
||||
|
||||
ScoringResponse: TypeAlias = RerankResponse | ScoreResponse
|
||||
@@ -1,315 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from fastapi.responses import JSONResponse, Response
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.serve.engine.protocol import UsageInfo
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import PoolingRequestOutput, ScoringRequestOutput
|
||||
from vllm.tasks import SCORE_TYPE_MAP, SupportedTask
|
||||
from vllm.utils import random_uuid
|
||||
from vllm.v1.pool.late_interaction import (
|
||||
build_late_interaction_doc_params,
|
||||
build_late_interaction_query_params,
|
||||
)
|
||||
|
||||
from ..base.io_processor import PoolingIOProcessor
|
||||
from ..base.serving import PoolingServing
|
||||
from .io_processor import ScoringIOProcessors, ScoringServeContext
|
||||
from .protocol import (
|
||||
RerankDocument,
|
||||
RerankRequest,
|
||||
RerankResponse,
|
||||
RerankResult,
|
||||
RerankUsage,
|
||||
ScoreRequest,
|
||||
ScoreResponse,
|
||||
ScoreResponseData,
|
||||
)
|
||||
from .typing import ScoreInput
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class ServingScores(PoolingServing):
|
||||
request_id_prefix = "score"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
engine_client: EngineClient,
|
||||
*args,
|
||||
supported_tasks: tuple[SupportedTask, ...],
|
||||
enable_flash_late_interaction: bool = True,
|
||||
**kwargs,
|
||||
):
|
||||
pooling_task = engine_client.model_config.get_pooling_task(supported_tasks)
|
||||
score_type = SCORE_TYPE_MAP.get(pooling_task, None) # type: ignore[arg-type]
|
||||
assert score_type is not None
|
||||
|
||||
self.io_processor_name: str = score_type
|
||||
self.enable_flash_late_interaction = (
|
||||
self.io_processor_name == "late-interaction"
|
||||
and enable_flash_late_interaction
|
||||
)
|
||||
|
||||
if self.enable_flash_late_interaction:
|
||||
self.io_processor_name = "flash-late-interaction"
|
||||
|
||||
if engine_client.model_config.architecture == "JinaForRanking":
|
||||
self.io_processor_name = "jina-reranking-scoring"
|
||||
self.enable_flash_late_interaction = False
|
||||
|
||||
super().__init__(engine_client, *args, **kwargs)
|
||||
|
||||
def init_io_processor(self, *args, **kwargs) -> PoolingIOProcessor:
|
||||
return ScoringIOProcessors[self.io_processor_name](*args, **kwargs)
|
||||
|
||||
async def __call__(self, *args, **kwargs) -> Response:
|
||||
if not self.enable_flash_late_interaction:
|
||||
return await super().__call__(*args, **kwargs)
|
||||
|
||||
return await self.flash_late_interaction(*args, **kwargs)
|
||||
|
||||
def _build_response(
|
||||
self,
|
||||
ctx: ScoringServeContext,
|
||||
) -> JSONResponse:
|
||||
final_res_batch = ctx.final_res_batch
|
||||
request_id = ctx.request_id
|
||||
created_time = ctx.created_time
|
||||
model_name = ctx.model_name
|
||||
assert ctx.engine_inputs is not None
|
||||
num_prompt_tokens = sum(
|
||||
self._extract_prompt_len(engine_input["prompts"])
|
||||
for engine_input in ctx.engine_inputs
|
||||
)
|
||||
|
||||
if isinstance(ctx.request, ScoreRequest):
|
||||
return self._request_output_to_score_response(
|
||||
final_res_batch,
|
||||
request_id,
|
||||
created_time,
|
||||
model_name,
|
||||
num_prompt_tokens,
|
||||
)
|
||||
elif isinstance(ctx.request, RerankRequest):
|
||||
return self._request_output_to_rerank_response(
|
||||
final_res_batch,
|
||||
request_id,
|
||||
model_name,
|
||||
ctx.request.documents,
|
||||
ctx.request.top_n if ctx.request.top_n > 0 else len(final_res_batch),
|
||||
num_prompt_tokens,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Invalid {self.request_id_prefix} request type")
|
||||
|
||||
def _request_output_to_score_response(
|
||||
self,
|
||||
final_res_batch: list[PoolingRequestOutput],
|
||||
request_id: str,
|
||||
created_time: int,
|
||||
model_name: str,
|
||||
num_prompt_tokens: int,
|
||||
) -> JSONResponse:
|
||||
items: list[ScoreResponseData] = []
|
||||
|
||||
for idx, final_res in enumerate(final_res_batch):
|
||||
classify_res = ScoringRequestOutput.from_base(final_res)
|
||||
|
||||
item = ScoreResponseData(
|
||||
index=idx,
|
||||
score=classify_res.outputs.score,
|
||||
)
|
||||
items.append(item)
|
||||
|
||||
usage = UsageInfo(
|
||||
prompt_tokens=num_prompt_tokens,
|
||||
total_tokens=num_prompt_tokens,
|
||||
)
|
||||
|
||||
response = ScoreResponse(
|
||||
id=request_id,
|
||||
created=created_time,
|
||||
model=model_name,
|
||||
data=items,
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
return JSONResponse(content=response.model_dump())
|
||||
|
||||
def _request_output_to_rerank_response(
|
||||
self,
|
||||
final_res_batch: list[PoolingRequestOutput],
|
||||
request_id: str,
|
||||
model_name: str,
|
||||
documents: ScoreInput | list[ScoreInput],
|
||||
top_n: int,
|
||||
num_prompt_tokens: int,
|
||||
) -> JSONResponse:
|
||||
if not isinstance(documents, list):
|
||||
documents = [documents]
|
||||
|
||||
results: list[RerankResult] = []
|
||||
for idx, final_res in enumerate(final_res_batch):
|
||||
classify_res = ScoringRequestOutput.from_base(final_res)
|
||||
|
||||
document = documents[idx]
|
||||
if isinstance(document, str):
|
||||
rerank_document = RerankDocument(text=document)
|
||||
else:
|
||||
rerank_document = RerankDocument(
|
||||
multi_modal=document.get("content", [])
|
||||
)
|
||||
|
||||
result = RerankResult(
|
||||
index=idx,
|
||||
document=rerank_document,
|
||||
relevance_score=classify_res.outputs.score,
|
||||
)
|
||||
results.append(result)
|
||||
|
||||
# sort by relevance, then return the top n if set
|
||||
results.sort(key=lambda x: x.relevance_score, reverse=True)
|
||||
if top_n < len(documents):
|
||||
results = results[:top_n]
|
||||
|
||||
response = RerankResponse(
|
||||
id=request_id,
|
||||
model=model_name,
|
||||
results=results,
|
||||
usage=RerankUsage(
|
||||
total_tokens=num_prompt_tokens, prompt_tokens=num_prompt_tokens
|
||||
),
|
||||
)
|
||||
|
||||
return JSONResponse(content=response.model_dump())
|
||||
|
||||
###################################################################################
|
||||
### Run pooling score MaxSim on worker side (GPU) in the API server process
|
||||
### Can significantly improve late-interaction scoring performance.
|
||||
|
||||
async def flash_late_interaction(self, *args, **kwargs) -> Response:
|
||||
ctx = await self._init_ctx(self.io_processor, *args, **kwargs)
|
||||
await self._preprocessing(self.io_processor, ctx)
|
||||
|
||||
try:
|
||||
# stage 1: encode queries and cache token embeddings on workers.
|
||||
await self._flash_late_interaction_encode_queries(ctx)
|
||||
# stage 2: encode docs and return scalar scores from workers.
|
||||
await self._flash_late_interaction_encode_docs(ctx)
|
||||
except BaseException:
|
||||
try:
|
||||
await self._cleanup_flash_late_interaction(ctx)
|
||||
except Exception:
|
||||
logger.exception("Failed to clean up late-interaction query cache.")
|
||||
raise
|
||||
|
||||
return await self._postprocessing_async(self.io_processor, ctx)
|
||||
|
||||
async def _cleanup_flash_late_interaction(self, ctx: ScoringServeContext) -> None:
|
||||
query_keys = ctx.late_interaction_query_keys
|
||||
if not query_keys:
|
||||
return
|
||||
|
||||
doc_keys = ctx.late_interaction_doc_keys or []
|
||||
|
||||
try:
|
||||
# Stop documents before removing the query tensors they reference.
|
||||
await self.engine_client.abort([*query_keys, *doc_keys])
|
||||
finally:
|
||||
await self.engine_client.collective_rpc(
|
||||
"release_late_interaction_query_cache",
|
||||
args=(query_keys,),
|
||||
)
|
||||
|
||||
async def _flash_late_interaction_encode_queries(self, ctx: ScoringServeContext):
|
||||
assert ctx.n_queries is not None
|
||||
assert ctx.engine_inputs is not None
|
||||
assert isinstance(ctx.pooling_params, PoolingParams)
|
||||
|
||||
n_queries = ctx.n_queries
|
||||
n_docs = len(ctx.engine_inputs) - n_queries
|
||||
query_engine_inputs = ctx.engine_inputs[:n_queries]
|
||||
|
||||
query_namespace = random_uuid()
|
||||
query_keys = [
|
||||
f"late-interaction-{query_namespace}-query-{i}" for i in range(n_queries)
|
||||
]
|
||||
ctx.late_interaction_query_keys = query_keys
|
||||
query_uses = [n_docs if n_queries == 1 else 1] * n_queries
|
||||
|
||||
for i in range(n_queries):
|
||||
pooling_params = ctx.pooling_params.clone()
|
||||
pooling_params.late_interaction_params = (
|
||||
build_late_interaction_query_params(
|
||||
query_key=query_keys[i],
|
||||
query_uses=query_uses[i],
|
||||
)
|
||||
)
|
||||
query_engine_inputs[i]["params"] = pooling_params
|
||||
|
||||
assert n_queries == len(query_engine_inputs) == len(query_keys)
|
||||
|
||||
query_ctx = ScoringServeContext(
|
||||
request=ctx.request,
|
||||
raw_request=ctx.raw_request,
|
||||
model_name=ctx.model_name,
|
||||
request_id=ctx.request_id,
|
||||
pooling_params=ctx.pooling_params,
|
||||
prompt_request_ids=query_keys,
|
||||
engine_inputs=query_engine_inputs,
|
||||
lora_request=ctx.lora_request,
|
||||
priorities=ctx.priorities,
|
||||
prompt_extras=ctx.prompt_extras,
|
||||
)
|
||||
|
||||
await self._prepare_generators(query_ctx)
|
||||
await self._collect_batch(query_ctx)
|
||||
ctx.query_final_res_batch = query_ctx.final_res_batch
|
||||
|
||||
async def _flash_late_interaction_encode_docs(self, ctx: ScoringServeContext):
|
||||
assert ctx.n_queries is not None
|
||||
assert ctx.engine_inputs is not None
|
||||
assert isinstance(ctx.pooling_params, PoolingParams)
|
||||
|
||||
n_queries = ctx.n_queries
|
||||
n_docs = len(ctx.engine_inputs) - n_queries
|
||||
doc_engine_inputs = ctx.engine_inputs[n_queries:]
|
||||
|
||||
query_keys = ctx.late_interaction_query_keys
|
||||
if query_keys is None:
|
||||
raise RuntimeError("Late-interaction query keys were not initialized.")
|
||||
doc_namespace = random_uuid()
|
||||
doc_keys = [f"late-interaction-{doc_namespace}-doc-{i}" for i in range(n_docs)]
|
||||
ctx.late_interaction_doc_keys = doc_keys
|
||||
|
||||
for i in range(n_docs):
|
||||
query_idx = 0 if n_queries == 1 else i
|
||||
pooling_params = ctx.pooling_params.clone()
|
||||
pooling_params.late_interaction_params = build_late_interaction_doc_params(
|
||||
query_key=query_keys[query_idx]
|
||||
)
|
||||
doc_engine_inputs[i]["params"] = pooling_params
|
||||
|
||||
assert n_docs == len(doc_engine_inputs) == len(doc_keys)
|
||||
|
||||
doc_ctx = ScoringServeContext(
|
||||
request=ctx.request,
|
||||
raw_request=ctx.raw_request,
|
||||
model_name=ctx.model_name,
|
||||
request_id=ctx.request_id,
|
||||
pooling_params=ctx.pooling_params,
|
||||
prompt_request_ids=doc_keys,
|
||||
engine_inputs=doc_engine_inputs,
|
||||
lora_request=ctx.lora_request,
|
||||
priorities=ctx.priorities,
|
||||
prompt_extras=ctx.prompt_extras,
|
||||
)
|
||||
|
||||
await self._prepare_generators(doc_ctx)
|
||||
await self._collect_batch(doc_ctx)
|
||||
|
||||
ctx.final_res_batch = doc_ctx.final_res_batch
|
||||
@@ -1,45 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from dataclasses import dataclass
|
||||
from typing import Required, TypeAlias
|
||||
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
from vllm.renderers.chat_utils import (
|
||||
ChatCompletionContentPartImageEmbedsParam,
|
||||
ChatCompletionContentPartImageParam,
|
||||
ChatCompletionContentPartTextParam,
|
||||
ChatCompletionContentPartVideoParam,
|
||||
)
|
||||
|
||||
ScoreContentPartParam: TypeAlias = (
|
||||
ChatCompletionContentPartImageParam
|
||||
| ChatCompletionContentPartImageEmbedsParam
|
||||
| ChatCompletionContentPartTextParam
|
||||
| ChatCompletionContentPartVideoParam
|
||||
)
|
||||
|
||||
|
||||
class ScoreMultiModalParam(TypedDict, total=False):
|
||||
"""A specialized parameter type for scoring multimodal content.
|
||||
|
||||
The reasons why don't reuse `CustomChatCompletionMessageParam` directly:
|
||||
1. Score tasks don't need the 'role' field (user/assistant/system) that's required in chat completions
|
||||
2. Including chat-specific fields would confuse users about their purpose in scoring
|
||||
3. This is a more focused interface that only exposes what's needed for scoring
|
||||
""" # noqa: E501
|
||||
|
||||
content: Required[list[ScoreContentPartParam]]
|
||||
"""The multimodal contents"""
|
||||
|
||||
|
||||
# Raw input data with content key in ScoreMultiModalParam.
|
||||
ScoreInput = str | ScoreMultiModalParam
|
||||
# Score data without content key.
|
||||
ScoreData = str | list[ScoreContentPartParam]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScoringData:
|
||||
data_1: list[ScoreData]
|
||||
data_2: list[ScoreData]
|
||||
@@ -1,291 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Iterable
|
||||
from typing import cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm import PromptType, TextPrompt
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.inputs import MultiModalDataDict, MultiModalUUIDDict
|
||||
from vllm.renderers.chat_utils import (
|
||||
BaseMultiModalItemTracker,
|
||||
ChatCompletionContentPartParam,
|
||||
ChatCompletionContentPartTextParam,
|
||||
ConversationMessage,
|
||||
MultiModalItemTracker,
|
||||
parse_chat_message_content_parts,
|
||||
)
|
||||
|
||||
from .typing import (
|
||||
ScoreContentPartParam,
|
||||
ScoreData,
|
||||
ScoreInput,
|
||||
ScoringData,
|
||||
)
|
||||
|
||||
|
||||
def get_num_special_tokens_for_pair(tokenizer) -> int:
|
||||
"""Get number of special tokens added for a text pair encoding."""
|
||||
method = getattr(tokenizer, "num_special_tokens_to_add", None)
|
||||
if method is not None:
|
||||
try:
|
||||
return method(pair=True)
|
||||
except TypeError:
|
||||
pass
|
||||
# Fallback: compute by tokenizing empty strings
|
||||
empty_encoding = tokenizer("", text_pair="", add_special_tokens=True)
|
||||
return len(empty_encoding["input_ids"])
|
||||
|
||||
|
||||
def truncate_text_to_tokens(
|
||||
text: str,
|
||||
tokenizer,
|
||||
max_tokens: int,
|
||||
) -> str:
|
||||
"""Truncate text to a maximum number of content tokens.
|
||||
|
||||
Uses offset_mapping to slice the original text at the exact character
|
||||
boundary, avoiding lossy encode→decode round-trips that can shift
|
||||
the token count due to BPE merge boundary changes.
|
||||
"""
|
||||
if max_tokens <= 0:
|
||||
return ""
|
||||
|
||||
while text:
|
||||
encoding = tokenizer(
|
||||
text,
|
||||
add_special_tokens=False,
|
||||
return_offsets_mapping=True,
|
||||
)
|
||||
offsets = encoding["offset_mapping"]
|
||||
if len(encoding["input_ids"]) <= max_tokens:
|
||||
break
|
||||
|
||||
char_end = min(offsets[max_tokens - 1][1], offsets[max_tokens][0])
|
||||
if char_end >= len(text):
|
||||
char_end = len(text) - 1
|
||||
text = text[:char_end]
|
||||
|
||||
return text
|
||||
|
||||
|
||||
def compute_maxsim_score(q_emb: torch.Tensor, d_emb: torch.Tensor) -> torch.Tensor:
|
||||
"""Compute ColBERT MaxSim score.
|
||||
|
||||
Args:
|
||||
q_emb: Query token embeddings [query_len, dim]
|
||||
d_emb: Document token embeddings [doc_len, dim]
|
||||
|
||||
Returns:
|
||||
MaxSim score (sum over query tokens of max similarity to any doc token)
|
||||
|
||||
"""
|
||||
# compute in float32 for numerical stability
|
||||
# [query_len, doc_len]
|
||||
token_scores = torch.matmul(q_emb.float(), d_emb.float().T)
|
||||
# Max over document tokens, sum over query tokens
|
||||
return token_scores.amax(dim=-1).sum()
|
||||
|
||||
|
||||
def _validate_mm_score_input(
|
||||
data: list[ScoreInput],
|
||||
is_multimodal_model: bool,
|
||||
architecture: str,
|
||||
) -> list[ScoreData]:
|
||||
out: list[ScoreData] = []
|
||||
for d in data:
|
||||
if isinstance(d, str):
|
||||
out.append(d)
|
||||
else:
|
||||
if not is_multimodal_model:
|
||||
raise VLLMValidationError(
|
||||
f"MultiModalParam is not supported for {architecture}"
|
||||
)
|
||||
content = cast(list[ScoreContentPartParam], d.get("content", []))
|
||||
out.append(content)
|
||||
return out
|
||||
|
||||
|
||||
def _validate_score_input_lens(
|
||||
data_1: list[ScoreData],
|
||||
data_2: list[ScoreData],
|
||||
):
|
||||
len_1 = len(data_1)
|
||||
len_2 = len(data_2)
|
||||
|
||||
if len_1 > 1 and len_1 != len_2:
|
||||
raise VLLMValidationError("Input lengths must be either 1:1, 1:N or N:N")
|
||||
if len_1 == 0:
|
||||
raise VLLMValidationError("At least one text element must be given")
|
||||
if len_2 == 0:
|
||||
raise VLLMValidationError("At least one text_pair element must be given")
|
||||
|
||||
|
||||
def validate_score_input(
|
||||
data_1: ScoreInput | list[ScoreInput],
|
||||
data_2: ScoreInput | list[ScoreInput],
|
||||
is_multimodal_model: bool,
|
||||
architecture: str,
|
||||
) -> ScoringData:
|
||||
if not isinstance(data_1, list):
|
||||
data_1 = [data_1]
|
||||
|
||||
if not isinstance(data_2, list):
|
||||
data_2 = [data_2]
|
||||
|
||||
score_input_1 = _validate_mm_score_input(data_1, is_multimodal_model, architecture)
|
||||
score_input_2 = _validate_mm_score_input(data_2, is_multimodal_model, architecture)
|
||||
_validate_score_input_lens(score_input_1, score_input_2)
|
||||
return ScoringData(data_1=score_input_1, data_2=score_input_2)
|
||||
|
||||
|
||||
def score_data_to_prompts(
|
||||
data_list: list[ScoreData],
|
||||
role: str,
|
||||
model_config: ModelConfig,
|
||||
) -> list[PromptType]:
|
||||
"""Convert a list of ScoreData into PromptType objects.
|
||||
|
||||
For plain text inputs, returns the string directly.
|
||||
For multimodal inputs (list of content parts), parses them into
|
||||
a :class:`TextPrompt` with attached ``multi_modal_data`` /
|
||||
``multi_modal_uuids``.
|
||||
|
||||
This is used by late-interaction scoring where each query/document
|
||||
is encoded independently.
|
||||
"""
|
||||
prompts: list[PromptType] = []
|
||||
for data in data_list:
|
||||
if isinstance(data, str):
|
||||
prompts.append(data)
|
||||
else:
|
||||
text, mm_data, mm_uuids = parse_score_data_single(data, role, model_config)
|
||||
prompt: TextPrompt = TextPrompt(prompt=text)
|
||||
if mm_data is not None:
|
||||
prompt["multi_modal_data"] = mm_data
|
||||
if mm_uuids is not None:
|
||||
prompt["multi_modal_uuids"] = mm_uuids
|
||||
prompts.append(prompt)
|
||||
return prompts
|
||||
|
||||
|
||||
def _ensure_str(content: list[ConversationMessage]) -> str:
|
||||
"""Extract a single string prompt from parsed conversation content."""
|
||||
assert len(content) == 1
|
||||
prompt = content[0]["content"]
|
||||
if prompt is not None and isinstance(prompt, str):
|
||||
return cast(str, prompt)
|
||||
raise ValueError(f"Only string content is supported, but got {content}.")
|
||||
|
||||
|
||||
def _parse_score_content(
|
||||
role: str,
|
||||
data: ScoreData,
|
||||
mm_tracker: BaseMultiModalItemTracker,
|
||||
) -> list[ConversationMessage]:
|
||||
parts: Iterable[ChatCompletionContentPartParam]
|
||||
if isinstance(data, str):
|
||||
parts = [ChatCompletionContentPartTextParam(type="text", text=data)]
|
||||
else:
|
||||
parts = cast(Iterable[ChatCompletionContentPartParam], data)
|
||||
|
||||
mm_parser = mm_tracker.create_parser()
|
||||
|
||||
parse_res = parse_chat_message_content_parts(
|
||||
role=role,
|
||||
parts=parts,
|
||||
mm_tracker=mm_tracker,
|
||||
wrap_dicts=False,
|
||||
interleave_strings=False,
|
||||
multimodal_content_part_separator="",
|
||||
)
|
||||
|
||||
if parse_res:
|
||||
return parse_res
|
||||
|
||||
mm_placeholder_storage = mm_parser.mm_placeholder_storage()
|
||||
|
||||
if (
|
||||
len(mm_placeholder_storage) != 1
|
||||
or len(next(iter(mm_placeholder_storage.values()))) != 1
|
||||
):
|
||||
raise ValueError("Only one multi-modal item is supported")
|
||||
|
||||
return next(iter(mm_placeholder_storage.values()))[0]
|
||||
|
||||
|
||||
def parse_score_data_single(
|
||||
data: ScoreData,
|
||||
role: str,
|
||||
model_config: ModelConfig,
|
||||
) -> tuple[str, MultiModalDataDict | None, MultiModalUUIDDict | None]:
|
||||
"""Parse **one** ScoreData into a text prompt and its own multi-modal
|
||||
data.
|
||||
|
||||
Unlike :func:`parse_score_data`, each call creates an **independent**
|
||||
:class:`MultiModalItemTracker` so multi-modal items are kept separate.
|
||||
This is the correct behaviour for late-interaction scoring, where
|
||||
query and document are encoded independently.
|
||||
"""
|
||||
mm_tracker = MultiModalItemTracker(model_config)
|
||||
content = _parse_score_content(role, data, mm_tracker)
|
||||
|
||||
prompt = _ensure_str(content)
|
||||
mm_items, mm_uuids = mm_tracker.resolve_items()
|
||||
return prompt, mm_items, mm_uuids
|
||||
|
||||
|
||||
def parse_score_data(
|
||||
data_1: ScoreData,
|
||||
data_2: ScoreData,
|
||||
model_config: ModelConfig,
|
||||
) -> tuple[str, str, MultiModalDataDict | None, MultiModalUUIDDict | None]:
|
||||
"""Parse a query-document pair into text prompts and shared multi-modal
|
||||
data.
|
||||
|
||||
Uses a **single** :class:`MultiModalItemTracker` so that multi-modal
|
||||
items from both inputs are merged into one ``mm_data`` dict. This is
|
||||
the correct behaviour for cross-encoder scoring, where query and
|
||||
document are concatenated into a single model prompt.
|
||||
"""
|
||||
mm_tracker = MultiModalItemTracker(model_config)
|
||||
|
||||
content_1 = _parse_score_content("query", data_1, mm_tracker)
|
||||
content_2 = _parse_score_content("document", data_2, mm_tracker)
|
||||
|
||||
prompt_1 = _ensure_str(content_1)
|
||||
prompt_2 = _ensure_str(content_2)
|
||||
mm_items, mm_uuids = mm_tracker.resolve_items()
|
||||
|
||||
return prompt_1, prompt_2, mm_items, mm_uuids
|
||||
|
||||
|
||||
def compress_token_type_ids(token_type_ids: list[int]) -> tuple[int, int]:
|
||||
"""Return the half-open range containing token type 1.
|
||||
|
||||
Right padding uses the tokenizer's padding token type, which is commonly
|
||||
zero. Therefore, a padded query-document pair can contain zeros followed
|
||||
by ones followed by trailing zeros.
|
||||
"""
|
||||
first_one = len(token_type_ids)
|
||||
first_zero_after_one = len(token_type_ids)
|
||||
err_msg = (
|
||||
"Token type ids are expected to be a sequence"
|
||||
" of zeros followed by ones and optional trailing zeros"
|
||||
)
|
||||
for i, type_id in enumerate(token_type_ids):
|
||||
if type_id not in (0, 1):
|
||||
raise ValueError(err_msg)
|
||||
if type_id == 1:
|
||||
if first_zero_after_one < len(token_type_ids):
|
||||
raise ValueError(err_msg)
|
||||
if first_one == len(token_type_ids):
|
||||
first_one = i
|
||||
elif first_one < len(token_type_ids) and first_zero_after_one == len(
|
||||
token_type_ids
|
||||
):
|
||||
first_zero_after_one = i
|
||||
|
||||
return first_one, first_zero_after_one
|
||||
@@ -1,214 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import time
|
||||
from collections.abc import AsyncGenerator, Callable, Generator, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Generic, NotRequired, TypeAlias, TypedDict, TypeVar
|
||||
|
||||
from fastapi import Request
|
||||
from pydantic import ConfigDict
|
||||
|
||||
from vllm import PoolingParams, PoolingRequestOutput, PromptType
|
||||
from vllm.inputs import DataPrompt, EngineInput
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.outputs import LateChunkingMetadata
|
||||
from vllm.renderers import ChatParams, TokenizeParams
|
||||
from vllm.renderers.chat_utils import ChatCompletionMessageParam
|
||||
from vllm.renderers.inputs import DictPrompt
|
||||
|
||||
from ...tasks import PoolingTask
|
||||
from .classify.protocol import (
|
||||
ClassificationChatRequest,
|
||||
ClassificationCompletionRequest,
|
||||
ClassificationResponse,
|
||||
)
|
||||
from .embed.protocol import (
|
||||
CohereEmbedRequest,
|
||||
EmbeddingBytesResponse,
|
||||
EmbeddingChatInputRequest,
|
||||
EmbeddingChatRequest,
|
||||
EmbeddingCompletionRequest,
|
||||
EmbeddingRequest,
|
||||
EmbeddingResponse,
|
||||
)
|
||||
from .pooling.protocol import (
|
||||
IOProcessorRequest,
|
||||
PoolingBytesResponse,
|
||||
PoolingChatRequest,
|
||||
PoolingCompletionRequest,
|
||||
PoolingResponse,
|
||||
)
|
||||
from .scoring.protocol import ScoringRequest, ScoringResponse
|
||||
from .scoring.typing import ScoreData, ScoringData
|
||||
|
||||
PoolingCompletionLikeRequest: TypeAlias = (
|
||||
EmbeddingCompletionRequest
|
||||
| ClassificationCompletionRequest
|
||||
| PoolingCompletionRequest
|
||||
)
|
||||
|
||||
PoolingChatLikeRequest: TypeAlias = (
|
||||
EmbeddingChatRequest
|
||||
| EmbeddingChatInputRequest
|
||||
| ClassificationChatRequest
|
||||
| PoolingChatRequest
|
||||
)
|
||||
|
||||
AnyPoolingRequest: TypeAlias = (
|
||||
EmbeddingRequest
|
||||
| PoolingCompletionLikeRequest
|
||||
| PoolingChatLikeRequest
|
||||
| IOProcessorRequest
|
||||
| ScoringRequest
|
||||
| CohereEmbedRequest
|
||||
)
|
||||
|
||||
AnyPoolingResponse: TypeAlias = (
|
||||
ClassificationResponse
|
||||
| EmbeddingResponse
|
||||
| EmbeddingBytesResponse
|
||||
| PoolingResponse
|
||||
| PoolingBytesResponse
|
||||
| ScoringResponse
|
||||
)
|
||||
|
||||
PoolingRequestT = TypeVar("PoolingRequestT", bound=AnyPoolingRequest)
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class ChunkedEmbeddingMetadata:
|
||||
prompt_index: int
|
||||
chunk_index: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class LateChunkingSlot:
|
||||
"""Per-document source ranges shared by frontend rendering and output contexts."""
|
||||
|
||||
metadata: LateChunkingMetadata | None = None
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class PoolingServeContext(Generic[PoolingRequestT]):
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
|
||||
request: PoolingRequestT
|
||||
raw_request: Request | None = None
|
||||
model_name: str
|
||||
request_id: str
|
||||
pooling_params: PoolingParams
|
||||
lora_request: LoRARequest | None
|
||||
priorities: int | Sequence[int] | None
|
||||
prompt_extras: dict[str, Any] | None
|
||||
|
||||
created_time: int = field(default_factory=lambda: int(time.time()))
|
||||
engine_inputs: Sequence["PoolingEngineInput"] | None = None
|
||||
prompt_request_ids: list[str] | None = None
|
||||
|
||||
result_generator: AsyncGenerator[tuple[int, PoolingRequestOutput], None] | None = (
|
||||
None
|
||||
)
|
||||
final_res_batch: list[PoolingRequestOutput] = field(default_factory=list)
|
||||
|
||||
## for Long Text Embedding with Chunked Processing
|
||||
original_engine_inputs: Sequence["PoolingEngineInput"] | None = None
|
||||
chunked_embedding_metadata: list[ChunkedEmbeddingMetadata] | None = None
|
||||
|
||||
## for bi-encoder & late-interaction
|
||||
n_queries: int | None = None
|
||||
|
||||
## for IOProcessorResponse
|
||||
response: Any | None = None
|
||||
|
||||
## for flash-late-interaction
|
||||
query_final_res_batch: list[PoolingRequestOutput] | None = None
|
||||
late_interaction_query_keys: list[str] | None = None
|
||||
late_interaction_doc_keys: list[str] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class OfflineInputsContext:
|
||||
pooling_task: PoolingTask
|
||||
tokenization_kwargs: dict[str, Any] | None
|
||||
lora_request: Sequence[LoRARequest | None] | None
|
||||
priorities: int | Sequence[int] | None
|
||||
late_chunking: list[LateChunkingSlot | None] = field(
|
||||
default_factory=list, kw_only=True
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class OfflineEncodeInputsContext(OfflineInputsContext):
|
||||
prompts: PromptType | Sequence[PromptType]
|
||||
pooling_params: PoolingParams | Sequence[PoolingParams] | None
|
||||
|
||||
|
||||
@dataclass
|
||||
class OfflineScoringInputsContext(OfflineInputsContext):
|
||||
scoring_data: ScoringData
|
||||
chat_template: str | None
|
||||
pooling_params: PoolingParams
|
||||
|
||||
|
||||
@dataclass
|
||||
class OfflinePluginInputsContext(OfflineInputsContext):
|
||||
prompts: DataPrompt
|
||||
pooling_params: PoolingParams | Sequence[PoolingParams] | None
|
||||
|
||||
|
||||
AnyOfflineInputsContext: TypeAlias = (
|
||||
OfflineEncodeInputsContext
|
||||
| OfflineScoringInputsContext
|
||||
| OfflinePluginInputsContext
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class OfflineOutputsContext:
|
||||
outputs: list[PoolingRequestOutput]
|
||||
|
||||
## for bi-encoder & late-interaction
|
||||
n_queries: int | None = None
|
||||
late_chunking: list[LateChunkingSlot | None] = field(default_factory=list)
|
||||
|
||||
|
||||
class RenderParams(TypedDict):
|
||||
tok_params: TokenizeParams
|
||||
prompt_extras: dict[str, Any] | None
|
||||
skip_mm_cache: bool
|
||||
|
||||
params: PoolingParams
|
||||
lora_requests: LoRARequest | None
|
||||
priorities: int
|
||||
late_chunking: NotRequired[LateChunkingSlot]
|
||||
|
||||
|
||||
class EncodeCMPLRenderParams(RenderParams):
|
||||
prompts: DictPrompt
|
||||
|
||||
|
||||
class EncodeChatRenderParams(RenderParams):
|
||||
conversations: list["ChatCompletionMessageParam"]
|
||||
chat_params: ChatParams
|
||||
|
||||
|
||||
class ScoringRenderParams(RenderParams):
|
||||
data_1: ScoreData
|
||||
data_2: ScoreData
|
||||
chat_template: str | None
|
||||
max_tokens_per_query: int
|
||||
max_tokens_per_doc: int
|
||||
|
||||
|
||||
AnyRenderParam: TypeAlias = (
|
||||
EncodeCMPLRenderParams | EncodeChatRenderParams | ScoringRenderParams
|
||||
)
|
||||
RequestGenerator: TypeAlias = Generator[AnyRenderParam]
|
||||
RequestFactory: TypeAlias = Callable[[], RequestGenerator]
|
||||
|
||||
|
||||
class PoolingEngineInput(TypedDict):
|
||||
prompts: EngineInput
|
||||
params: PoolingParams
|
||||
lora_requests: LoRARequest | None
|
||||
priorities: int
|
||||
@@ -1,250 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import math
|
||||
from collections.abc import Callable, Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache, partial
|
||||
from typing import Any, Literal, cast
|
||||
|
||||
import pybase64
|
||||
import torch
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.serve.engine.protocol import UsageInfo
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import PoolingRequestOutput
|
||||
from vllm.tasks import SupportedTask
|
||||
from vllm.utils.serial_utils import (
|
||||
EMBED_DTYPES,
|
||||
EmbedDType,
|
||||
Endianness,
|
||||
binary2tensor,
|
||||
tensor2binary,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
JsonEncodingFormat = Literal["float", "base64"]
|
||||
BytesEncodingFormat = Literal["bytes", "bytes_only"]
|
||||
FloatEncodedPoolingOutput = list[float] | list[list[float]]
|
||||
JsonEncodedPoolingOutput = FloatEncodedPoolingOutput | str
|
||||
|
||||
|
||||
@dataclass
|
||||
class MetadataItem:
|
||||
index: int
|
||||
embed_dtype: EmbedDType
|
||||
endianness: Endianness
|
||||
start: int
|
||||
end: int
|
||||
shape: tuple[int, ...]
|
||||
|
||||
|
||||
def build_metadata_items(
|
||||
embed_dtype: EmbedDType,
|
||||
endianness: Endianness,
|
||||
shape: tuple[int, ...],
|
||||
n_request: int,
|
||||
) -> list[MetadataItem]:
|
||||
n_bytes = EMBED_DTYPES[embed_dtype].nbytes
|
||||
size = math.prod(shape)
|
||||
|
||||
return [
|
||||
MetadataItem(
|
||||
index=i,
|
||||
embed_dtype=embed_dtype,
|
||||
endianness=endianness,
|
||||
start=i * size * n_bytes,
|
||||
end=(i + 1) * size * n_bytes,
|
||||
shape=shape,
|
||||
)
|
||||
for i in range(n_request)
|
||||
]
|
||||
|
||||
|
||||
def encode_pooling_output_float(
|
||||
output: PoolingRequestOutput,
|
||||
) -> FloatEncodedPoolingOutput:
|
||||
return output.outputs.data.tolist()
|
||||
|
||||
|
||||
def encode_pooling_output_float_or_ndarray(output: PoolingRequestOutput) -> Any:
|
||||
"""Return an ndarray when the response renderer can serialize NumPy."""
|
||||
try:
|
||||
data = output.outputs.data
|
||||
if not data.is_contiguous():
|
||||
data = data.contiguous()
|
||||
return data.numpy()
|
||||
except (RuntimeError, TypeError):
|
||||
return output.outputs.data.tolist()
|
||||
|
||||
|
||||
def encode_pooling_output_base64(
|
||||
output: PoolingRequestOutput,
|
||||
embed_dtype: EmbedDType,
|
||||
endianness: Endianness,
|
||||
) -> str:
|
||||
embedding_bytes = tensor2binary(output.outputs.data, embed_dtype, endianness)
|
||||
return pybase64.b64encode(embedding_bytes).decode("utf-8")
|
||||
|
||||
|
||||
def encode_pooling_bytes(
|
||||
pooling_outputs: list[PoolingRequestOutput],
|
||||
embed_dtype: EmbedDType,
|
||||
endianness: Endianness,
|
||||
) -> tuple[list[bytes], list[dict[str, Any]]]:
|
||||
items: list[dict[str, Any]] = []
|
||||
body: list[bytes] = []
|
||||
offset = 0
|
||||
for idx, output in enumerate(pooling_outputs):
|
||||
binary = tensor2binary(
|
||||
tensor=output.outputs.data,
|
||||
embed_dtype=embed_dtype,
|
||||
endianness=endianness,
|
||||
)
|
||||
size = len(binary)
|
||||
|
||||
# Dictionary form of MetadataItem
|
||||
item = dict(
|
||||
index=idx,
|
||||
embed_dtype=embed_dtype,
|
||||
endianness=endianness,
|
||||
start=offset,
|
||||
end=offset + size,
|
||||
shape=output.outputs.data.shape,
|
||||
)
|
||||
|
||||
body.append(binary)
|
||||
items.append(item)
|
||||
offset += size
|
||||
|
||||
return body, items
|
||||
|
||||
|
||||
def get_pooling_output_encoder(
|
||||
encoding_format: JsonEncodingFormat,
|
||||
embed_dtype: EmbedDType,
|
||||
endianness: Endianness,
|
||||
) -> Callable[[PoolingRequestOutput], JsonEncodedPoolingOutput]:
|
||||
return cast(
|
||||
Callable[[PoolingRequestOutput], JsonEncodedPoolingOutput],
|
||||
(
|
||||
encode_pooling_output_float
|
||||
if encoding_format == "float"
|
||||
else partial(
|
||||
encode_pooling_output_base64,
|
||||
embed_dtype=embed_dtype,
|
||||
endianness=endianness,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def get_pooling_usage(
|
||||
pooling_outputs: Sequence[PoolingRequestOutput],
|
||||
) -> UsageInfo:
|
||||
num_prompt_tokens = sum(
|
||||
len(output.prompt_token_ids) if output.prompt_token_ids is not None else 0
|
||||
for output in pooling_outputs
|
||||
)
|
||||
return UsageInfo(
|
||||
prompt_tokens=num_prompt_tokens,
|
||||
total_tokens=num_prompt_tokens,
|
||||
)
|
||||
|
||||
|
||||
def get_pooling_usage_payload(
|
||||
pooling_outputs: Sequence[PoolingRequestOutput],
|
||||
) -> dict[str, int]:
|
||||
usage = get_pooling_usage(pooling_outputs)
|
||||
return {
|
||||
"prompt_tokens": usage.prompt_tokens,
|
||||
"total_tokens": usage.total_tokens,
|
||||
}
|
||||
|
||||
|
||||
def build_pooling_bytes_streaming_response(
|
||||
pooling_outputs: list[PoolingRequestOutput],
|
||||
request_id: str,
|
||||
created_time: int,
|
||||
model_name: str,
|
||||
encoding_format: BytesEncodingFormat,
|
||||
embed_dtype: EmbedDType,
|
||||
endianness: Endianness,
|
||||
) -> StreamingResponse:
|
||||
content, items = encode_pooling_bytes(
|
||||
pooling_outputs=pooling_outputs,
|
||||
embed_dtype=embed_dtype,
|
||||
endianness=endianness,
|
||||
)
|
||||
|
||||
headers = (
|
||||
None
|
||||
if encoding_format == "bytes_only"
|
||||
else {
|
||||
"metadata": json.dumps(
|
||||
{
|
||||
"id": request_id,
|
||||
"created": created_time,
|
||||
"model": model_name,
|
||||
"data": items,
|
||||
"usage": get_pooling_usage_payload(pooling_outputs),
|
||||
}
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
return StreamingResponse(
|
||||
content=content,
|
||||
headers=headers,
|
||||
media_type="application/octet-stream",
|
||||
)
|
||||
|
||||
|
||||
def decode_pooling_output(items: list[MetadataItem], body: bytes) -> list[torch.Tensor]:
|
||||
return [
|
||||
binary2tensor(
|
||||
body[item.start : item.end],
|
||||
item.shape,
|
||||
item.embed_dtype,
|
||||
item.endianness,
|
||||
)
|
||||
for item in sorted(items, key=lambda x: x.index)
|
||||
]
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_json_response_cls() -> type[JSONResponse]:
|
||||
if importlib.util.find_spec("orjson") is not None:
|
||||
from fastapi.responses import ORJSONResponse
|
||||
|
||||
return ORJSONResponse
|
||||
logger.warning_once(
|
||||
"To make v1/embeddings API fast, please install orjson by `pip install orjson`"
|
||||
)
|
||||
return JSONResponse
|
||||
|
||||
|
||||
def enable_scoring_api(
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
model_config: ModelConfig | None = None,
|
||||
) -> bool:
|
||||
if model_config is None:
|
||||
return False
|
||||
|
||||
pooling_task = model_config.get_pooling_task(supported_tasks)
|
||||
if pooling_task in ("embed", "token_embed"):
|
||||
return True
|
||||
|
||||
if pooling_task == "classify":
|
||||
num_labels = getattr(model_config.hf_config, "num_labels", 0)
|
||||
if num_labels != 1:
|
||||
logger.debug_once("Scoring API is only enabled for num_labels == 1.")
|
||||
return False
|
||||
return True
|
||||
|
||||
return False
|
||||
@@ -11,7 +11,6 @@ from vllm.entrypoints.openai.models.serving import (
|
||||
OpenAIModelRegistry,
|
||||
OpenAIServingModels,
|
||||
)
|
||||
from vllm.entrypoints.pooling.typing import AnyPoolingRequest
|
||||
from vllm.entrypoints.serve import create_error_response
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.typing import AnyRequest
|
||||
@@ -48,7 +47,7 @@ class BaseServing:
|
||||
|
||||
async def _check_model(
|
||||
self,
|
||||
request: AnyRequest | AnyPoolingRequest,
|
||||
request: AnyRequest,
|
||||
) -> ErrorResponse | None:
|
||||
if self.request_logger is not None:
|
||||
self.request_logger.log_request_body(request)
|
||||
@@ -141,7 +140,7 @@ class BaseServing:
|
||||
|
||||
return random_uuid() if default is None else default
|
||||
|
||||
def _get_message_types(self, request: AnyRequest | AnyPoolingRequest) -> set[str]:
|
||||
def _get_message_types(self, request: AnyRequest) -> set[str]:
|
||||
"""Retrieve the set of types from message content dicts up
|
||||
until `_`; we use this to match potential multimodal data
|
||||
with default per modality loras.
|
||||
@@ -166,9 +165,7 @@ class BaseServing:
|
||||
message_types.add(content_dict["type"].split("_")[0])
|
||||
return message_types
|
||||
|
||||
def _get_active_default_mm_loras(
|
||||
self, request: AnyRequest | AnyPoolingRequest
|
||||
) -> LoRARequest | None:
|
||||
def _get_active_default_mm_loras(self, request: AnyRequest) -> LoRARequest | None:
|
||||
"""Determine if there are any active default multimodal loras."""
|
||||
# TODO: Currently this is only enabled for chat completions
|
||||
# to be better aligned with only being enabled for .generate
|
||||
@@ -193,7 +190,7 @@ class BaseServing:
|
||||
|
||||
def _maybe_get_adapters(
|
||||
self,
|
||||
request: AnyRequest | AnyPoolingRequest,
|
||||
request: AnyRequest,
|
||||
supports_default_mm_loras: bool = False,
|
||||
) -> LoRARequest | None:
|
||||
if request.model in self.models.lora_requests:
|
||||
|
||||
@@ -12,7 +12,6 @@ from fastapi.responses import JSONResponse, Response
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.generate.factories import get_generate_invocation_types
|
||||
from vllm.entrypoints.pooling.factories import get_pooling_invocation_types
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.instrumentator.basic import base
|
||||
@@ -79,9 +78,7 @@ def _attach_router(
|
||||
router = APIRouter()
|
||||
|
||||
# NOTE: Construct the TypeAdapters only once
|
||||
INVOCATION_TYPES = get_generate_invocation_types(
|
||||
supported_tasks, model_config
|
||||
) + get_pooling_invocation_types(supported_tasks, model_config)
|
||||
INVOCATION_TYPES = get_generate_invocation_types(supported_tasks, model_config)
|
||||
|
||||
INVOCATION_VALIDATORS = [
|
||||
(pydantic.TypeAdapter(request_type), (get_handler, endpoint))
|
||||
|
||||
@@ -7,7 +7,6 @@ from collections.abc import Sequence
|
||||
import torch
|
||||
|
||||
from vllm import envs
|
||||
from vllm.entrypoints.pooling.typing import AnyPoolingRequest
|
||||
from vllm.entrypoints.serve.engine.typing import AnyRequest
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
@@ -70,7 +69,7 @@ class RequestLogger:
|
||||
lora_request,
|
||||
)
|
||||
|
||||
def log_request_body(self, request: AnyRequest | AnyPoolingRequest) -> None:
|
||||
def log_request_body(self, request: AnyRequest) -> None:
|
||||
try:
|
||||
level = logging.getLevelNamesMapping()[envs.VLLM_REQUEST_BODY_LOG_LEVEL]
|
||||
except KeyError:
|
||||
|
||||
@@ -95,7 +95,6 @@ if TYPE_CHECKING:
|
||||
MAX_JOBS: str | None = None
|
||||
NVCC_THREADS: str | None = None
|
||||
VLLM_USE_PRECOMPILED: bool = False
|
||||
VLLM_USE_PRECOMPILED_RUST: bool = False
|
||||
VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX: bool = False
|
||||
VLLM_DOCKER_BUILD_CONTEXT: bool = False
|
||||
VLLM_BUILD_COMMIT: str = "unknown"
|
||||
@@ -163,9 +162,6 @@ if TYPE_CHECKING:
|
||||
VLLM_DISABLE_COMPILE_CACHE: bool = False
|
||||
VLLM_REPLICATE_EMBED: bool = False
|
||||
VLLM_USE_LAYERNAME: bool = True
|
||||
VLLM_USE_RUST_FRONTEND: bool = False
|
||||
VLLM_USE_RUST_BENCH: bool = False
|
||||
VLLM_RUST_FRONTEND_PATH: str | None = "auto"
|
||||
VLLM_SERVER_DEV_MODE: bool = False
|
||||
VLLM_V1_OUTPUT_PROC_CHUNK_SIZE: int = 128
|
||||
VLLM_MLA_DISABLE: bool = False
|
||||
@@ -579,41 +575,6 @@ def get_env_or_set_default(
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _resolve_rust_cli_path() -> str | None:
|
||||
"""Resolve the vllm-rs binary path.
|
||||
|
||||
Returns None unless VLLM_USE_RUST_FRONTEND or VLLM_USE_RUST_BENCH is enabled.
|
||||
When enabled, resolves VLLM_RUST_FRONTEND_PATH ("auto" by default)
|
||||
to the actual binary path.
|
||||
"""
|
||||
use_rust = bool(int(os.environ.get("VLLM_USE_RUST_FRONTEND", "0"))) or bool(
|
||||
int(os.environ.get("VLLM_USE_RUST_BENCH", "0"))
|
||||
)
|
||||
raw = os.environ.get("VLLM_RUST_FRONTEND_PATH", "auto")
|
||||
|
||||
if not use_rust:
|
||||
if os.environ.get("VLLM_RUST_FRONTEND_PATH") is not None:
|
||||
logger.warning(
|
||||
"VLLM_RUST_FRONTEND_PATH is set without enabling "
|
||||
"VLLM_USE_RUST_FRONTEND or VLLM_USE_RUST_BENCH. "
|
||||
"Set one of them to 1 to use the vllm-rs binary."
|
||||
)
|
||||
return None
|
||||
|
||||
if raw.lower() in ("auto", "1", "true"):
|
||||
pkg_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
candidate = os.path.join(pkg_dir, "vllm-rs")
|
||||
if os.path.isfile(candidate) and os.access(candidate, os.X_OK):
|
||||
return candidate
|
||||
|
||||
raise FileNotFoundError(
|
||||
"VLLM_RUST_FRONTEND_PATH=auto but the vllm-rs binary was "
|
||||
f"not found at {candidate}. "
|
||||
"Build with setuptools-rust or set the path explicitly."
|
||||
)
|
||||
return raw
|
||||
|
||||
|
||||
environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# ================== Installation Time Env Vars ==================
|
||||
# Target device of vLLM, supporting [cuda (by default),
|
||||
@@ -668,10 +629,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
os.environ.get("VLLM_USE_PRECOMPILED", "").strip().lower() in ("1", "true")
|
||||
or bool(os.environ.get("VLLM_PRECOMPILED_WHEEL_LOCATION"))
|
||||
),
|
||||
# If set, vllm will use the precompiled Rust frontend binary (vllm-rs).
|
||||
"VLLM_USE_PRECOMPILED_RUST": lambda: (
|
||||
os.environ.get("VLLM_USE_PRECOMPILED_RUST", "").strip().lower() in ("1", "true")
|
||||
),
|
||||
# If set, skip adding +precompiled suffix to version string
|
||||
"VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX": lambda: bool(
|
||||
int(os.environ.get("VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX", "0"))
|
||||
@@ -1452,17 +1409,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# If set to "0", disable LayerName opaque type for layer_name
|
||||
# parameters in custom ops. Defaults to enabled on torch >= 2.11.
|
||||
"VLLM_USE_LAYERNAME": lambda: bool(int(os.getenv("VLLM_USE_LAYERNAME", "1"))),
|
||||
# If set, use the Rust frontend binary instead of the Python API server
|
||||
# process(es).
|
||||
"VLLM_USE_RUST_FRONTEND": lambda: bool(
|
||||
int(os.getenv("VLLM_USE_RUST_FRONTEND", "0"))
|
||||
),
|
||||
# If set, use the packaged Rust client for `vllm bench serve`.
|
||||
"VLLM_USE_RUST_BENCH": lambda: bool(int(os.getenv("VLLM_USE_RUST_BENCH", "0"))),
|
||||
# Path to the vllm-rs binary. Defaults to "auto" which discovers the
|
||||
# binary installed with the vllm package. Used when VLLM_USE_RUST_FRONTEND=1
|
||||
# or VLLM_USE_RUST_BENCH=1.
|
||||
"VLLM_RUST_FRONTEND_PATH": lambda: _resolve_rust_cli_path(),
|
||||
# If set, vllm will run in development mode, which will enable
|
||||
# some additional endpoints for developing and debugging,
|
||||
# e.g. `/reset_prefix_cache`
|
||||
|
||||
@@ -68,7 +68,8 @@ class PluggableLayer(nn.Module):
|
||||
@classmethod
|
||||
def register(cls, name: str):
|
||||
def decorator(op_cls):
|
||||
assert name not in op_registry, f"Duplicate op name: {name}"
|
||||
if name in op_registry:
|
||||
return op_registry[name]
|
||||
op_cls.name = name
|
||||
op_registry[name] = op_cls
|
||||
return op_cls
|
||||
|
||||
@@ -11,7 +11,6 @@ from vllm.model_executor.layers.attention.cross_attention import CrossAttention
|
||||
from vllm.model_executor.layers.attention.encoder_only_attention import (
|
||||
EncoderOnlyAttention,
|
||||
)
|
||||
from vllm.model_executor.layers.attention.mla_attention import MLAAttention
|
||||
from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention
|
||||
from vllm.model_executor.layers.attention.prefill_prefix_lm_attention import (
|
||||
PrefillPrefixLMAttention,
|
||||
@@ -35,7 +34,6 @@ __all__ = [
|
||||
"ChunkedLocalAttention",
|
||||
"CrossAttention",
|
||||
"EncoderOnlyAttention",
|
||||
"MLAAttention",
|
||||
"MMEncoderAttention",
|
||||
"PrefillPrefixLMAttention",
|
||||
"RSWAAttention",
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,25 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.model_executor.layers.hpc.gated_mla import (
|
||||
hpc_gated_mla_gemm,
|
||||
hpc_gated_mla_supported,
|
||||
)
|
||||
from vllm.model_executor.layers.hpc.hpc_ihc import (
|
||||
HpcIHCHead,
|
||||
HpcIHCPost,
|
||||
HpcIHCPre,
|
||||
)
|
||||
from vllm.model_executor.layers.hpc.hpc_module import HpcModule
|
||||
from vllm.model_executor.layers.hpc.rope_norm import HpcRopeNorm, QkNormPolicy
|
||||
|
||||
__all__ = [
|
||||
"HpcIHCHead",
|
||||
"HpcIHCPost",
|
||||
"HpcIHCPre",
|
||||
"HpcModule",
|
||||
"HpcRopeNorm",
|
||||
"QkNormPolicy",
|
||||
"hpc_gated_mla_gemm",
|
||||
"hpc_gated_mla_supported",
|
||||
]
|
||||
@@ -1,114 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Enablement gate for the HPC fused gated-MLA GEMM.
|
||||
|
||||
``hpc.gated_mla_gemm`` fuses the MLA output gating -- the gate projection, the
|
||||
sigmoid and the elementwise product -- into a single kernel launch, replacing
|
||||
the eager ``attn_out * sigmoid(hidden_states @ weight.T)`` sequence.
|
||||
|
||||
Kernel constraints:
|
||||
- Requires VLLM_ENABLE_HPC_OPS=1
|
||||
- Requires the hpc package (.so) built for the current arch
|
||||
- Only sm100 / sm103 (compute capability 100, 103)
|
||||
- All three operands must be bfloat16 and contiguous
|
||||
- The gate width must be a multiple of 256
|
||||
- Only elementwise gating (headwise has no fused form)
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm import envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.hpc import has_hpc
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# hpc.gated_mla_gemm fuses `attn_out * sigmoid(x @ weight.T)` into one launch.
|
||||
# It is built for sm100/sm103 and needs all three operands bf16 + contiguous,
|
||||
# with the gate width a multiple of 256.
|
||||
_HPC_GATED_GEMM_N_ALIGN = 256
|
||||
_HPC_GATED_GEMM_CAPABILITIES: frozenset[int] = frozenset({100, 103})
|
||||
|
||||
|
||||
def hpc_gated_mla_supported(
|
||||
gating_type: str | None, attn_output_gate: torch.nn.Module | None
|
||||
) -> bool:
|
||||
"""Gate for running the MLA output gating through hpc.gated_mla_gemm."""
|
||||
if not envs.VLLM_ENABLE_HPC_OPS:
|
||||
return False
|
||||
|
||||
if attn_output_gate is None:
|
||||
return False
|
||||
|
||||
if not has_hpc():
|
||||
logger.warning_once(
|
||||
"HPC gated MLA disabled: 'hpc' package is not installed. "
|
||||
"Please install the HPC library (.so) to enable the fused kernel."
|
||||
)
|
||||
return False
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_cuda():
|
||||
logger.warning_once("HPC gated MLA disabled: only CUDA is supported.")
|
||||
return False
|
||||
|
||||
capability = current_platform.get_device_capability()
|
||||
if capability is None or capability.to_int() not in _HPC_GATED_GEMM_CAPABILITIES:
|
||||
logger.warning_once(
|
||||
"HPC gated MLA disabled: compute capability %s not in %s.",
|
||||
capability,
|
||||
_HPC_GATED_GEMM_CAPABILITIES,
|
||||
)
|
||||
return False
|
||||
|
||||
# Headwise gating broadcasts one scalar per head over v_head_dim; the
|
||||
# kernel only implements the elementwise product.
|
||||
if gating_type != "elementwise":
|
||||
logger.warning_once(
|
||||
"HPC gated MLA disabled: gating_type '%s' has no fused form.",
|
||||
gating_type,
|
||||
)
|
||||
return False
|
||||
|
||||
weight = getattr(attn_output_gate, "weight", None)
|
||||
if weight is None or weight.dtype != torch.bfloat16:
|
||||
logger.warning_once(
|
||||
"HPC gated MLA disabled: gate weight dtype is %s, the kernel needs "
|
||||
"bfloat16 (keep the layer out of quantization).",
|
||||
None if weight is None else weight.dtype,
|
||||
)
|
||||
return False
|
||||
|
||||
if weight.shape[0] % _HPC_GATED_GEMM_N_ALIGN != 0:
|
||||
logger.warning_once(
|
||||
"HPC gated MLA disabled: gate width %d is not a multiple of %d.",
|
||||
weight.shape[0],
|
||||
_HPC_GATED_GEMM_N_ALIGN,
|
||||
)
|
||||
return False
|
||||
|
||||
logger.info_once("HPC gated MLA enabled by set VLLM_ENABLE_HPC_OPS.")
|
||||
return True
|
||||
|
||||
|
||||
def hpc_gated_mla_gemm(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
atten_output: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Fused ``atten_output * sigmoid(x @ weight.T)``.
|
||||
|
||||
Args:
|
||||
x: ``[m, k]`` bfloat16, contiguous.
|
||||
weight: ``[n, k]`` bfloat16, contiguous and row-major (the kernel
|
||||
transposes it). ``n`` must be a multiple of 256.
|
||||
atten_output: ``[m, n]`` bfloat16, contiguous.
|
||||
|
||||
Returns:
|
||||
``[m, n]`` bfloat16.
|
||||
|
||||
"""
|
||||
from hpc.gemm import gated_mla_gemm
|
||||
|
||||
return gated_mla_gemm(x, weight, atten_output)
|
||||
@@ -1,237 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""HPC fused iHC (independent Hyper-Connections) kernels for HY V4.
|
||||
|
||||
Replaces the eager HYV4HCPreLayer / HYV4HCPostLayer / HYV4HCHeadLayer bodies
|
||||
with single-kernel HPC implementations:
|
||||
|
||||
pre : rms square sum + w projection + sigmoid gates + hc-dim weighted sum
|
||||
post : y[n, i, :] = H_post[n, i] * x[n, :] + residual[n, i, :]
|
||||
head : same as pre but hc_mult projection rows and no H_post output
|
||||
|
||||
The eager path issues 20 / 5 / 15 kernels for pre / post / head; each HPC op is
|
||||
a single kernel.
|
||||
|
||||
Constraints:
|
||||
- Requires VLLM_ENABLE_HPC_OPS=1
|
||||
- Requires the hpc package (.so) built for the current arch
|
||||
- Only sm100 / sm103 (compute capability 100, 103)
|
||||
- Only hc_mult == 4 and hidden_size in {4096, 6144}
|
||||
- x / residual must be bfloat16; hc_fn weights stay float32 (the checkpoint
|
||||
keeps them out of fp8 quantization via modules_to_not_convert)
|
||||
|
||||
The fused modules own no parameters: their forwards read the weights of the
|
||||
eager layer they replace directly via the captured owner reference.
|
||||
|
||||
NOTE: The reference implementation also offers a cross-layer post+pre fusion
|
||||
(``HpcIHCPostPre``) that folds one segment's post into the next segment's pre.
|
||||
It requires reworking the decoder-layer forward scheduling and is not ported
|
||||
here. TODO: add it once the decoder dataflow is restructured for it.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm import envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.hpc.hpc_module import HpcModule
|
||||
from vllm.utils.hpc import has_hpc
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Shapes the HPC kernels are instantiated for. Other shapes launch nothing,
|
||||
# so gate on this rather than relying on a silent no-op.
|
||||
_SUPPORTED_HIDDEN_SIZES: frozenset[int] = frozenset({4096, 6144})
|
||||
_SUPPORTED_HC_MULTS: frozenset[int] = frozenset({4})
|
||||
_SUPPORTED_CAPABILITIES: frozenset[int] = frozenset({100, 103})
|
||||
|
||||
|
||||
def _ihc_supported(hc_mult: int, hidden_size: int) -> bool:
|
||||
"""Shared gate for all three iHC ops."""
|
||||
if not envs.VLLM_ENABLE_HPC_OPS:
|
||||
return False
|
||||
|
||||
if not has_hpc():
|
||||
logger.warning_once(
|
||||
"HPC iHC disabled: 'hpc' package is not installed. "
|
||||
"Please install the HPC library (.so) to enable fused kernels."
|
||||
)
|
||||
return False
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_cuda():
|
||||
logger.warning_once("HPC iHC disabled: only CUDA is supported.")
|
||||
return False
|
||||
|
||||
capability = current_platform.get_device_capability()
|
||||
if capability is None or capability.to_int() not in _SUPPORTED_CAPABILITIES:
|
||||
logger.warning_once(
|
||||
"HPC iHC disabled: compute capability %s not in %s.",
|
||||
capability,
|
||||
_SUPPORTED_CAPABILITIES,
|
||||
)
|
||||
return False
|
||||
|
||||
if hc_mult not in _SUPPORTED_HC_MULTS:
|
||||
logger.warning_once(
|
||||
"HPC iHC disabled: hc_mult=%d not in %s.", hc_mult, _SUPPORTED_HC_MULTS
|
||||
)
|
||||
return False
|
||||
|
||||
if hidden_size not in _SUPPORTED_HIDDEN_SIZES:
|
||||
logger.warning_once(
|
||||
"HPC iHC disabled: hidden_size=%d not in %s.",
|
||||
hidden_size,
|
||||
_SUPPORTED_HIDDEN_SIZES,
|
||||
)
|
||||
return False
|
||||
|
||||
if envs.VLLM_BATCH_INVARIANT:
|
||||
# Batch-invariant mode relies on torch's deterministic reductions; the
|
||||
# HPC kernel reduces in a different order, so keep the two regimes
|
||||
# apart.
|
||||
logger.warning_once(
|
||||
"HPC iHC disabled: not supported under VLLM_BATCH_INVARIANT."
|
||||
)
|
||||
return False
|
||||
|
||||
logger.info_once("HPC iHC enabled by set VLLM_ENABLE_HPC_OPS.")
|
||||
return True
|
||||
|
||||
|
||||
class HpcIHCPre(HpcModule):
|
||||
"""Fused iHC pre block.
|
||||
|
||||
Computes, in one kernel:
|
||||
x_flat = x.flatten(1)
|
||||
r = rsqrt(x_flat.square().mean(-1) + rms_norm_eps)
|
||||
mixes = (x_flat @ w.T) * r
|
||||
H_pre = sigmoid(mixes[:, :hc] * hc_scale[0] + hc_base[:hc]) + hc_eps
|
||||
H_post = magnitude * sigmoid(
|
||||
mixes[:, hc:] * hc_scale[1] + hc_base[hc:]) + hc_eps
|
||||
y = sum_i H_pre[:, i] * x[:, i, :]
|
||||
|
||||
Args:
|
||||
hc_mult: HC expand ratio.
|
||||
hidden_size: Model hidden dimension.
|
||||
magnitude: H_post multiplier (config.hc_magnitude).
|
||||
hc_eps: Epsilon added to both gates (config.hc_eps).
|
||||
norm_eps: Epsilon inside the rsqrt (config.rms_norm_eps).
|
||||
fallback_op: The eager HYV4HCPreLayer to source weights from.
|
||||
norm_owner: Optional RMSNorm that immediately follows this pre block.
|
||||
When given, its weight/eps are folded into the kernel so a single
|
||||
launch covers pre + RMSNorm (the caller then skips the separate
|
||||
layernorm). None keeps the original pre-only behaviour.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hc_mult: int,
|
||||
hidden_size: int,
|
||||
magnitude: float,
|
||||
hc_eps: float,
|
||||
norm_eps: float,
|
||||
fallback_op: torch.nn.Module,
|
||||
norm_owner: torch.nn.Module | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hc_mult = hc_mult
|
||||
self.hidden_size = hidden_size
|
||||
self.magnitude = magnitude
|
||||
self.hc_eps = hc_eps
|
||||
self.norm_eps = norm_eps
|
||||
# Stash the owner outside nn.Module's attribute machinery: callers pass
|
||||
# their own `self` here, and registering that as a submodule would make
|
||||
# the module tree cyclic (state_dict() then hits RecursionError).
|
||||
object.__setattr__(self, "_fallback_op", fallback_op)
|
||||
# Same anti-cycle idiom for the fused RMSNorm owner (a sibling module).
|
||||
object.__setattr__(self, "_norm_owner", norm_owner)
|
||||
|
||||
@classmethod
|
||||
def support(cls, hc_mult: int, hidden_size: int) -> bool:
|
||||
return _ihc_supported(hc_mult, hidden_size)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
import hpc
|
||||
|
||||
owner = self._fallback_op
|
||||
norm_owner = self._norm_owner
|
||||
return hpc.fuse_ihc_pre(
|
||||
x,
|
||||
owner.hc_fn.weight,
|
||||
owner.hc_scale,
|
||||
owner.hc_base,
|
||||
self.norm_eps,
|
||||
self.hc_eps,
|
||||
self.magnitude,
|
||||
norm_owner.weight if norm_owner is not None else None,
|
||||
norm_owner.variance_epsilon if norm_owner is not None else 0.0,
|
||||
)
|
||||
|
||||
|
||||
class HpcIHCPost(HpcModule):
|
||||
"""Fused iHC post block: H_post gating plus multi-channel residual add.
|
||||
|
||||
Unlike mHC there is no comb matrix, so each output channel only needs its
|
||||
own residual channel and the whole thing is one fused multiply-add per
|
||||
element.
|
||||
"""
|
||||
|
||||
def __init__(self, hc_mult: int, hidden_size: int) -> None:
|
||||
super().__init__()
|
||||
self.hc_mult = hc_mult
|
||||
self.hidden_size = hidden_size
|
||||
|
||||
@classmethod
|
||||
def support(cls, hc_mult: int, hidden_size: int) -> bool:
|
||||
return _ihc_supported(hc_mult, hidden_size)
|
||||
|
||||
def forward(
|
||||
self, x: torch.Tensor, residual: torch.Tensor, H_post: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
import hpc
|
||||
|
||||
return hpc.fuse_ihc_post(x, residual, H_post)
|
||||
|
||||
|
||||
class HpcIHCHead(HpcModule):
|
||||
"""Fused iHC head block: merge the hc channels back into one hidden state.
|
||||
|
||||
Same structure as HpcIHCPre but the projection emits only hc_mult gate
|
||||
logits and there is no H_post output. Called once per forward, after the
|
||||
last decoder layer.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hc_mult: int,
|
||||
hidden_size: int,
|
||||
hc_eps: float,
|
||||
norm_eps: float,
|
||||
fallback_op: torch.nn.Module,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hc_mult = hc_mult
|
||||
self.hidden_size = hidden_size
|
||||
self.hc_eps = hc_eps
|
||||
self.norm_eps = norm_eps
|
||||
# See HpcIHCPre: keep the owner out of the module tree.
|
||||
object.__setattr__(self, "_fallback_op", fallback_op)
|
||||
|
||||
@classmethod
|
||||
def support(cls, hc_mult: int, hidden_size: int) -> bool:
|
||||
return _ihc_supported(hc_mult, hidden_size)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
import hpc
|
||||
|
||||
owner = self._fallback_op
|
||||
return hpc.fuse_ihc_head(
|
||||
x,
|
||||
owner.hc_head_fn.weight,
|
||||
owner.hc_head_scale,
|
||||
owner.hc_head_base,
|
||||
self.norm_eps,
|
||||
self.hc_eps,
|
||||
)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user