Add pydocstyle to the ruff rules (#52136)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Signed-off-by: Harry Mellor <hej.mellor@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 16:44:29 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 804e5377aa
commit 8c1557a79c
2002 changed files with 15024 additions and 14269 deletions
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Convert a plain-text file (local path or URL) into a JSONL dataset
"""Convert a plain-text file (local path or URL) into a JSONL dataset
compatible with ``CustomDataset`` (``--dataset-name custom``), by
randomly slicing the tokenized text into prompts.
@@ -66,8 +65,8 @@ def create_txt_slices_jsonl(
trust_remote_code: bool = False,
) -> None:
"""Read *input_path*, slice it into prompts, and write JSONL to
*output_path*."""
*output_path*.
"""
tokenizer = AutoTokenizer.from_pretrained(
tokenizer_name, trust_remote_code=trust_remote_code
)
+65 -98
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
This module defines a framework for sampling benchmark requests from various
"""This module defines a framework for sampling benchmark requests from various
datasets. Each dataset subclass of BenchmarkDataset must implement sample
generation. Supported dataset types include:
- ShareGPT
@@ -75,9 +74,7 @@ DEFAULT_NUM_PROMPTS = 1000
@dataclass
class SampleRequest:
"""
Represents a single inference request for benchmarking.
"""
"""Represents a single inference request for benchmarking."""
prompt: str | list[str] | list[int] | list[list[int]] | list[dict]
prompt_len: int
@@ -112,8 +109,7 @@ class BenchmarkDataset(ABC):
disable_shuffle: bool = False,
**kwargs,
) -> None:
"""
Initialize the BenchmarkDataset with an optional dataset path and random
"""Initialize the BenchmarkDataset with an optional dataset path and random
seed.
Args:
@@ -121,6 +117,9 @@ class BenchmarkDataset(ABC):
indicates that a default or random dataset might be used.
random_seed (int): Seed value for reproducible shuffling or
sampling. Defaults to DEFAULT_SEED.
disable_shuffle (bool): If True, keep the dataset in its original
order instead of shuffling it.
"""
self.dataset_path = dataset_path
# Set the random seed, ensuring that a None value is replaced with the
@@ -134,8 +133,7 @@ class BenchmarkDataset(ABC):
prompt: str,
mm_content: MultiModalDataDict | dict | list[dict] | None = None,
) -> list[dict]:
"""
Transform a prompt and optional multimodal content into a chat format.
"""Transform a prompt and optional multimodal content into a chat format.
This method is used for chat models that expect a specific conversation
format.
"""
@@ -152,14 +150,14 @@ class BenchmarkDataset(ABC):
return [{"role": "user", "content": content}]
def load_data(self) -> None:
"""
Load data from the dataset path into self.data.
"""Load data from the dataset path into self.data.
This method must be overridden by subclasses since the method to load
data will vary depending on the dataset format and source.
Raises:
NotImplementedError: If a subclass does not implement this method.
"""
# TODO (jenniferzhao): add support for downloading data
raise NotImplementedError("load_data must be implemented in subclasses.")
@@ -169,8 +167,7 @@ class BenchmarkDataset(ABC):
max_loras: int | None = None,
lora_path: str | None = None,
) -> LoRARequest | None:
"""
Optionally select a random LoRA request.
"""Optionally select a random LoRA request.
This method is used when LoRA parameters are provided. It randomly
selects a LoRA based on max_loras.
@@ -184,6 +181,7 @@ class BenchmarkDataset(ABC):
Returns:
A new [`LoRARequest`][vllm.lora.request.LoRARequest]
(or `None` if not applicable).
"""
if max_loras is None or lora_path is None:
return None
@@ -203,8 +201,7 @@ class BenchmarkDataset(ABC):
max_loras: int | None = None,
lora_path: str | None = None,
) -> LoRARequest | None:
"""
Optionally select a LoRA request using deterministic round-robin.
"""Optionally select a LoRA request using deterministic round-robin.
This method cycles through LoRA IDs in order based on the request
index, providing reproducible LoRA assignment.
@@ -219,6 +216,7 @@ class BenchmarkDataset(ABC):
Returns:
A new [`LoRARequest`][vllm.lora.request.LoRARequest]
(or `None` if not applicable).
"""
if max_loras is None or lora_path is None:
return None
@@ -239,8 +237,7 @@ class BenchmarkDataset(ABC):
lora_path: str | None = None,
lora_assignment: str = "random",
) -> LoRARequest | None:
"""
Select a LoRA request using the specified assignment strategy.
"""Select a LoRA request using the specified assignment strategy.
Args:
index (int): The request index (used for round-robin).
@@ -252,6 +249,7 @@ class BenchmarkDataset(ABC):
Returns:
A new [`LoRARequest`][vllm.lora.request.LoRARequest]
(or `None` if not applicable).
"""
if lora_assignment == "round-robin":
return BenchmarkDataset.get_round_robin_lora_request(
@@ -270,8 +268,7 @@ class BenchmarkDataset(ABC):
no_oversample: bool = False,
**kwargs,
) -> list[SampleRequest]:
"""
Abstract method to generate sample requests from the dataset.
"""Abstract method to generate sample requests from the dataset.
Subclasses must override this method to implement dataset-specific logic
for generating a list of SampleRequest objects.
@@ -281,10 +278,13 @@ class BenchmarkDataset(ABC):
for processing the dataset's text.
num_requests (int): The number of sample requests to generate.
request_id_prefix (str): The prefix of request_id.
no_oversample (bool): If True, do not oversample the dataset to
reach num_requests.
Returns:
list[SampleRequest]: A list of sample requests generated from the
dataset.
"""
raise NotImplementedError("sample must be implemented in subclasses.")
@@ -295,8 +295,7 @@ class BenchmarkDataset(ABC):
request_id_prefix: str = "",
no_oversample: bool = False,
) -> None:
"""
Oversamples the list of requests if its size is less than the desired
"""Oversamples the list of requests if its size is less than the desired
number.
Args:
@@ -305,6 +304,8 @@ class BenchmarkDataset(ABC):
num_requests (int): The target number of requests.
request_id_prefix (str): The prefix applied to generated request
identifiers.
no_oversample (bool): If True, return the requests unchanged
instead of oversampling up to num_requests.
"""
if no_oversample:
@@ -346,8 +347,7 @@ def is_valid_sequence(
max_total_len: int = 2048,
skip_min_output_len_check: bool = False,
) -> bool:
"""
Validate a sequence based on prompt and output lengths.
"""Validate a sequence based on prompt and output lengths.
Default pruning criteria are copied from the original `sample_hf_requests`
and `sample_sharegpt_requests` functions in benchmark_serving.py, as well as
@@ -379,8 +379,7 @@ def process_image(
*,
ensure_client_side_data: bool = False,
) -> Mapping[str, Any]:
"""
Process a single image input and return a multimedia content dictionary.
"""Process a single image input and return a multimedia content dictionary.
Supports the following input types:
@@ -402,6 +401,7 @@ def process_image(
Raises:
ValueError: If the input is not a supported type.
"""
if isinstance(image, dict) and "bytes" in image:
image = Image.open(BytesIO(image["bytes"]))
@@ -437,8 +437,7 @@ def process_image(
def process_video(video: Any) -> Mapping[str, Any]:
"""
Process a single video input and return a multimedia content dictionary.
"""Process a single video input and return a multimedia content dictionary.
Supports the following input types:
@@ -451,6 +450,7 @@ def process_video(video: Any) -> Mapping[str, Any]:
Raises:
ValueError: If the input is not a supported type.
"""
if isinstance(video, dict) and "bytes" in video:
video_bytes = video["bytes"]
@@ -474,8 +474,7 @@ def process_video(video: Any) -> Mapping[str, Any]:
def process_audio(audio: Any) -> tuple:
"""
Process a single audio input and return a (array, sample_rate) tuple.
"""Process a single audio input and return a (array, sample_rate) tuple.
Supports:
1. String: treated as a file path, loaded with soundfile.
@@ -502,8 +501,7 @@ def gen_prompt_decode_to_target_len(
add_special_tokens: bool = False,
rng: np.random.Generator | None = None,
) -> tuple[str, list[int], int]:
"""
Ensure decoded-then-encoded prompt length matches the target token length.
"""Ensure decoded-then-encoded prompt length matches the target token length.
This function decodes an initial token sequence to text and re-encodes it
, iteratively adjusting the token sequence length to match a target.
@@ -557,8 +555,7 @@ def gen_prompt_decode_to_target_len(
class RandomDataset(BenchmarkDataset):
"""
Synthetic text-only dataset for serving/throughput benchmarks.
"""Synthetic text-only dataset for serving/throughput benchmarks.
Strategy:
- Sample input/output token lengths per request from integer-uniform ranges
@@ -699,9 +696,7 @@ class RandomDataset(BenchmarkDataset):
allowed_tokens: np.ndarray,
prefix_len: int,
) -> list[int]:
"""
Get the prefix for the dataset.
"""
"""Get the prefix for the dataset."""
if prefix_len <= 0:
return []
@@ -738,8 +733,7 @@ class RandomDataset(BenchmarkDataset):
index: int,
allowed_tokens: np.ndarray,
) -> tuple[str, int, int]:
"""
Returns (prompt, total_input_len).
"""Returns (prompt, total_input_len).
NOTE: After decoding the prompt we have to encode and decode it again.
This is done because in some cases N consecutive tokens
@@ -778,8 +772,7 @@ class RandomDataset(BenchmarkDataset):
class RandomDatasetForReranking(RandomDataset):
"""
Random dataset specialized for the needs of scoring:
"""Random dataset specialized for the needs of scoring:
- Batches of inputs
- Inputs composed of pairs
"""
@@ -906,8 +899,7 @@ class RandomDatasetForReranking(RandomDataset):
class RandomMultiModalDataset(RandomDataset):
"""
Synthetic multimodal dataset (text + images) that extends RandomDataset.
"""Synthetic multimodal dataset (text + images) that extends RandomDataset.
Status:
- Images: supported via synthetic RGB data.
@@ -1022,8 +1014,7 @@ class RandomMultiModalDataset(RandomDataset):
def normalize_bucket_config(
self, bucket_config: dict[tuple[int, int, int], float]
) -> dict[tuple[int, int, int], float]:
"""
Remove zero probability entries
"""Remove zero probability entries
and normalize the bucket config to sum to 1.
"""
# Raise error if value is negative
@@ -1044,13 +1035,11 @@ class RandomMultiModalDataset(RandomDataset):
self,
mm_item_config: tuple[int, int, int],
) -> Mapping[str, Any]:
"""
Create synthetic images and videos and
"""Create synthetic images and videos and
apply process_image/process_video respectively.
This follows the OpenAI API chat completions
https://github.com/openai/openai-python
"""
if self.map_config_to_modality(mm_item_config) == "image":
return process_image(
self.generate_synthetic_image(mm_item_config[1], mm_item_config[0])
@@ -1071,9 +1060,7 @@ class RandomMultiModalDataset(RandomDataset):
limit_mm_per_prompt: dict[str, int],
bucket_config: dict[tuple[int, int, int], float],
) -> tuple[int, int, dict[str, int], dict[tuple[int, int, int], float]]:
"""
Get the sampling parameters for the multimodal items.
"""
"""Get the sampling parameters for the multimodal items."""
# Enforce num_mm_items_range_ratio <= 1
if not (0.0 <= num_mm_items_range_ratio <= 1.0):
raise ValueError("num_mm_items_range_ratio must be in [0, 1].")
@@ -1146,8 +1133,7 @@ class RandomMultiModalDataset(RandomDataset):
bucket_config: dict[tuple[int, int, int], float],
limit_mm_per_prompt: dict[str, int],
) -> Iterator[tuple[int, int, int]]:
"""
Iterator over the multimodal items for each request
"""Iterator over the multimodal items for each request
whose size is between min_num_mm_items and max_num_mm_items.
Loop over the bucket config and sample a multimodal item.
@@ -1160,6 +1146,7 @@ class RandomMultiModalDataset(RandomDataset):
`bucket_config` (tuple->float). The original dict passed to
`sample` is not mutated. If this ever changes, a test
is implemented and will fail.
"""
# Get the number of multimodal items to sample
request_num_mm_items = int(
@@ -1346,8 +1333,7 @@ class RandomMultiModalDataset(RandomDataset):
class ShareGPTDataset(BenchmarkDataset):
"""
Implements the ShareGPT dataset. Loads data from a JSON file and generates
"""Implements the ShareGPT dataset. Loads data from a JSON file and generates
sample requests based on conversation turns.
"""
@@ -1437,8 +1423,7 @@ class ShareGPTDataset(BenchmarkDataset):
class TimedTrace(BenchmarkDataset):
"""
Implements a base class to replay various timed traces.
"""Implements a base class to replay various timed traces.
Loads data from a JSON file and generates sample requests
based on the timing information in the traces.
"""
@@ -1927,6 +1912,7 @@ def add_random_dataset_base_args(
Args:
parser_or_group: Either a parser or an argument group to add arguments to.
"""
parser_or_group.add_argument(
"--random-input-len",
@@ -1988,6 +1974,7 @@ def add_random_multimodal_dataset_args(
Args:
parser_or_group: Either a parser or an argument group to add arguments to.
"""
parser_or_group.add_argument(
"--random-mm-base-items-per-request",
@@ -2503,8 +2490,7 @@ def get_samples(
class CustomDataset(BenchmarkDataset):
"""
Implements the Custom dataset. Loads data from a JSONL file and generates
"""Implements the Custom dataset. Loads data from a JSONL file and generates
sample requests based on conversation turns. E.g.,
```
{"prompt": "What is the capital of India?", "output_tokens": 10}
@@ -2632,8 +2618,7 @@ class CustomDataset(BenchmarkDataset):
class CustomImageDataset(CustomDataset):
"""
Implements the Custom image dataset. Loads data from a JSONL file and generates
"""Implements the Custom image dataset. Loads data from a JSONL file and generates
sample requests based on conversation turns. E.g.,
```
{
@@ -2917,8 +2902,7 @@ class CustomImageDataset(CustomDataset):
class CustomAudioDataset(CustomDataset):
"""
Custom dataset for audio benchmarking. Loads data from a JSONL file. E.g.,
"""Custom dataset for audio benchmarking. Loads data from a JSONL file. E.g.,
{"prompt": "Transcribe the audio.", "audio": "/path/to/audio.wav"}
Supports both:
@@ -3027,8 +3011,7 @@ class CustomAudioDataset(CustomDataset):
class SpecBench(CustomDataset):
"""
Implements the SpecBench dataset: https://github.com/hemingkx/Spec-Bench
"""Implements the SpecBench dataset: https://github.com/hemingkx/Spec-Bench
Download the dataset using:
wget https://raw.githubusercontent.com/hemingkx/Spec-Bench/refs/heads/main/data/spec_bench/question.jsonl
""" # noqa: E501
@@ -3100,8 +3083,7 @@ class SpecBench(CustomDataset):
"SonnetDataset is deprecated and will be removed in a future version.",
)
class SonnetDataset(BenchmarkDataset):
"""
Simplified implementation of the Sonnet dataset. Loads poem lines from a
"""Simplified implementation of the Sonnet dataset. Loads poem lines from a
text file and generates sample requests. Default values here copied from
`benchmark_serving.py` for the sonnet dataset.
"""
@@ -3198,8 +3180,7 @@ class SonnetDataset(BenchmarkDataset):
class BurstGPTDataset(BenchmarkDataset):
"""
Implements the BurstGPT dataset. Loads data from a CSV file and generates
"""Implements the BurstGPT dataset. Loads data from a CSV file and generates
sample requests based on synthetic prompt generation. Only rows with Model
"GPT-4" and positive response tokens are used.
"""
@@ -3447,9 +3428,7 @@ class MultiModalConversationDataset(HuggingFaceDataset):
class VisionArenaDataset(HuggingFaceDataset):
"""
Vision Arena Dataset.
"""
"""Vision Arena Dataset."""
DEFAULT_OUTPUT_LEN = 128
SUPPORTED_DATASET_PATHS = {
@@ -3506,8 +3485,7 @@ class VisionArenaDataset(HuggingFaceDataset):
class MMVUDataset(HuggingFaceDataset):
"""
MMVU Dataset.
"""MMVU Dataset.
https://huggingface.co/datasets/yale-nlp/MMVU
"""
@@ -3586,8 +3564,7 @@ class MMVUDataset(HuggingFaceDataset):
class InstructCoderDataset(HuggingFaceDataset):
"""
InstructCoder Dataset.
"""InstructCoder Dataset.
https://huggingface.co/datasets/likaixin/InstructCoder
InstructCoder is the dataset designed for general code editing. It consists
@@ -3655,8 +3632,7 @@ class InstructCoderDataset(HuggingFaceDataset):
class MTBenchDataset(HuggingFaceDataset):
"""
MT-Bench Dataset.
"""MT-Bench Dataset.
https://huggingface.co/datasets/philschmid/mt-bench
We create a single turn dataset for MT-Bench.
@@ -3718,8 +3694,7 @@ class MTBenchDataset(HuggingFaceDataset):
class HumanEvalDataset(HuggingFaceDataset):
"""
HumanEvalDataset Dataset.
"""HumanEvalDataset Dataset.
https://huggingface.co/datasets/openai/openai_humaneval
We create a single turn dataset for HumanEval.
@@ -3779,8 +3754,7 @@ class HumanEvalDataset(HuggingFaceDataset):
class GSM8KDataset(HuggingFaceDataset):
"""
GSM8K Dataset.
"""GSM8K Dataset.
https://huggingface.co/datasets/openai/gsm8k
We create a single turn dataset for GSM8K.
@@ -3840,8 +3814,7 @@ class GSM8KDataset(HuggingFaceDataset):
class BlazeditDataset(HuggingFaceDataset):
"""
Blazedit Dataset.
"""Blazedit Dataset.
https://github.com/ise-uiuc/blazedit
5k char version: vdaita/edit_5k_char
@@ -3932,9 +3905,7 @@ Please generate the new code file in the "New file" section below.""" # noqa: E
class AIMODataset(HuggingFaceDataset):
"""
Dataset class for processing a AIMO dataset with reasoning questions.
"""
"""Dataset class for processing a AIMO dataset with reasoning questions."""
SUPPORTED_DATASET_PATHS = {
"AI-MO/aimo-validation-aime",
@@ -4026,6 +3997,7 @@ def _format_zeta_prompt(
Returns:
A dictionary with the formatted prompts and expected outputs.
"""
events = sample["events"]
input = sample["input"]
@@ -4042,9 +4014,7 @@ def _format_zeta_prompt(
class NextEditPredictionDataset(HuggingFaceDataset):
"""
Dataset class for processing a Next Edit Prediction dataset.
"""
"""Dataset class for processing a Next Edit Prediction dataset."""
SUPPORTED_DATASET_PATHS = {
"zed-industries/zeta",
@@ -4092,8 +4062,7 @@ class NextEditPredictionDataset(HuggingFaceDataset):
class ASRDataset(HuggingFaceDataset):
"""
Dataset class for processing a ASR dataset for transcription.
"""Dataset class for processing a ASR dataset for transcription.
Tested on the following set:
+---------------------------+----------------------------------------+--------------------------+-----------------------------+
@@ -4290,8 +4259,7 @@ class ASRDataset(HuggingFaceDataset):
class MLPerfDataset(HuggingFaceDataset):
"""
MLPerf Inference Dataset.
"""MLPerf Inference Dataset.
Dataset on HF:
https://huggingface.co/datasets/mgoin/mlperf-inference-llama2-data
@@ -4415,7 +4383,8 @@ class PrefixRepetitionRandomDataset(BenchmarkDataset):
def _generate_exact_length_tokens(target_length: int) -> tuple[list[int], int]:
"""Generate tokens that decode and re-encode to exactly
target_length."""
target_length.
"""
# Generate random tokens
tokens = np.random.randint(0, vocab_size, size=target_length).tolist()
@@ -4470,8 +4439,7 @@ class PrefixRepetitionRandomDataset(BenchmarkDataset):
class MMStarDataset(HuggingFaceDataset):
"""
Lin-Chen/MMStar: https://huggingface.co/datasets/Lin-Chen/MMStar
"""Lin-Chen/MMStar: https://huggingface.co/datasets/Lin-Chen/MMStar
refer to: https://github.com/sgl-project/SpecForge/pull/106
"""
@@ -4760,8 +4728,7 @@ class BFCLDataset(HuggingFaceDataset):
class SpeedBench(CustomDataset):
"""
SPEED-Bench dataset: https://huggingface.co/datasets/nvidia/SPEED-Bench
"""SPEED-Bench dataset: https://huggingface.co/datasets/nvidia/SPEED-Bench.
Download the dataset using:
+3 -5
View File
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Shared utilities for benchmark dataset sampling.
"""
"""Shared utilities for benchmark dataset sampling."""
import logging
import math
@@ -46,8 +44,7 @@ def get_sampling_params(
output_len: int,
tokenizer: TokenizerLike,
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
"""
Sample per-request input/output token lengths and vocab offsets.
"""Sample per-request input/output token lengths and vocab offsets.
Lengths are drawn uniformly from integer ranges around the configured
means, controlled by *range_ratio*. It may be a single ``float``
@@ -60,6 +57,7 @@ def get_sampling_params(
Returns:
(input_lens, output_lens, offsets) – three 1-D ``np.ndarray`` of
shape ``(num_requests,)``.
"""
input_range_ratio, output_range_ratio = _resolve_range_ratios(range_ratio)
+6 -2
View File
@@ -22,7 +22,8 @@ AIOHTTP_TIMEOUT = aiohttp.ClientTimeout(total=6 * 60 * 60)
class StreamedResponseHandler:
"""Handles streaming HTTP responses by accumulating chunks until complete
messages are available."""
messages are available.
"""
def __init__(self):
self.buffer = ""
@@ -30,7 +31,8 @@ class StreamedResponseHandler:
def add_chunk(self, chunk_bytes: bytes) -> list[str]:
"""Add a chunk of bytes to the buffer and return any complete
messages."""
messages.
"""
chunk_str = self._decoder.decode(chunk_bytes)
self.buffer += chunk_str
@@ -167,10 +169,12 @@ async def async_request_openai_completions(
Args:
request_func_input: The input for the request function.
session: The aiohttp session used to issue the request.
pbar: The progress bar to display the progress.
Returns:
The output of the request function.
"""
api_url = request_func_input.api_url
_validate_api_url(api_url, "OpenAI Completions API", "completions")
+3 -2
View File
@@ -22,12 +22,12 @@ async def wait_for_endpoint(
timeout_seconds: int = 600,
retry_interval: int = 5,
) -> RequestFuncOutput:
"""
Wait for an endpoint to become available before starting benchmarks.
"""Wait for an endpoint to become available before starting benchmarks.
Args:
request_func: The async request function to call
test_input: The RequestFuncInput to test with
session: The aiohttp session used to issue the probe requests
timeout_seconds: Maximum time to wait in seconds (default: 10 minutes)
retry_interval: Time between retries in seconds (default: 5 seconds)
@@ -36,6 +36,7 @@ async def wait_for_endpoint(
Raises:
ValueError: If the endpoint doesn't become available within the timeout
"""
deadline = time.perf_counter() + timeout_seconds
output = RequestFuncOutput(success=False)
+2 -5
View File
@@ -36,9 +36,7 @@ def extract_field(
def use_compile(args: argparse.Namespace, extra_info: dict[str, Any]) -> bool:
"""
Check if the benchmark is run with torch.compile
"""
"""Check if the benchmark is run with torch.compile."""
return not (
extract_field(args, extra_info, "compilation_config.mode") == "0"
or "eager" in getattr(args, "output_json", "")
@@ -49,8 +47,7 @@ def use_compile(args: argparse.Namespace, extra_info: dict[str, Any]) -> bool:
def convert_to_pytorch_benchmark_format(
args: argparse.Namespace, metrics: dict[str, list], extra_info: dict[str, Any]
) -> list:
"""
Save the benchmark results in the format used by PyTorch OSS benchmark with
"""Save the benchmark results in the format used by PyTorch OSS benchmark with
on metric per record
https://github.com/pytorch/pytorch/wiki/How-to-integrate-with-PyTorch-OSS-benchmark-database
"""
+6 -14
View File
@@ -44,8 +44,7 @@ else:
def get_timing_stats_from_engine(llm_engine: LLMEngine) -> dict[str, dict[str, float]]:
"""
Get all multimodal timing stats from the LLM engine.
"""Get all multimodal timing stats from the LLM engine.
Collects both preprocessing stats (HF processor, hashing, cache lookup,
prompt update) and encoder forward pass timing, merged by request_id.
@@ -70,6 +69,7 @@ def get_timing_stats_from_engine(llm_engine: LLMEngine) -> dict[str, dict[str, f
'num_encoder_calls': 1
}
}
"""
observability_config = llm_engine.vllm_config.observability_config
if not observability_config or not observability_config.enable_mm_processor_stats:
@@ -126,8 +126,7 @@ def get_timing_stats_from_engine(llm_engine: LLMEngine) -> dict[str, dict[str, f
def collect_mm_processor_stats(llm_engine: LLMEngine) -> dict[str, list[float]]:
"""
Collect multimodal processor timing stats.
"""Collect multimodal processor timing stats.
Returns a dictionary mapping stage names to lists of timing values (in seconds).
"""
all_stats = get_timing_stats_from_engine(llm_engine)
@@ -147,9 +146,7 @@ def calculate_mm_processor_metrics(
*,
unit: Literal["us", "ms", "s"] = "ms",
) -> dict[str, dict[str, float]]:
"""
Calculate aggregate metrics from stats by stage.
"""
"""Calculate aggregate metrics from stats by stage."""
unit2mult = {"us": 1000000, "ms": 1000, "s": 1}
unit_mult = unit2mult[unit]
@@ -181,9 +178,7 @@ def calculate_mm_processor_metrics(
def validate_args(args):
"""
Validate command-line arguments for mm_processor benchmark.
"""
"""Validate command-line arguments for mm_processor benchmark."""
if not getattr(args, "tokenizer", None):
args.tokenizer = args.model
if not hasattr(args, "dataset_path"):
@@ -214,9 +209,7 @@ def validate_args(args):
def benchmark_multimodal_processor(
args: argparse.Namespace,
) -> dict[str, Any]:
"""
Run the multimodal processor benchmark.
"""
"""Run the multimodal processor benchmark."""
from vllm import LLM, SamplingParams
validate_args(args)
@@ -455,7 +448,6 @@ def add_cli_args(parser: FlexibleArgumentParser) -> None:
def main(args: argparse.Namespace) -> None:
"""Main entry point for the multimodal processor benchmark."""
print("Starting multimodal processor benchmark...")
result = benchmark_multimodal_processor(args)
+6 -7
View File
@@ -29,8 +29,7 @@ def generate_timeline_plot(
itl_thresholds: list[float] | None = None,
labels: list[str] | None = None,
) -> None:
"""
Generate an HTML timeline plot from benchmark results.
"""Generate an HTML timeline plot from benchmark results.
Args:
results: List of per-request result dictionaries containing:
@@ -44,8 +43,8 @@ def generate_timeline_plot(
colors: List of colors for ITL categories (default: green, orange, red, black)
itl_thresholds: ITL thresholds in seconds (default: [1.0, 4.0, 6.0])
labels: Labels for ITL categories (default based on thresholds)
"""
"""
# Set defaults
if colors is None:
colors = ["#109618", "#FF7F0E", "#D62728"]
@@ -118,8 +117,7 @@ def construct_timeline_data(
itl_thresholds: list[float],
labels: list[str],
) -> list[dict[str, Any]]:
"""
Construct timeline data from request results.
"""Construct timeline data from request results.
Args:
requests_data: List of per-request result dictionaries
@@ -128,6 +126,7 @@ def construct_timeline_data(
Returns:
List of timeline segments for plotting
"""
def tostr(sec_time: float) -> str:
@@ -229,8 +228,7 @@ def generate_dataset_stats_plot(
results: list[dict[str, Any]],
output_path: Path,
) -> None:
"""
Generate a matplotlib figure with dataset statistics.
"""Generate a matplotlib figure with dataset statistics.
Creates a figure with 4 subplots:
- Top-left: Prompt tokens distribution (histogram)
@@ -243,6 +241,7 @@ def generate_dataset_stats_plot(
- prompt_len: Number of prompt tokens
- output_tokens: Number of output tokens
output_path: Path where the figure will be saved
"""
# Extract data
prompt_tokens = []
+8 -2
View File
@@ -407,13 +407,15 @@ async def get_request(
ramp_up_end_rps: int | None = None,
self_timed: bool = False,
) -> AsyncGenerator[tuple[SampleRequest, float], None]:
"""
Asynchronously generates requests at a specified rate
"""Asynchronously generates requests at a specified rate
with OPTIONAL burstiness and OPTIONAL ramp-up strategy.
Args:
input_requests:
A list of input requests, each represented as a SampleRequest.
self_timed:
If True, the requests carry their own arrival timing and no
request rate, burstiness or ramp-up is applied.
request_rate:
The rate at which requests are generated (requests/s).
burstiness (optional):
@@ -431,6 +433,7 @@ async def get_request(
The starting request rate for ramp-up.
ramp_up_end_rps (optional):
The ending request rate for ramp-up.
"""
assert burstiness > 0, (
f"A positive burstiness factor is expected, but given {burstiness}."
@@ -526,6 +529,7 @@ def calculate_metrics_for_embeddings(
Returns:
The calculated benchmark metrics.
"""
total_input = 0
total_input_sequences = 0
@@ -585,6 +589,7 @@ def calculate_metrics(
Returns:
A tuple of the benchmark metrics and the actual output lengths.
"""
actual_output_lens: list[int] = []
total_input = 0
@@ -1571,6 +1576,7 @@ def compute_result_filename(
Returns:
The computed filename path or None if no result saving is requested
"""
if not (args.plot_timeline or args.save_result or args.append_result):
return None
+3 -7
View File
@@ -107,8 +107,7 @@ def _metric_to_json(m: MetricStats) -> dict[str, Any]:
@contextmanager
def cold_startup():
"""
Context manager to measure cold startup time:
"""Context manager to measure cold startup time:
1. Uses a temporary directory for vLLM cache to avoid any pollution
between cold startup iterations.
2. Uses inductor's fresh_cache to clear torch.compile caches.
@@ -132,8 +131,7 @@ def cold_startup():
def run_startup_in_subprocess(engine_args, result_queue):
"""
Run LLM startup in a subprocess and return timing metrics via a queue.
"""Run LLM startup in a subprocess and return timing metrics via a queue.
This ensures complete isolation between iterations.
"""
try:
@@ -229,11 +227,9 @@ def main(args: argparse.Namespace):
engine_args = EngineArgs.from_cli_args(args)
def create_llm_and_measure_startup():
"""
Create LLM instance in a subprocess and measure startup time.
"""Create LLM instance in a subprocess and measure startup time.
Returns timing metrics, using subprocess for complete isolation.
"""
# Create a queue for inter-process communication
result_queue: multiprocessing.Queue[Any] = multiprocessing.Queue()
process = multiprocessing.Process(
+4 -6
View File
@@ -18,14 +18,14 @@ class ParameterSweep(list["ParameterSweepItem"]):
@classmethod
def read_from_dict(cls, data: dict[str, dict[str, object]]):
"""
Read parameter sweep from a dict format where keys are names.
"""Read parameter sweep from a dict format where keys are names.
Example:
{
"experiment1": {"max_tokens": 100, "temperature": 0.7},
"experiment2": {"max_tokens": 200, "temperature": 0.9}
}
"""
records = [{"_benchmark_name": name, **params} for name, params in data.items()]
return cls.from_records(records)
@@ -66,8 +66,7 @@ class ParameterSweepItem(dict[str, object]):
@property
def name(self) -> str:
"""
Get the name for this parameter sweep item.
"""Get the name for this parameter sweep item.
Returns the '_benchmark_name' field if present, otherwise returns a text
representation of all parameters.
@@ -103,8 +102,7 @@ class ParameterSweepItem(dict[str, object]):
return any(k in self for k in self._iter_param_key_candidates(param_key))
def _normalize_cmd_kv_pair(self, k: str, v: object) -> list[str]:
"""
Normalize a key-value pair into command-line arguments.
"""Normalize a key-value pair into command-line arguments.
Returns a list containing either:
- A single element for boolean flags (e.g., ['--flag'] or ['--flag=true'])
+1 -2
View File
@@ -178,8 +178,7 @@ def _json_load_bytes(path: Path) -> list[dict[str, object]]:
def _convert_inf_nan_strings(data: list[dict[str, object]]) -> list[dict[str, object]]:
"""
Convert string values "inf", "-inf", and "nan" to their float equivalents.
"""Convert string values "inf", "-inf", and "nan" to their float equivalents.
This handles the case where JSON serialization represents inf/nan as strings.
"""
+3 -6
View File
@@ -195,8 +195,7 @@ def run_vllm_chat(
warmup_requests: list[SampleRequest] | None = None,
prequeue_requests: bool = False,
) -> tuple[float, list[RequestOutput]]:
"""
Run vLLM chat benchmark. This function is recommended ONLY for benchmarking
"""Run vLLM chat benchmark. This function is recommended ONLY for benchmarking
multimodal models as it properly handles multimodal inputs and chat
formatting. For non-multimodal models, use run_vllm() instead.
"""
@@ -536,6 +535,7 @@ def _to_serve_args(args: argparse.Namespace) -> argparse.Namespace:
Returns:
A namespace satisfying get_samples's attribute reads.
"""
d = vars(args).copy()
# random_*: prefer --random-* over legacy --input/output/prefix-len.
@@ -627,10 +627,7 @@ def assign_loras(requests, args):
def validate_args(args):
"""
Validate command-line arguments.
"""
"""Validate command-line arguments."""
# === Deprecation and Defaulting ===
if args.dataset is not None:
warnings.warn(