Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Convert a plain-text file (local path or URL) into a JSONL dataset
|
||||
"""Convert a plain-text file (local path or URL) into a JSONL dataset
|
||||
compatible with ``CustomDataset`` (``--dataset-name custom``), by
|
||||
randomly slicing the tokenized text into prompts.
|
||||
|
||||
@@ -66,8 +65,8 @@ def create_txt_slices_jsonl(
|
||||
trust_remote_code: bool = False,
|
||||
) -> None:
|
||||
"""Read *input_path*, slice it into prompts, and write JSONL to
|
||||
*output_path*."""
|
||||
|
||||
*output_path*.
|
||||
"""
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
tokenizer_name, trust_remote_code=trust_remote_code
|
||||
)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This module defines a framework for sampling benchmark requests from various
|
||||
"""This module defines a framework for sampling benchmark requests from various
|
||||
datasets. Each dataset subclass of BenchmarkDataset must implement sample
|
||||
generation. Supported dataset types include:
|
||||
- ShareGPT
|
||||
@@ -75,9 +74,7 @@ DEFAULT_NUM_PROMPTS = 1000
|
||||
|
||||
@dataclass
|
||||
class SampleRequest:
|
||||
"""
|
||||
Represents a single inference request for benchmarking.
|
||||
"""
|
||||
"""Represents a single inference request for benchmarking."""
|
||||
|
||||
prompt: str | list[str] | list[int] | list[list[int]] | list[dict]
|
||||
prompt_len: int
|
||||
@@ -112,8 +109,7 @@ class BenchmarkDataset(ABC):
|
||||
disable_shuffle: bool = False,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
"""
|
||||
Initialize the BenchmarkDataset with an optional dataset path and random
|
||||
"""Initialize the BenchmarkDataset with an optional dataset path and random
|
||||
seed.
|
||||
|
||||
Args:
|
||||
@@ -121,6 +117,9 @@ class BenchmarkDataset(ABC):
|
||||
indicates that a default or random dataset might be used.
|
||||
random_seed (int): Seed value for reproducible shuffling or
|
||||
sampling. Defaults to DEFAULT_SEED.
|
||||
disable_shuffle (bool): If True, keep the dataset in its original
|
||||
order instead of shuffling it.
|
||||
|
||||
"""
|
||||
self.dataset_path = dataset_path
|
||||
# Set the random seed, ensuring that a None value is replaced with the
|
||||
@@ -134,8 +133,7 @@ class BenchmarkDataset(ABC):
|
||||
prompt: str,
|
||||
mm_content: MultiModalDataDict | dict | list[dict] | None = None,
|
||||
) -> list[dict]:
|
||||
"""
|
||||
Transform a prompt and optional multimodal content into a chat format.
|
||||
"""Transform a prompt and optional multimodal content into a chat format.
|
||||
This method is used for chat models that expect a specific conversation
|
||||
format.
|
||||
"""
|
||||
@@ -152,14 +150,14 @@ class BenchmarkDataset(ABC):
|
||||
return [{"role": "user", "content": content}]
|
||||
|
||||
def load_data(self) -> None:
|
||||
"""
|
||||
Load data from the dataset path into self.data.
|
||||
"""Load data from the dataset path into self.data.
|
||||
|
||||
This method must be overridden by subclasses since the method to load
|
||||
data will vary depending on the dataset format and source.
|
||||
|
||||
Raises:
|
||||
NotImplementedError: If a subclass does not implement this method.
|
||||
|
||||
"""
|
||||
# TODO (jenniferzhao): add support for downloading data
|
||||
raise NotImplementedError("load_data must be implemented in subclasses.")
|
||||
@@ -169,8 +167,7 @@ class BenchmarkDataset(ABC):
|
||||
max_loras: int | None = None,
|
||||
lora_path: str | None = None,
|
||||
) -> LoRARequest | None:
|
||||
"""
|
||||
Optionally select a random LoRA request.
|
||||
"""Optionally select a random LoRA request.
|
||||
|
||||
This method is used when LoRA parameters are provided. It randomly
|
||||
selects a LoRA based on max_loras.
|
||||
@@ -184,6 +181,7 @@ class BenchmarkDataset(ABC):
|
||||
Returns:
|
||||
A new [`LoRARequest`][vllm.lora.request.LoRARequest]
|
||||
(or `None` if not applicable).
|
||||
|
||||
"""
|
||||
if max_loras is None or lora_path is None:
|
||||
return None
|
||||
@@ -203,8 +201,7 @@ class BenchmarkDataset(ABC):
|
||||
max_loras: int | None = None,
|
||||
lora_path: str | None = None,
|
||||
) -> LoRARequest | None:
|
||||
"""
|
||||
Optionally select a LoRA request using deterministic round-robin.
|
||||
"""Optionally select a LoRA request using deterministic round-robin.
|
||||
|
||||
This method cycles through LoRA IDs in order based on the request
|
||||
index, providing reproducible LoRA assignment.
|
||||
@@ -219,6 +216,7 @@ class BenchmarkDataset(ABC):
|
||||
Returns:
|
||||
A new [`LoRARequest`][vllm.lora.request.LoRARequest]
|
||||
(or `None` if not applicable).
|
||||
|
||||
"""
|
||||
if max_loras is None or lora_path is None:
|
||||
return None
|
||||
@@ -239,8 +237,7 @@ class BenchmarkDataset(ABC):
|
||||
lora_path: str | None = None,
|
||||
lora_assignment: str = "random",
|
||||
) -> LoRARequest | None:
|
||||
"""
|
||||
Select a LoRA request using the specified assignment strategy.
|
||||
"""Select a LoRA request using the specified assignment strategy.
|
||||
|
||||
Args:
|
||||
index (int): The request index (used for round-robin).
|
||||
@@ -252,6 +249,7 @@ class BenchmarkDataset(ABC):
|
||||
Returns:
|
||||
A new [`LoRARequest`][vllm.lora.request.LoRARequest]
|
||||
(or `None` if not applicable).
|
||||
|
||||
"""
|
||||
if lora_assignment == "round-robin":
|
||||
return BenchmarkDataset.get_round_robin_lora_request(
|
||||
@@ -270,8 +268,7 @@ class BenchmarkDataset(ABC):
|
||||
no_oversample: bool = False,
|
||||
**kwargs,
|
||||
) -> list[SampleRequest]:
|
||||
"""
|
||||
Abstract method to generate sample requests from the dataset.
|
||||
"""Abstract method to generate sample requests from the dataset.
|
||||
|
||||
Subclasses must override this method to implement dataset-specific logic
|
||||
for generating a list of SampleRequest objects.
|
||||
@@ -281,10 +278,13 @@ class BenchmarkDataset(ABC):
|
||||
for processing the dataset's text.
|
||||
num_requests (int): The number of sample requests to generate.
|
||||
request_id_prefix (str): The prefix of request_id.
|
||||
no_oversample (bool): If True, do not oversample the dataset to
|
||||
reach num_requests.
|
||||
|
||||
Returns:
|
||||
list[SampleRequest]: A list of sample requests generated from the
|
||||
dataset.
|
||||
|
||||
"""
|
||||
raise NotImplementedError("sample must be implemented in subclasses.")
|
||||
|
||||
@@ -295,8 +295,7 @@ class BenchmarkDataset(ABC):
|
||||
request_id_prefix: str = "",
|
||||
no_oversample: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
Oversamples the list of requests if its size is less than the desired
|
||||
"""Oversamples the list of requests if its size is less than the desired
|
||||
number.
|
||||
|
||||
Args:
|
||||
@@ -305,6 +304,8 @@ class BenchmarkDataset(ABC):
|
||||
num_requests (int): The target number of requests.
|
||||
request_id_prefix (str): The prefix applied to generated request
|
||||
identifiers.
|
||||
no_oversample (bool): If True, return the requests unchanged
|
||||
instead of oversampling up to num_requests.
|
||||
|
||||
"""
|
||||
if no_oversample:
|
||||
@@ -346,8 +347,7 @@ def is_valid_sequence(
|
||||
max_total_len: int = 2048,
|
||||
skip_min_output_len_check: bool = False,
|
||||
) -> bool:
|
||||
"""
|
||||
Validate a sequence based on prompt and output lengths.
|
||||
"""Validate a sequence based on prompt and output lengths.
|
||||
|
||||
Default pruning criteria are copied from the original `sample_hf_requests`
|
||||
and `sample_sharegpt_requests` functions in benchmark_serving.py, as well as
|
||||
@@ -379,8 +379,7 @@ def process_image(
|
||||
*,
|
||||
ensure_client_side_data: bool = False,
|
||||
) -> Mapping[str, Any]:
|
||||
"""
|
||||
Process a single image input and return a multimedia content dictionary.
|
||||
"""Process a single image input and return a multimedia content dictionary.
|
||||
|
||||
Supports the following input types:
|
||||
|
||||
@@ -402,6 +401,7 @@ def process_image(
|
||||
|
||||
Raises:
|
||||
ValueError: If the input is not a supported type.
|
||||
|
||||
"""
|
||||
if isinstance(image, dict) and "bytes" in image:
|
||||
image = Image.open(BytesIO(image["bytes"]))
|
||||
@@ -437,8 +437,7 @@ def process_image(
|
||||
|
||||
|
||||
def process_video(video: Any) -> Mapping[str, Any]:
|
||||
"""
|
||||
Process a single video input and return a multimedia content dictionary.
|
||||
"""Process a single video input and return a multimedia content dictionary.
|
||||
|
||||
Supports the following input types:
|
||||
|
||||
@@ -451,6 +450,7 @@ def process_video(video: Any) -> Mapping[str, Any]:
|
||||
|
||||
Raises:
|
||||
ValueError: If the input is not a supported type.
|
||||
|
||||
"""
|
||||
if isinstance(video, dict) and "bytes" in video:
|
||||
video_bytes = video["bytes"]
|
||||
@@ -474,8 +474,7 @@ def process_video(video: Any) -> Mapping[str, Any]:
|
||||
|
||||
|
||||
def process_audio(audio: Any) -> tuple:
|
||||
"""
|
||||
Process a single audio input and return a (array, sample_rate) tuple.
|
||||
"""Process a single audio input and return a (array, sample_rate) tuple.
|
||||
|
||||
Supports:
|
||||
1. String: treated as a file path, loaded with soundfile.
|
||||
@@ -502,8 +501,7 @@ def gen_prompt_decode_to_target_len(
|
||||
add_special_tokens: bool = False,
|
||||
rng: np.random.Generator | None = None,
|
||||
) -> tuple[str, list[int], int]:
|
||||
"""
|
||||
Ensure decoded-then-encoded prompt length matches the target token length.
|
||||
"""Ensure decoded-then-encoded prompt length matches the target token length.
|
||||
|
||||
This function decodes an initial token sequence to text and re-encodes it
|
||||
, iteratively adjusting the token sequence length to match a target.
|
||||
@@ -557,8 +555,7 @@ def gen_prompt_decode_to_target_len(
|
||||
|
||||
|
||||
class RandomDataset(BenchmarkDataset):
|
||||
"""
|
||||
Synthetic text-only dataset for serving/throughput benchmarks.
|
||||
"""Synthetic text-only dataset for serving/throughput benchmarks.
|
||||
|
||||
Strategy:
|
||||
- Sample input/output token lengths per request from integer-uniform ranges
|
||||
@@ -699,9 +696,7 @@ class RandomDataset(BenchmarkDataset):
|
||||
allowed_tokens: np.ndarray,
|
||||
prefix_len: int,
|
||||
) -> list[int]:
|
||||
"""
|
||||
Get the prefix for the dataset.
|
||||
"""
|
||||
"""Get the prefix for the dataset."""
|
||||
if prefix_len <= 0:
|
||||
return []
|
||||
|
||||
@@ -738,8 +733,7 @@ class RandomDataset(BenchmarkDataset):
|
||||
index: int,
|
||||
allowed_tokens: np.ndarray,
|
||||
) -> tuple[str, int, int]:
|
||||
"""
|
||||
Returns (prompt, total_input_len).
|
||||
"""Returns (prompt, total_input_len).
|
||||
|
||||
NOTE: After decoding the prompt we have to encode and decode it again.
|
||||
This is done because in some cases N consecutive tokens
|
||||
@@ -778,8 +772,7 @@ class RandomDataset(BenchmarkDataset):
|
||||
|
||||
|
||||
class RandomDatasetForReranking(RandomDataset):
|
||||
"""
|
||||
Random dataset specialized for the needs of scoring:
|
||||
"""Random dataset specialized for the needs of scoring:
|
||||
- Batches of inputs
|
||||
- Inputs composed of pairs
|
||||
"""
|
||||
@@ -906,8 +899,7 @@ class RandomDatasetForReranking(RandomDataset):
|
||||
|
||||
|
||||
class RandomMultiModalDataset(RandomDataset):
|
||||
"""
|
||||
Synthetic multimodal dataset (text + images) that extends RandomDataset.
|
||||
"""Synthetic multimodal dataset (text + images) that extends RandomDataset.
|
||||
|
||||
Status:
|
||||
- Images: supported via synthetic RGB data.
|
||||
@@ -1022,8 +1014,7 @@ class RandomMultiModalDataset(RandomDataset):
|
||||
def normalize_bucket_config(
|
||||
self, bucket_config: dict[tuple[int, int, int], float]
|
||||
) -> dict[tuple[int, int, int], float]:
|
||||
"""
|
||||
Remove zero probability entries
|
||||
"""Remove zero probability entries
|
||||
and normalize the bucket config to sum to 1.
|
||||
"""
|
||||
# Raise error if value is negative
|
||||
@@ -1044,13 +1035,11 @@ class RandomMultiModalDataset(RandomDataset):
|
||||
self,
|
||||
mm_item_config: tuple[int, int, int],
|
||||
) -> Mapping[str, Any]:
|
||||
"""
|
||||
Create synthetic images and videos and
|
||||
"""Create synthetic images and videos and
|
||||
apply process_image/process_video respectively.
|
||||
This follows the OpenAI API chat completions
|
||||
https://github.com/openai/openai-python
|
||||
"""
|
||||
|
||||
if self.map_config_to_modality(mm_item_config) == "image":
|
||||
return process_image(
|
||||
self.generate_synthetic_image(mm_item_config[1], mm_item_config[0])
|
||||
@@ -1071,9 +1060,7 @@ class RandomMultiModalDataset(RandomDataset):
|
||||
limit_mm_per_prompt: dict[str, int],
|
||||
bucket_config: dict[tuple[int, int, int], float],
|
||||
) -> tuple[int, int, dict[str, int], dict[tuple[int, int, int], float]]:
|
||||
"""
|
||||
Get the sampling parameters for the multimodal items.
|
||||
"""
|
||||
"""Get the sampling parameters for the multimodal items."""
|
||||
# Enforce num_mm_items_range_ratio <= 1
|
||||
if not (0.0 <= num_mm_items_range_ratio <= 1.0):
|
||||
raise ValueError("num_mm_items_range_ratio must be in [0, 1].")
|
||||
@@ -1146,8 +1133,7 @@ class RandomMultiModalDataset(RandomDataset):
|
||||
bucket_config: dict[tuple[int, int, int], float],
|
||||
limit_mm_per_prompt: dict[str, int],
|
||||
) -> Iterator[tuple[int, int, int]]:
|
||||
"""
|
||||
Iterator over the multimodal items for each request
|
||||
"""Iterator over the multimodal items for each request
|
||||
whose size is between min_num_mm_items and max_num_mm_items.
|
||||
|
||||
Loop over the bucket config and sample a multimodal item.
|
||||
@@ -1160,6 +1146,7 @@ class RandomMultiModalDataset(RandomDataset):
|
||||
`bucket_config` (tuple->float). The original dict passed to
|
||||
`sample` is not mutated. If this ever changes, a test
|
||||
is implemented and will fail.
|
||||
|
||||
"""
|
||||
# Get the number of multimodal items to sample
|
||||
request_num_mm_items = int(
|
||||
@@ -1346,8 +1333,7 @@ class RandomMultiModalDataset(RandomDataset):
|
||||
|
||||
|
||||
class ShareGPTDataset(BenchmarkDataset):
|
||||
"""
|
||||
Implements the ShareGPT dataset. Loads data from a JSON file and generates
|
||||
"""Implements the ShareGPT dataset. Loads data from a JSON file and generates
|
||||
sample requests based on conversation turns.
|
||||
"""
|
||||
|
||||
@@ -1437,8 +1423,7 @@ class ShareGPTDataset(BenchmarkDataset):
|
||||
|
||||
|
||||
class TimedTrace(BenchmarkDataset):
|
||||
"""
|
||||
Implements a base class to replay various timed traces.
|
||||
"""Implements a base class to replay various timed traces.
|
||||
Loads data from a JSON file and generates sample requests
|
||||
based on the timing information in the traces.
|
||||
"""
|
||||
@@ -1927,6 +1912,7 @@ def add_random_dataset_base_args(
|
||||
|
||||
Args:
|
||||
parser_or_group: Either a parser or an argument group to add arguments to.
|
||||
|
||||
"""
|
||||
parser_or_group.add_argument(
|
||||
"--random-input-len",
|
||||
@@ -1988,6 +1974,7 @@ def add_random_multimodal_dataset_args(
|
||||
|
||||
Args:
|
||||
parser_or_group: Either a parser or an argument group to add arguments to.
|
||||
|
||||
"""
|
||||
parser_or_group.add_argument(
|
||||
"--random-mm-base-items-per-request",
|
||||
@@ -2503,8 +2490,7 @@ def get_samples(
|
||||
|
||||
|
||||
class CustomDataset(BenchmarkDataset):
|
||||
"""
|
||||
Implements the Custom dataset. Loads data from a JSONL file and generates
|
||||
"""Implements the Custom dataset. Loads data from a JSONL file and generates
|
||||
sample requests based on conversation turns. E.g.,
|
||||
```
|
||||
{"prompt": "What is the capital of India?", "output_tokens": 10}
|
||||
@@ -2632,8 +2618,7 @@ class CustomDataset(BenchmarkDataset):
|
||||
|
||||
|
||||
class CustomImageDataset(CustomDataset):
|
||||
"""
|
||||
Implements the Custom image dataset. Loads data from a JSONL file and generates
|
||||
"""Implements the Custom image dataset. Loads data from a JSONL file and generates
|
||||
sample requests based on conversation turns. E.g.,
|
||||
```
|
||||
{
|
||||
@@ -2917,8 +2902,7 @@ class CustomImageDataset(CustomDataset):
|
||||
|
||||
|
||||
class CustomAudioDataset(CustomDataset):
|
||||
"""
|
||||
Custom dataset for audio benchmarking. Loads data from a JSONL file. E.g.,
|
||||
"""Custom dataset for audio benchmarking. Loads data from a JSONL file. E.g.,
|
||||
{"prompt": "Transcribe the audio.", "audio": "/path/to/audio.wav"}
|
||||
|
||||
Supports both:
|
||||
@@ -3027,8 +3011,7 @@ class CustomAudioDataset(CustomDataset):
|
||||
|
||||
|
||||
class SpecBench(CustomDataset):
|
||||
"""
|
||||
Implements the SpecBench dataset: https://github.com/hemingkx/Spec-Bench
|
||||
"""Implements the SpecBench dataset: https://github.com/hemingkx/Spec-Bench
|
||||
Download the dataset using:
|
||||
wget https://raw.githubusercontent.com/hemingkx/Spec-Bench/refs/heads/main/data/spec_bench/question.jsonl
|
||||
""" # noqa: E501
|
||||
@@ -3100,8 +3083,7 @@ class SpecBench(CustomDataset):
|
||||
"SonnetDataset is deprecated and will be removed in a future version.",
|
||||
)
|
||||
class SonnetDataset(BenchmarkDataset):
|
||||
"""
|
||||
Simplified implementation of the Sonnet dataset. Loads poem lines from a
|
||||
"""Simplified implementation of the Sonnet dataset. Loads poem lines from a
|
||||
text file and generates sample requests. Default values here copied from
|
||||
`benchmark_serving.py` for the sonnet dataset.
|
||||
"""
|
||||
@@ -3198,8 +3180,7 @@ class SonnetDataset(BenchmarkDataset):
|
||||
|
||||
|
||||
class BurstGPTDataset(BenchmarkDataset):
|
||||
"""
|
||||
Implements the BurstGPT dataset. Loads data from a CSV file and generates
|
||||
"""Implements the BurstGPT dataset. Loads data from a CSV file and generates
|
||||
sample requests based on synthetic prompt generation. Only rows with Model
|
||||
"GPT-4" and positive response tokens are used.
|
||||
"""
|
||||
@@ -3447,9 +3428,7 @@ class MultiModalConversationDataset(HuggingFaceDataset):
|
||||
|
||||
|
||||
class VisionArenaDataset(HuggingFaceDataset):
|
||||
"""
|
||||
Vision Arena Dataset.
|
||||
"""
|
||||
"""Vision Arena Dataset."""
|
||||
|
||||
DEFAULT_OUTPUT_LEN = 128
|
||||
SUPPORTED_DATASET_PATHS = {
|
||||
@@ -3506,8 +3485,7 @@ class VisionArenaDataset(HuggingFaceDataset):
|
||||
|
||||
|
||||
class MMVUDataset(HuggingFaceDataset):
|
||||
"""
|
||||
MMVU Dataset.
|
||||
"""MMVU Dataset.
|
||||
https://huggingface.co/datasets/yale-nlp/MMVU
|
||||
"""
|
||||
|
||||
@@ -3586,8 +3564,7 @@ class MMVUDataset(HuggingFaceDataset):
|
||||
|
||||
|
||||
class InstructCoderDataset(HuggingFaceDataset):
|
||||
"""
|
||||
InstructCoder Dataset.
|
||||
"""InstructCoder Dataset.
|
||||
https://huggingface.co/datasets/likaixin/InstructCoder
|
||||
|
||||
InstructCoder is the dataset designed for general code editing. It consists
|
||||
@@ -3655,8 +3632,7 @@ class InstructCoderDataset(HuggingFaceDataset):
|
||||
|
||||
|
||||
class MTBenchDataset(HuggingFaceDataset):
|
||||
"""
|
||||
MT-Bench Dataset.
|
||||
"""MT-Bench Dataset.
|
||||
https://huggingface.co/datasets/philschmid/mt-bench
|
||||
|
||||
We create a single turn dataset for MT-Bench.
|
||||
@@ -3718,8 +3694,7 @@ class MTBenchDataset(HuggingFaceDataset):
|
||||
|
||||
|
||||
class HumanEvalDataset(HuggingFaceDataset):
|
||||
"""
|
||||
HumanEvalDataset Dataset.
|
||||
"""HumanEvalDataset Dataset.
|
||||
https://huggingface.co/datasets/openai/openai_humaneval
|
||||
|
||||
We create a single turn dataset for HumanEval.
|
||||
@@ -3779,8 +3754,7 @@ class HumanEvalDataset(HuggingFaceDataset):
|
||||
|
||||
|
||||
class GSM8KDataset(HuggingFaceDataset):
|
||||
"""
|
||||
GSM8K Dataset.
|
||||
"""GSM8K Dataset.
|
||||
https://huggingface.co/datasets/openai/gsm8k
|
||||
|
||||
We create a single turn dataset for GSM8K.
|
||||
@@ -3840,8 +3814,7 @@ class GSM8KDataset(HuggingFaceDataset):
|
||||
|
||||
|
||||
class BlazeditDataset(HuggingFaceDataset):
|
||||
"""
|
||||
Blazedit Dataset.
|
||||
"""Blazedit Dataset.
|
||||
https://github.com/ise-uiuc/blazedit
|
||||
|
||||
5k char version: vdaita/edit_5k_char
|
||||
@@ -3932,9 +3905,7 @@ Please generate the new code file in the "New file" section below.""" # noqa: E
|
||||
|
||||
|
||||
class AIMODataset(HuggingFaceDataset):
|
||||
"""
|
||||
Dataset class for processing a AIMO dataset with reasoning questions.
|
||||
"""
|
||||
"""Dataset class for processing a AIMO dataset with reasoning questions."""
|
||||
|
||||
SUPPORTED_DATASET_PATHS = {
|
||||
"AI-MO/aimo-validation-aime",
|
||||
@@ -4026,6 +3997,7 @@ def _format_zeta_prompt(
|
||||
|
||||
Returns:
|
||||
A dictionary with the formatted prompts and expected outputs.
|
||||
|
||||
"""
|
||||
events = sample["events"]
|
||||
input = sample["input"]
|
||||
@@ -4042,9 +4014,7 @@ def _format_zeta_prompt(
|
||||
|
||||
|
||||
class NextEditPredictionDataset(HuggingFaceDataset):
|
||||
"""
|
||||
Dataset class for processing a Next Edit Prediction dataset.
|
||||
"""
|
||||
"""Dataset class for processing a Next Edit Prediction dataset."""
|
||||
|
||||
SUPPORTED_DATASET_PATHS = {
|
||||
"zed-industries/zeta",
|
||||
@@ -4092,8 +4062,7 @@ class NextEditPredictionDataset(HuggingFaceDataset):
|
||||
|
||||
|
||||
class ASRDataset(HuggingFaceDataset):
|
||||
"""
|
||||
Dataset class for processing a ASR dataset for transcription.
|
||||
"""Dataset class for processing a ASR dataset for transcription.
|
||||
Tested on the following set:
|
||||
|
||||
+---------------------------+----------------------------------------+--------------------------+-----------------------------+
|
||||
@@ -4290,8 +4259,7 @@ class ASRDataset(HuggingFaceDataset):
|
||||
|
||||
|
||||
class MLPerfDataset(HuggingFaceDataset):
|
||||
"""
|
||||
MLPerf Inference Dataset.
|
||||
"""MLPerf Inference Dataset.
|
||||
|
||||
Dataset on HF:
|
||||
https://huggingface.co/datasets/mgoin/mlperf-inference-llama2-data
|
||||
@@ -4415,7 +4383,8 @@ class PrefixRepetitionRandomDataset(BenchmarkDataset):
|
||||
|
||||
def _generate_exact_length_tokens(target_length: int) -> tuple[list[int], int]:
|
||||
"""Generate tokens that decode and re-encode to exactly
|
||||
target_length."""
|
||||
target_length.
|
||||
"""
|
||||
# Generate random tokens
|
||||
tokens = np.random.randint(0, vocab_size, size=target_length).tolist()
|
||||
|
||||
@@ -4470,8 +4439,7 @@ class PrefixRepetitionRandomDataset(BenchmarkDataset):
|
||||
|
||||
|
||||
class MMStarDataset(HuggingFaceDataset):
|
||||
"""
|
||||
Lin-Chen/MMStar: https://huggingface.co/datasets/Lin-Chen/MMStar
|
||||
"""Lin-Chen/MMStar: https://huggingface.co/datasets/Lin-Chen/MMStar
|
||||
refer to: https://github.com/sgl-project/SpecForge/pull/106
|
||||
"""
|
||||
|
||||
@@ -4760,8 +4728,7 @@ class BFCLDataset(HuggingFaceDataset):
|
||||
|
||||
|
||||
class SpeedBench(CustomDataset):
|
||||
"""
|
||||
SPEED-Bench dataset: https://huggingface.co/datasets/nvidia/SPEED-Bench
|
||||
"""SPEED-Bench dataset: https://huggingface.co/datasets/nvidia/SPEED-Bench.
|
||||
|
||||
Download the dataset using:
|
||||
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Shared utilities for benchmark dataset sampling.
|
||||
"""
|
||||
"""Shared utilities for benchmark dataset sampling."""
|
||||
|
||||
import logging
|
||||
import math
|
||||
@@ -46,8 +44,7 @@ def get_sampling_params(
|
||||
output_len: int,
|
||||
tokenizer: TokenizerLike,
|
||||
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
|
||||
"""
|
||||
Sample per-request input/output token lengths and vocab offsets.
|
||||
"""Sample per-request input/output token lengths and vocab offsets.
|
||||
|
||||
Lengths are drawn uniformly from integer ranges around the configured
|
||||
means, controlled by *range_ratio*. It may be a single ``float``
|
||||
@@ -60,6 +57,7 @@ def get_sampling_params(
|
||||
Returns:
|
||||
(input_lens, output_lens, offsets) – three 1-D ``np.ndarray`` of
|
||||
shape ``(num_requests,)``.
|
||||
|
||||
"""
|
||||
input_range_ratio, output_range_ratio = _resolve_range_ratios(range_ratio)
|
||||
|
||||
|
||||
@@ -22,7 +22,8 @@ AIOHTTP_TIMEOUT = aiohttp.ClientTimeout(total=6 * 60 * 60)
|
||||
|
||||
class StreamedResponseHandler:
|
||||
"""Handles streaming HTTP responses by accumulating chunks until complete
|
||||
messages are available."""
|
||||
messages are available.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.buffer = ""
|
||||
@@ -30,7 +31,8 @@ class StreamedResponseHandler:
|
||||
|
||||
def add_chunk(self, chunk_bytes: bytes) -> list[str]:
|
||||
"""Add a chunk of bytes to the buffer and return any complete
|
||||
messages."""
|
||||
messages.
|
||||
"""
|
||||
chunk_str = self._decoder.decode(chunk_bytes)
|
||||
self.buffer += chunk_str
|
||||
|
||||
@@ -167,10 +169,12 @@ async def async_request_openai_completions(
|
||||
|
||||
Args:
|
||||
request_func_input: The input for the request function.
|
||||
session: The aiohttp session used to issue the request.
|
||||
pbar: The progress bar to display the progress.
|
||||
|
||||
Returns:
|
||||
The output of the request function.
|
||||
|
||||
"""
|
||||
api_url = request_func_input.api_url
|
||||
_validate_api_url(api_url, "OpenAI Completions API", "completions")
|
||||
|
||||
@@ -22,12 +22,12 @@ async def wait_for_endpoint(
|
||||
timeout_seconds: int = 600,
|
||||
retry_interval: int = 5,
|
||||
) -> RequestFuncOutput:
|
||||
"""
|
||||
Wait for an endpoint to become available before starting benchmarks.
|
||||
"""Wait for an endpoint to become available before starting benchmarks.
|
||||
|
||||
Args:
|
||||
request_func: The async request function to call
|
||||
test_input: The RequestFuncInput to test with
|
||||
session: The aiohttp session used to issue the probe requests
|
||||
timeout_seconds: Maximum time to wait in seconds (default: 10 minutes)
|
||||
retry_interval: Time between retries in seconds (default: 5 seconds)
|
||||
|
||||
@@ -36,6 +36,7 @@ async def wait_for_endpoint(
|
||||
|
||||
Raises:
|
||||
ValueError: If the endpoint doesn't become available within the timeout
|
||||
|
||||
"""
|
||||
deadline = time.perf_counter() + timeout_seconds
|
||||
output = RequestFuncOutput(success=False)
|
||||
|
||||
@@ -36,9 +36,7 @@ def extract_field(
|
||||
|
||||
|
||||
def use_compile(args: argparse.Namespace, extra_info: dict[str, Any]) -> bool:
|
||||
"""
|
||||
Check if the benchmark is run with torch.compile
|
||||
"""
|
||||
"""Check if the benchmark is run with torch.compile."""
|
||||
return not (
|
||||
extract_field(args, extra_info, "compilation_config.mode") == "0"
|
||||
or "eager" in getattr(args, "output_json", "")
|
||||
@@ -49,8 +47,7 @@ def use_compile(args: argparse.Namespace, extra_info: dict[str, Any]) -> bool:
|
||||
def convert_to_pytorch_benchmark_format(
|
||||
args: argparse.Namespace, metrics: dict[str, list], extra_info: dict[str, Any]
|
||||
) -> list:
|
||||
"""
|
||||
Save the benchmark results in the format used by PyTorch OSS benchmark with
|
||||
"""Save the benchmark results in the format used by PyTorch OSS benchmark with
|
||||
on metric per record
|
||||
https://github.com/pytorch/pytorch/wiki/How-to-integrate-with-PyTorch-OSS-benchmark-database
|
||||
"""
|
||||
|
||||
@@ -44,8 +44,7 @@ else:
|
||||
|
||||
|
||||
def get_timing_stats_from_engine(llm_engine: LLMEngine) -> dict[str, dict[str, float]]:
|
||||
"""
|
||||
Get all multimodal timing stats from the LLM engine.
|
||||
"""Get all multimodal timing stats from the LLM engine.
|
||||
|
||||
Collects both preprocessing stats (HF processor, hashing, cache lookup,
|
||||
prompt update) and encoder forward pass timing, merged by request_id.
|
||||
@@ -70,6 +69,7 @@ def get_timing_stats_from_engine(llm_engine: LLMEngine) -> dict[str, dict[str, f
|
||||
'num_encoder_calls': 1
|
||||
}
|
||||
}
|
||||
|
||||
"""
|
||||
observability_config = llm_engine.vllm_config.observability_config
|
||||
if not observability_config or not observability_config.enable_mm_processor_stats:
|
||||
@@ -126,8 +126,7 @@ def get_timing_stats_from_engine(llm_engine: LLMEngine) -> dict[str, dict[str, f
|
||||
|
||||
|
||||
def collect_mm_processor_stats(llm_engine: LLMEngine) -> dict[str, list[float]]:
|
||||
"""
|
||||
Collect multimodal processor timing stats.
|
||||
"""Collect multimodal processor timing stats.
|
||||
Returns a dictionary mapping stage names to lists of timing values (in seconds).
|
||||
"""
|
||||
all_stats = get_timing_stats_from_engine(llm_engine)
|
||||
@@ -147,9 +146,7 @@ def calculate_mm_processor_metrics(
|
||||
*,
|
||||
unit: Literal["us", "ms", "s"] = "ms",
|
||||
) -> dict[str, dict[str, float]]:
|
||||
"""
|
||||
Calculate aggregate metrics from stats by stage.
|
||||
"""
|
||||
"""Calculate aggregate metrics from stats by stage."""
|
||||
unit2mult = {"us": 1000000, "ms": 1000, "s": 1}
|
||||
unit_mult = unit2mult[unit]
|
||||
|
||||
@@ -181,9 +178,7 @@ def calculate_mm_processor_metrics(
|
||||
|
||||
|
||||
def validate_args(args):
|
||||
"""
|
||||
Validate command-line arguments for mm_processor benchmark.
|
||||
"""
|
||||
"""Validate command-line arguments for mm_processor benchmark."""
|
||||
if not getattr(args, "tokenizer", None):
|
||||
args.tokenizer = args.model
|
||||
if not hasattr(args, "dataset_path"):
|
||||
@@ -214,9 +209,7 @@ def validate_args(args):
|
||||
def benchmark_multimodal_processor(
|
||||
args: argparse.Namespace,
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Run the multimodal processor benchmark.
|
||||
"""
|
||||
"""Run the multimodal processor benchmark."""
|
||||
from vllm import LLM, SamplingParams
|
||||
|
||||
validate_args(args)
|
||||
@@ -455,7 +448,6 @@ def add_cli_args(parser: FlexibleArgumentParser) -> None:
|
||||
|
||||
def main(args: argparse.Namespace) -> None:
|
||||
"""Main entry point for the multimodal processor benchmark."""
|
||||
|
||||
print("Starting multimodal processor benchmark...")
|
||||
result = benchmark_multimodal_processor(args)
|
||||
|
||||
|
||||
@@ -29,8 +29,7 @@ def generate_timeline_plot(
|
||||
itl_thresholds: list[float] | None = None,
|
||||
labels: list[str] | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Generate an HTML timeline plot from benchmark results.
|
||||
"""Generate an HTML timeline plot from benchmark results.
|
||||
|
||||
Args:
|
||||
results: List of per-request result dictionaries containing:
|
||||
@@ -44,8 +43,8 @@ def generate_timeline_plot(
|
||||
colors: List of colors for ITL categories (default: green, orange, red, black)
|
||||
itl_thresholds: ITL thresholds in seconds (default: [1.0, 4.0, 6.0])
|
||||
labels: Labels for ITL categories (default based on thresholds)
|
||||
"""
|
||||
|
||||
"""
|
||||
# Set defaults
|
||||
if colors is None:
|
||||
colors = ["#109618", "#FF7F0E", "#D62728"]
|
||||
@@ -118,8 +117,7 @@ def construct_timeline_data(
|
||||
itl_thresholds: list[float],
|
||||
labels: list[str],
|
||||
) -> list[dict[str, Any]]:
|
||||
"""
|
||||
Construct timeline data from request results.
|
||||
"""Construct timeline data from request results.
|
||||
|
||||
Args:
|
||||
requests_data: List of per-request result dictionaries
|
||||
@@ -128,6 +126,7 @@ def construct_timeline_data(
|
||||
|
||||
Returns:
|
||||
List of timeline segments for plotting
|
||||
|
||||
"""
|
||||
|
||||
def tostr(sec_time: float) -> str:
|
||||
@@ -229,8 +228,7 @@ def generate_dataset_stats_plot(
|
||||
results: list[dict[str, Any]],
|
||||
output_path: Path,
|
||||
) -> None:
|
||||
"""
|
||||
Generate a matplotlib figure with dataset statistics.
|
||||
"""Generate a matplotlib figure with dataset statistics.
|
||||
|
||||
Creates a figure with 4 subplots:
|
||||
- Top-left: Prompt tokens distribution (histogram)
|
||||
@@ -243,6 +241,7 @@ def generate_dataset_stats_plot(
|
||||
- prompt_len: Number of prompt tokens
|
||||
- output_tokens: Number of output tokens
|
||||
output_path: Path where the figure will be saved
|
||||
|
||||
"""
|
||||
# Extract data
|
||||
prompt_tokens = []
|
||||
|
||||
@@ -407,13 +407,15 @@ async def get_request(
|
||||
ramp_up_end_rps: int | None = None,
|
||||
self_timed: bool = False,
|
||||
) -> AsyncGenerator[tuple[SampleRequest, float], None]:
|
||||
"""
|
||||
Asynchronously generates requests at a specified rate
|
||||
"""Asynchronously generates requests at a specified rate
|
||||
with OPTIONAL burstiness and OPTIONAL ramp-up strategy.
|
||||
|
||||
Args:
|
||||
input_requests:
|
||||
A list of input requests, each represented as a SampleRequest.
|
||||
self_timed:
|
||||
If True, the requests carry their own arrival timing and no
|
||||
request rate, burstiness or ramp-up is applied.
|
||||
request_rate:
|
||||
The rate at which requests are generated (requests/s).
|
||||
burstiness (optional):
|
||||
@@ -431,6 +433,7 @@ async def get_request(
|
||||
The starting request rate for ramp-up.
|
||||
ramp_up_end_rps (optional):
|
||||
The ending request rate for ramp-up.
|
||||
|
||||
"""
|
||||
assert burstiness > 0, (
|
||||
f"A positive burstiness factor is expected, but given {burstiness}."
|
||||
@@ -526,6 +529,7 @@ def calculate_metrics_for_embeddings(
|
||||
|
||||
Returns:
|
||||
The calculated benchmark metrics.
|
||||
|
||||
"""
|
||||
total_input = 0
|
||||
total_input_sequences = 0
|
||||
@@ -585,6 +589,7 @@ def calculate_metrics(
|
||||
|
||||
Returns:
|
||||
A tuple of the benchmark metrics and the actual output lengths.
|
||||
|
||||
"""
|
||||
actual_output_lens: list[int] = []
|
||||
total_input = 0
|
||||
@@ -1571,6 +1576,7 @@ def compute_result_filename(
|
||||
|
||||
Returns:
|
||||
The computed filename path or None if no result saving is requested
|
||||
|
||||
"""
|
||||
if not (args.plot_timeline or args.save_result or args.append_result):
|
||||
return None
|
||||
|
||||
@@ -107,8 +107,7 @@ def _metric_to_json(m: MetricStats) -> dict[str, Any]:
|
||||
|
||||
@contextmanager
|
||||
def cold_startup():
|
||||
"""
|
||||
Context manager to measure cold startup time:
|
||||
"""Context manager to measure cold startup time:
|
||||
1. Uses a temporary directory for vLLM cache to avoid any pollution
|
||||
between cold startup iterations.
|
||||
2. Uses inductor's fresh_cache to clear torch.compile caches.
|
||||
@@ -132,8 +131,7 @@ def cold_startup():
|
||||
|
||||
|
||||
def run_startup_in_subprocess(engine_args, result_queue):
|
||||
"""
|
||||
Run LLM startup in a subprocess and return timing metrics via a queue.
|
||||
"""Run LLM startup in a subprocess and return timing metrics via a queue.
|
||||
This ensures complete isolation between iterations.
|
||||
"""
|
||||
try:
|
||||
@@ -229,11 +227,9 @@ def main(args: argparse.Namespace):
|
||||
engine_args = EngineArgs.from_cli_args(args)
|
||||
|
||||
def create_llm_and_measure_startup():
|
||||
"""
|
||||
Create LLM instance in a subprocess and measure startup time.
|
||||
"""Create LLM instance in a subprocess and measure startup time.
|
||||
Returns timing metrics, using subprocess for complete isolation.
|
||||
"""
|
||||
|
||||
# Create a queue for inter-process communication
|
||||
result_queue: multiprocessing.Queue[Any] = multiprocessing.Queue()
|
||||
process = multiprocessing.Process(
|
||||
|
||||
@@ -18,14 +18,14 @@ class ParameterSweep(list["ParameterSweepItem"]):
|
||||
|
||||
@classmethod
|
||||
def read_from_dict(cls, data: dict[str, dict[str, object]]):
|
||||
"""
|
||||
Read parameter sweep from a dict format where keys are names.
|
||||
"""Read parameter sweep from a dict format where keys are names.
|
||||
|
||||
Example:
|
||||
{
|
||||
"experiment1": {"max_tokens": 100, "temperature": 0.7},
|
||||
"experiment2": {"max_tokens": 200, "temperature": 0.9}
|
||||
}
|
||||
|
||||
"""
|
||||
records = [{"_benchmark_name": name, **params} for name, params in data.items()]
|
||||
return cls.from_records(records)
|
||||
@@ -66,8 +66,7 @@ class ParameterSweepItem(dict[str, object]):
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
"""
|
||||
Get the name for this parameter sweep item.
|
||||
"""Get the name for this parameter sweep item.
|
||||
|
||||
Returns the '_benchmark_name' field if present, otherwise returns a text
|
||||
representation of all parameters.
|
||||
@@ -103,8 +102,7 @@ class ParameterSweepItem(dict[str, object]):
|
||||
return any(k in self for k in self._iter_param_key_candidates(param_key))
|
||||
|
||||
def _normalize_cmd_kv_pair(self, k: str, v: object) -> list[str]:
|
||||
"""
|
||||
Normalize a key-value pair into command-line arguments.
|
||||
"""Normalize a key-value pair into command-line arguments.
|
||||
|
||||
Returns a list containing either:
|
||||
- A single element for boolean flags (e.g., ['--flag'] or ['--flag=true'])
|
||||
|
||||
@@ -178,8 +178,7 @@ def _json_load_bytes(path: Path) -> list[dict[str, object]]:
|
||||
|
||||
|
||||
def _convert_inf_nan_strings(data: list[dict[str, object]]) -> list[dict[str, object]]:
|
||||
"""
|
||||
Convert string values "inf", "-inf", and "nan" to their float equivalents.
|
||||
"""Convert string values "inf", "-inf", and "nan" to their float equivalents.
|
||||
|
||||
This handles the case where JSON serialization represents inf/nan as strings.
|
||||
"""
|
||||
|
||||
@@ -195,8 +195,7 @@ def run_vllm_chat(
|
||||
warmup_requests: list[SampleRequest] | None = None,
|
||||
prequeue_requests: bool = False,
|
||||
) -> tuple[float, list[RequestOutput]]:
|
||||
"""
|
||||
Run vLLM chat benchmark. This function is recommended ONLY for benchmarking
|
||||
"""Run vLLM chat benchmark. This function is recommended ONLY for benchmarking
|
||||
multimodal models as it properly handles multimodal inputs and chat
|
||||
formatting. For non-multimodal models, use run_vllm() instead.
|
||||
"""
|
||||
@@ -536,6 +535,7 @@ def _to_serve_args(args: argparse.Namespace) -> argparse.Namespace:
|
||||
|
||||
Returns:
|
||||
A namespace satisfying get_samples's attribute reads.
|
||||
|
||||
"""
|
||||
d = vars(args).copy()
|
||||
# random_*: prefer --random-* over legacy --input/output/prefix-len.
|
||||
@@ -627,10 +627,7 @@ def assign_loras(requests, args):
|
||||
|
||||
|
||||
def validate_args(args):
|
||||
"""
|
||||
Validate command-line arguments.
|
||||
"""
|
||||
|
||||
"""Validate command-line arguments."""
|
||||
# === Deprecation and Defaulting ===
|
||||
if args.dataset is not None:
|
||||
warnings.warn(
|
||||
|
||||
Reference in New Issue
Block a user