feat: add TxtSlicesDataset to allow sampling slices from txt file for benchmarking (#30156)
Signed-off-by: jdebache <jdebache@nvidia.com>
This commit is contained in:
@@ -0,0 +1,84 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.benchmarks.datasets.datasets import (
|
||||
DEFAULT_NUM_PROMPTS,
|
||||
AIMODataset,
|
||||
ASRDataset,
|
||||
BenchmarkDataset,
|
||||
BlazeditDataset,
|
||||
BurstGPTDataset,
|
||||
ConversationDataset,
|
||||
CustomDataset,
|
||||
CustomMMDataset,
|
||||
HuggingFaceDataset,
|
||||
InstructCoderDataset,
|
||||
MLPerfDataset,
|
||||
MMStarDataset,
|
||||
MMVUDataset,
|
||||
MTBenchDataset,
|
||||
MultiModalConversationDataset,
|
||||
NextEditPredictionDataset,
|
||||
PrefixRepetitionRandomDataset,
|
||||
RandomDataset,
|
||||
RandomDatasetForReranking,
|
||||
RandomMultiModalDataset,
|
||||
SampleRequest,
|
||||
ShareGPTDataset,
|
||||
SonnetDataset,
|
||||
SpecBench,
|
||||
VisionArenaDataset,
|
||||
add_dataset_parser,
|
||||
add_random_dataset_base_args,
|
||||
add_random_multimodal_dataset_args,
|
||||
gen_prompt_decode_to_target_len,
|
||||
get_samples,
|
||||
is_valid_sequence,
|
||||
lora_path_on_disk,
|
||||
lora_tokenizer_cache,
|
||||
process_image,
|
||||
process_video,
|
||||
zeta_prompt,
|
||||
)
|
||||
from vllm.benchmarks.datasets.utils import RangeRatio
|
||||
|
||||
__all__ = [
|
||||
"DEFAULT_NUM_PROMPTS",
|
||||
"AIMODataset",
|
||||
"ASRDataset",
|
||||
"BenchmarkDataset",
|
||||
"BlazeditDataset",
|
||||
"BurstGPTDataset",
|
||||
"ConversationDataset",
|
||||
"CustomDataset",
|
||||
"CustomMMDataset",
|
||||
"HuggingFaceDataset",
|
||||
"InstructCoderDataset",
|
||||
"MLPerfDataset",
|
||||
"MMStarDataset",
|
||||
"MMVUDataset",
|
||||
"MTBenchDataset",
|
||||
"MultiModalConversationDataset",
|
||||
"NextEditPredictionDataset",
|
||||
"PrefixRepetitionRandomDataset",
|
||||
"RandomDataset",
|
||||
"RandomDatasetForReranking",
|
||||
"RandomMultiModalDataset",
|
||||
"SampleRequest",
|
||||
"ShareGPTDataset",
|
||||
"SonnetDataset",
|
||||
"SpecBench",
|
||||
"VisionArenaDataset",
|
||||
"add_dataset_parser",
|
||||
"add_random_dataset_base_args",
|
||||
"add_random_multimodal_dataset_args",
|
||||
"gen_prompt_decode_to_target_len",
|
||||
"get_samples",
|
||||
"is_valid_sequence",
|
||||
"lora_path_on_disk",
|
||||
"lora_tokenizer_cache",
|
||||
"process_image",
|
||||
"process_video",
|
||||
"RangeRatio",
|
||||
"zeta_prompt",
|
||||
]
|
||||
@@ -0,0 +1,209 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Convert a plain-text file (local path or URL) into a JSONL dataset
|
||||
compatible with ``CustomDataset`` (``--dataset-name custom``), by
|
||||
randomly slicing the tokenized text into prompts.
|
||||
|
||||
Each line of the output JSONL contains a ``prompt`` (decoded from a random
|
||||
slice of the tokenized source text) and an ``output_tokens`` count.
|
||||
|
||||
Usage
|
||||
-----
|
||||
::
|
||||
|
||||
python -m vllm.benchmarks.datasets.create_txt_slices_dataset \\
|
||||
--input sonnet.txt \\
|
||||
--output sonnet_dataset.jsonl \\
|
||||
--tokenizer gpt2 \\
|
||||
--num-prompts 1000 \\
|
||||
--input-len 1024 \\
|
||||
--output-len 128
|
||||
|
||||
The resulting JSONL file can then be used with the serving benchmark::
|
||||
|
||||
python -m vllm.benchmarks.serve \\
|
||||
--dataset-name custom \\
|
||||
--dataset-path sonnet_dataset.jsonl \\
|
||||
...
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import random
|
||||
import urllib.request
|
||||
|
||||
import numpy as np
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from vllm.benchmarks.datasets.utils import RangeRatio, get_sampling_params
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def load_text(path: str) -> str:
|
||||
"""Load text from a local file or URL."""
|
||||
if path.startswith(("http://", "https://")):
|
||||
with urllib.request.urlopen(path) as response:
|
||||
return response.read().decode("utf-8")
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
def create_txt_slices_jsonl(
|
||||
*,
|
||||
input_path: str,
|
||||
output_path: str,
|
||||
tokenizer_name: str,
|
||||
num_prompts: int,
|
||||
input_len: int,
|
||||
output_len: int,
|
||||
range_ratio: RangeRatio = 0.0,
|
||||
seed: int = 0,
|
||||
trust_remote_code: bool = False,
|
||||
) -> None:
|
||||
"""Read *input_path*, slice it into prompts, and write JSONL to
|
||||
*output_path*."""
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
tokenizer_name, trust_remote_code=trust_remote_code
|
||||
)
|
||||
|
||||
text = load_text(input_path)
|
||||
if not text:
|
||||
raise ValueError("The text file is empty and cannot be sampled from.")
|
||||
|
||||
token_ids = tokenizer(text, add_special_tokens=False).input_ids
|
||||
if not token_ids:
|
||||
raise ValueError("Tokenizing the text produced zero tokens; cannot sample.")
|
||||
|
||||
rng_np = np.random.default_rng(seed)
|
||||
rng_py = random.Random(seed)
|
||||
|
||||
input_lens, output_lens, _ = get_sampling_params(
|
||||
rng_np,
|
||||
num_prompts,
|
||||
range_ratio,
|
||||
input_len,
|
||||
output_len,
|
||||
tokenizer,
|
||||
)
|
||||
|
||||
num_available_tokens = len(token_ids)
|
||||
|
||||
records: list[dict[str, object]] = []
|
||||
for i in range(num_prompts):
|
||||
req_input_len = int(input_lens[i])
|
||||
req_output_len = int(output_lens[i])
|
||||
|
||||
# Randomly select a start position and slice with cycling
|
||||
start_pos = rng_py.randint(0, num_available_tokens - 1)
|
||||
prompt_token_ids = [
|
||||
token_ids[(start_pos + j) % num_available_tokens]
|
||||
for j in range(req_input_len)
|
||||
]
|
||||
prompt = tokenizer.decode(prompt_token_ids, skip_special_tokens=False)
|
||||
|
||||
records.append({"prompt": prompt, "output_tokens": req_output_len})
|
||||
|
||||
with open(output_path, "w", encoding="utf-8") as f:
|
||||
for record in records:
|
||||
f.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||||
|
||||
logger.info(
|
||||
"Wrote %d prompts to %s",
|
||||
len(records),
|
||||
output_path,
|
||||
)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Convert a plain-text file into a JSONL dataset "
|
||||
"for CustomDataset (--dataset-name custom).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--input",
|
||||
required=True,
|
||||
help="Path or URL to the source text file.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
required=True,
|
||||
help="Path for the output JSONL file.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--tokenizer",
|
||||
required=True,
|
||||
help="HuggingFace tokenizer name or path.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num-prompts",
|
||||
type=int,
|
||||
default=1000,
|
||||
help="Number of prompt samples to generate (default: 1000).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--input-len",
|
||||
type=int,
|
||||
default=1024,
|
||||
help="Target number of input tokens per prompt (default: 1024).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-len",
|
||||
type=int,
|
||||
default=128,
|
||||
help="Target number of output tokens per prompt (default: 128).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--range-ratio",
|
||||
type=str,
|
||||
default="0.0",
|
||||
help="Range ratio for input/output length sampling (default: 0.0). "
|
||||
"A single float applies to both ISL and OSL. "
|
||||
'A JSON dict like \'{"input": 0.3, "output": 0.5}\' sets them '
|
||||
"independently. Values must be in [0, 1).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--seed",
|
||||
type=int,
|
||||
default=0,
|
||||
help="Random seed for reproducibility (default: 0).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--trust-remote-code",
|
||||
action="store_true",
|
||||
help="Trust remote code from HuggingFace.",
|
||||
)
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
# Parse --range-ratio: try float first, then JSON dict.
|
||||
range_ratio: RangeRatio
|
||||
try:
|
||||
range_ratio = float(args.range_ratio)
|
||||
except ValueError:
|
||||
import json as _json
|
||||
|
||||
range_ratio = _json.loads(args.range_ratio)
|
||||
|
||||
create_txt_slices_jsonl(
|
||||
input_path=args.input,
|
||||
output_path=args.output,
|
||||
tokenizer_name=args.tokenizer,
|
||||
num_prompts=args.num_prompts,
|
||||
input_len=args.input_len,
|
||||
output_len=args.output_len,
|
||||
range_ratio=range_ratio,
|
||||
seed=args.seed,
|
||||
trust_remote_code=args.trust_remote_code,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,101 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Shared utilities for benchmark dataset sampling.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import math
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Type alias: a single float applies to both ISL and OSL; a dict allows
|
||||
# specifying them independently via ``{"input": …, "output": …}``.
|
||||
RangeRatio = float | dict[str, float]
|
||||
|
||||
|
||||
def _resolve_range_ratios(
|
||||
range_ratio: RangeRatio,
|
||||
) -> tuple[float, float]:
|
||||
"""Return ``(input_range_ratio, output_range_ratio)`` from *range_ratio*.
|
||||
|
||||
*range_ratio* is either a single float (used for both input and output)
|
||||
or a dict with ``"input"`` and ``"output"`` keys.
|
||||
"""
|
||||
if isinstance(range_ratio, dict):
|
||||
try:
|
||||
return float(range_ratio["input"]), float(range_ratio["output"])
|
||||
except KeyError as exc:
|
||||
raise ValueError(
|
||||
"When range_ratio is a dict it must contain 'input' and "
|
||||
f"'output' keys, got: {sorted(range_ratio)}"
|
||||
) from exc
|
||||
ratio = float(range_ratio)
|
||||
return ratio, ratio
|
||||
|
||||
|
||||
def get_sampling_params(
|
||||
rng: np.random.Generator,
|
||||
num_requests: int,
|
||||
range_ratio: RangeRatio,
|
||||
input_len: int,
|
||||
output_len: int,
|
||||
tokenizer: TokenizerLike,
|
||||
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
|
||||
"""
|
||||
Sample per-request input/output token lengths and vocab offsets.
|
||||
|
||||
Lengths are drawn uniformly from integer ranges around the configured
|
||||
means, controlled by *range_ratio*. It may be a single ``float``
|
||||
(applied to both input and output) or a ``dict`` with ``"input"`` and
|
||||
``"output"`` keys for independent control.
|
||||
|
||||
Tokenizer special tokens are subtracted from ``input_len`` before
|
||||
computing the sampling interval.
|
||||
|
||||
Returns:
|
||||
(input_lens, output_lens, offsets) – three 1-D ``np.ndarray`` of
|
||||
shape ``(num_requests,)``.
|
||||
"""
|
||||
input_range_ratio, output_range_ratio = _resolve_range_ratios(range_ratio)
|
||||
|
||||
if not (0.0 <= input_range_ratio < 1.0):
|
||||
raise ValueError("input_range_ratio must be in [0, 1).")
|
||||
if not (0.0 <= output_range_ratio < 1.0):
|
||||
raise ValueError("output_range_ratio must be in [0, 1).")
|
||||
num_special_tokens = int(tokenizer.num_special_tokens_to_add())
|
||||
real_input_len = max(0, int(input_len) - num_special_tokens)
|
||||
input_low = math.floor(real_input_len * (1 - input_range_ratio))
|
||||
input_high = math.ceil(real_input_len * (1 + input_range_ratio))
|
||||
output_low = math.floor(output_len * (1 - output_range_ratio))
|
||||
output_high = math.ceil(output_len * (1 + output_range_ratio))
|
||||
# Ensure the lower bound for output length is at least 1 to
|
||||
# prevent sampling 0 tokens.
|
||||
output_low = max(output_low, 1)
|
||||
output_high = max(output_high, 1)
|
||||
|
||||
if input_low > input_high:
|
||||
raise ValueError(
|
||||
f"Invalid input sampling interval: low={input_low} > high={input_high}"
|
||||
)
|
||||
if output_low > output_high:
|
||||
raise ValueError(
|
||||
f"Invalid output sampling interval: low={output_low} > high={output_high}"
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Sampling input_len from [%s, %s] and output_len from [%s, %s]",
|
||||
input_low,
|
||||
input_high,
|
||||
output_low,
|
||||
output_high,
|
||||
)
|
||||
|
||||
input_lens = rng.integers(input_low, input_high + 1, size=num_requests)
|
||||
output_lens = rng.integers(output_low, output_high + 1, size=num_requests)
|
||||
offsets = rng.integers(0, tokenizer.vocab_size, size=num_requests)
|
||||
return input_lens, output_lens, offsets
|
||||
Reference in New Issue
Block a user