[Deprecation] Deprecate sonet dataset as scheduled (#60129)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-10-06 08:15:12 -04:00
committed by GitHub
parent 3313825983
commit 3e182185aa
18 changed files with 22 additions and 1432 deletions
+1 -164
View File
@@ -5,7 +5,6 @@ datasets. Each dataset subclass of BenchmarkDataset must implement sample
generation. Supported dataset types include:
- ShareGPT
- Random (synthetic)
- Sonnet
- BurstGPT
- HuggingFace
- VisionArena
@@ -31,7 +30,6 @@ from typing import Any, cast
import numpy as np
import pybase64 as base64
from PIL import Image
from typing_extensions import deprecated
from vllm.benchmarks.datasets.utils import (
RangeRatio,
@@ -1598,7 +1596,6 @@ def add_dataset_parser(parser: FlexibleArgumentParser):
choices=[
"sharegpt",
"burstgpt",
"sonnet",
"random",
"random-mm",
"random-rerank",
@@ -1622,8 +1619,7 @@ def add_dataset_parser(parser: FlexibleArgumentParser):
"--dataset-path",
type=str,
default=None,
help="Path to the sharegpt/sonnet dataset or the HF dataset ID if "
"using HF dataset.",
help="Path to the ShareGPT dataset or the HF dataset ID if using HF dataset.",
)
parser.add_argument(
"--no-oversample",
@@ -1681,26 +1677,6 @@ def add_dataset_parser(parser: FlexibleArgumentParser):
help="Category for spec bench dataset. If None, use all categories.",
)
sonnet_group = parser.add_argument_group("sonnet dataset options")
sonnet_group.add_argument(
"--sonnet-input-len",
type=int,
default=550,
help="Number of input tokens per request, used only for sonnet dataset.",
)
sonnet_group.add_argument(
"--sonnet-output-len",
type=int,
default=150,
help="Number of output tokens per request, used only for sonnet dataset.",
)
sonnet_group.add_argument(
"--sonnet-prefix-len",
type=int,
default=200,
help="Number of prefix tokens per request, used only for sonnet dataset.",
)
sharegpt_group = parser.add_argument_group("sharegpt dataset options")
sharegpt_group.add_argument(
"--sharegpt-output-len",
@@ -2134,45 +2110,6 @@ def get_samples(
no_oversample=args.no_oversample,
)
elif args.dataset_name == "sonnet":
assert tokenizer is not None, (
"Tokenizer must be initialized for the 'sonnet' dataset."
)
sonnet_dataset = SonnetDataset(
dataset_path=args.dataset_path, disable_shuffle=args.disable_shuffle
)
# For the "sonnet" dataset, formatting depends on the backend.
# Chat-style backends leave templating to the server; completions
# backends need the prompt rendered client side.
if args.backend in ("openai-chat", "openai-responses"):
input_requests = sonnet_dataset.sample(
num_requests=args.num_prompts,
input_len=args.sonnet_input_len,
output_len=args.sonnet_output_len,
prefix_len=args.sonnet_prefix_len,
tokenizer=tokenizer,
return_prompt_formatted=False,
request_id_prefix=args.request_id_prefix,
no_oversample=args.no_oversample,
)
else:
assert (
hasattr(tokenizer, "chat_template") and tokenizer.chat_template
) or (
hasattr(tokenizer, "default_chat_template")
and tokenizer.default_chat_template
), "Tokenizer/model must have chat template for sonnet dataset."
input_requests = sonnet_dataset.sample(
num_requests=args.num_prompts,
input_len=args.sonnet_input_len,
output_len=args.sonnet_output_len,
prefix_len=args.sonnet_prefix_len,
tokenizer=tokenizer,
return_prompt_formatted=True,
request_id_prefix=args.request_id_prefix,
no_oversample=args.no_oversample,
)
elif args.dataset_name == "hf":
# all following datasets are implemented from the
# HuggingFaceDataset base class
@@ -3070,106 +3007,6 @@ class SpecBench(CustomDataset):
)
# -----------------------------------------------------------------------------
# Sonnet Dataset Implementation
# -----------------------------------------------------------------------------
@deprecated(
"SonnetDataset is deprecated and will be removed in a future version.",
)
class SonnetDataset(BenchmarkDataset):
"""Simplified implementation of the Sonnet dataset. Loads poem lines from a
text file and generates sample requests. Default values here copied from
`benchmark_serving.py` for the sonnet dataset.
"""
DEFAULT_PREFIX_LEN = 200
DEFAULT_INPUT_LEN = 550
DEFAULT_OUTPUT_LEN = 150
def __init__(
self,
**kwargs,
) -> None:
super().__init__(**kwargs)
self.load_data()
def load_data(self) -> None:
if not self.dataset_path:
raise ValueError("dataset_path must be provided.")
with open(self.dataset_path, encoding="utf-8") as f:
self.data = f.readlines()
def sample(
self,
tokenizer: TokenizerLike,
num_requests: int,
request_id_prefix: str = "",
no_oversample: bool = False,
prefix_len: int = DEFAULT_PREFIX_LEN,
input_len: int = DEFAULT_INPUT_LEN,
output_len: int = DEFAULT_OUTPUT_LEN,
return_prompt_formatted: bool = False,
**kwargs,
) -> list[SampleRequest]:
poem_lines = self.data
assert poem_lines is not None
# Calculate average token length for a poem line.
tokenized_lines = [tokenizer(line).input_ids for line in poem_lines]
avg_len = sum(len(tokens) for tokens in tokenized_lines) / len(tokenized_lines)
# Build the base prompt.
base_prompt = "Pick as many lines as you can from these poem lines:\n"
base_msg = [{"role": "user", "content": base_prompt}]
base_fmt = tokenizer.apply_chat_template(
base_msg, # type: ignore[arg-type]
add_generation_prompt=True,
tokenize=False,
)
assert isinstance(base_fmt, str)
base_offset = len(tokenizer(base_fmt).input_ids)
if input_len <= base_offset:
raise ValueError(
f"'input_len' must be higher than the base prompt length "
f"({base_offset})."
)
# Determine how many poem lines to use.
num_input_lines = max(round((input_len - base_offset) / avg_len), 1)
num_prefix_lines = max(round((prefix_len - base_offset) / avg_len), 0)
prefix_lines = poem_lines[:num_prefix_lines]
samples: list[SampleRequest] = []
ind = 0
while len(samples) < num_requests:
extra_lines = random.choices(
poem_lines, k=num_input_lines - num_prefix_lines
)
prompt = f"{base_prompt}{''.join(prefix_lines + extra_lines)}"
msg = [{"role": "user", "content": prompt}]
prompt_formatted = tokenizer.apply_chat_template(
msg, # type: ignore[arg-type]
add_generation_prompt=True,
tokenize=False,
)
assert isinstance(prompt_formatted, str)
prompt_len = len(tokenizer(prompt_formatted).input_ids)
if prompt_len <= input_len:
samples.append(
SampleRequest(
prompt=(
prompt_formatted if return_prompt_formatted else prompt
),
prompt_len=prompt_len,
expected_output_len=output_len,
request_id=request_id_prefix + str(ind),
)
)
ind += 1
return samples
# -----------------------------------------------------------------------------
# BurstGPT Dataset Implementation
# -----------------------------------------------------------------------------