[MyPy] Fix mypy for vllm/benchmarks (#39896)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Harry Mellor
parent
547d2c40d7
commit
275b43183c
@@ -81,7 +81,7 @@ class SampleRequest:
|
||||
|
||||
prompt: str | list[str] | list[dict]
|
||||
prompt_len: int
|
||||
expected_output_len: int | None
|
||||
expected_output_len: int = 0
|
||||
multi_modal_data: MultiModalDataDict | dict | list[dict] | None = None
|
||||
lora_request: LoRARequest | None = None
|
||||
request_id: str | None = None
|
||||
@@ -634,7 +634,7 @@ class RandomDataset(BenchmarkDataset):
|
||||
# Generate prefix once
|
||||
prefix_token_ids = self.get_prefix(tokenizer, allowed_tokens, prefix_len)
|
||||
|
||||
requests = []
|
||||
requests: list[SampleRequest] = []
|
||||
token_mismatch_total = 0
|
||||
for i in range(num_requests):
|
||||
prompt, total_input_len, token_mismatch = self.generate_token_sequence( # noqa: E501
|
||||
@@ -665,15 +665,14 @@ class RandomDataset(BenchmarkDataset):
|
||||
)
|
||||
# only used for embeddings benchmark.
|
||||
if batchsize > 1:
|
||||
batch_requests = []
|
||||
batch_requests: list[SampleRequest] = []
|
||||
# Create batched requests
|
||||
for i in range(0, num_requests, batchsize):
|
||||
batch = requests[i : i + batchsize]
|
||||
batch_requests.append(
|
||||
SampleRequest(
|
||||
prompt=[req.prompt for req in batch],
|
||||
prompt=[req.prompt for req in batch], # type: ignore[arg-type]
|
||||
prompt_len=sum(req.prompt_len for req in batch),
|
||||
expected_output_len=0,
|
||||
request_id=request_id_prefix + str(i // batchsize),
|
||||
)
|
||||
)
|
||||
@@ -797,6 +796,9 @@ class RandomDatasetForReranking(RandomDataset):
|
||||
input_len: int = RandomDataset.DEFAULT_INPUT_LEN,
|
||||
output_len: int = RandomDataset.DEFAULT_OUTPUT_LEN,
|
||||
batchsize: int = 1,
|
||||
max_loras: int | None = None,
|
||||
lora_path: str | None = None,
|
||||
lora_assignment: str = "random",
|
||||
is_reranker: bool = True,
|
||||
**kwargs,
|
||||
) -> list[SampleRequest]:
|
||||
@@ -1207,6 +1209,9 @@ class RandomMultiModalDataset(RandomDataset):
|
||||
input_len: int = RandomDataset.DEFAULT_INPUT_LEN,
|
||||
output_len: int = RandomDataset.DEFAULT_OUTPUT_LEN,
|
||||
batchsize: int = 1,
|
||||
max_loras: int | None = None,
|
||||
lora_path: str | None = None,
|
||||
lora_assignment: str = "random",
|
||||
limit_mm_per_prompt: dict[str, int] = DEFAULT_LIMIT_MM_PER_PROMPT,
|
||||
base_items_per_request: int = DEFAULT_BASE_ITEMS_PER_REQUEST,
|
||||
num_mm_items_range_ratio: float = DEFAULT_NUM_MM_ITEMS_RANGE_RATIO,
|
||||
@@ -1251,6 +1256,7 @@ class RandomMultiModalDataset(RandomDataset):
|
||||
# We want to exclude placeholder tokens and all
|
||||
# tokens that indicate start/end of image as it
|
||||
# may break prompt replacement logic.
|
||||
assert hasattr(tokenizer, "added_tokens_decoder")
|
||||
prohibited_tokens = list(
|
||||
tok_id
|
||||
for tok_id, token in tokenizer.added_tokens_decoder.items()
|
||||
@@ -1376,6 +1382,7 @@ class ShareGPTDataset(BenchmarkDataset):
|
||||
lora_assignment: str = "random",
|
||||
**kwargs,
|
||||
) -> list[SampleRequest]:
|
||||
assert self.data is not None, "Dataset must be loaded before sampling"
|
||||
samples: list[SampleRequest] = []
|
||||
ind = 0
|
||||
for entry in self.data:
|
||||
@@ -1454,7 +1461,7 @@ class TimedTrace(BenchmarkDataset):
|
||||
f'label_output_length: "{self.label_output_length}", '
|
||||
f'label_hash_ids: "{self.label_hash_ids}"'
|
||||
)
|
||||
self._expanded_generated_prompts = {}
|
||||
self._expanded_generated_prompts: dict[str, Any] = {}
|
||||
self.load_data()
|
||||
|
||||
def load_data(self) -> None:
|
||||
@@ -1529,10 +1536,12 @@ class TimedTrace(BenchmarkDataset):
|
||||
tokenizer: TokenizerLike,
|
||||
num_requests: int,
|
||||
request_id_prefix: str = "",
|
||||
no_oversample: bool = False,
|
||||
**kwargs,
|
||||
) -> list:
|
||||
samples: list = []
|
||||
) -> list[SampleRequest]:
|
||||
samples: list[SampleRequest] = []
|
||||
assert tokenizer is not None, "Tokenizer must be provided, now is Null"
|
||||
assert self.data is not None, "Data must be loaded before sampling"
|
||||
|
||||
for ind, entry in enumerate(self.data):
|
||||
if len(samples) >= num_requests:
|
||||
@@ -2139,12 +2148,12 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
|
||||
)
|
||||
|
||||
elif args.dataset_name == "sonnet":
|
||||
dataset = SonnetDataset(
|
||||
sonnet_dataset = SonnetDataset(
|
||||
dataset_path=args.dataset_path, disable_shuffle=args.disable_shuffle
|
||||
)
|
||||
# For the "sonnet" dataset, formatting depends on the backend.
|
||||
if args.backend == "openai-chat":
|
||||
input_requests = dataset.sample(
|
||||
input_requests = sonnet_dataset.sample(
|
||||
num_requests=args.num_prompts,
|
||||
input_len=args.sonnet_input_len,
|
||||
output_len=args.sonnet_output_len,
|
||||
@@ -2155,10 +2164,13 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
|
||||
no_oversample=args.no_oversample,
|
||||
)
|
||||
else:
|
||||
assert tokenizer.chat_template or tokenizer.default_chat_template, (
|
||||
"Tokenizer/model must have chat template for sonnet dataset."
|
||||
)
|
||||
input_requests = dataset.sample(
|
||||
assert (
|
||||
hasattr(tokenizer, "chat_template") and tokenizer.chat_template
|
||||
) or (
|
||||
hasattr(tokenizer, "default_chat_template")
|
||||
and tokenizer.default_chat_template
|
||||
), "Tokenizer/model must have chat template for sonnet dataset."
|
||||
input_requests = sonnet_dataset.sample(
|
||||
num_requests=args.num_prompts,
|
||||
input_len=args.sonnet_input_len,
|
||||
output_len=args.sonnet_output_len,
|
||||
@@ -2173,6 +2185,7 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
|
||||
# all following datasets are implemented from the
|
||||
# HuggingFaceDataset base class
|
||||
hf_kwargs = {}
|
||||
dataset_class: type[BenchmarkDataset]
|
||||
if (
|
||||
args.dataset_path in VisionArenaDataset.SUPPORTED_DATASET_PATHS
|
||||
or args.hf_name in VisionArenaDataset.SUPPORTED_DATASET_PATHS
|
||||
@@ -2548,23 +2561,23 @@ class CustomDataset(BenchmarkDataset):
|
||||
|
||||
if tokenizer is None:
|
||||
new_output_len = 1
|
||||
elif output_len is None or output_len == -1:
|
||||
# check that the request has an 'output_tokens' field
|
||||
if "output_tokens" not in item:
|
||||
raise ValueError(
|
||||
"If no output length is provided the "
|
||||
"custom dataset must contain an 'output_tokens' field."
|
||||
)
|
||||
# Use number of output tokens from the request data
|
||||
try:
|
||||
new_output_len = int(item["output_tokens"])
|
||||
except (ValueError, TypeError) as e:
|
||||
raise ValueError(
|
||||
f"Invalid value for 'output_tokens' in custom dataset: "
|
||||
f"'{item['output_tokens']}'. Must be an integer."
|
||||
) from e
|
||||
else:
|
||||
new_output_len = output_len
|
||||
if output_len is None or output_len == -1:
|
||||
# check that the request has an 'output_tokens' field
|
||||
if "output_tokens" not in item:
|
||||
raise ValueError(
|
||||
"If no output length is provided the "
|
||||
"custom dataset must contain an 'output_tokens' field."
|
||||
)
|
||||
# Use number of output tokens from the request data
|
||||
try:
|
||||
new_output_len = int(item["output_tokens"])
|
||||
except (ValueError, TypeError) as e:
|
||||
raise ValueError(
|
||||
f"Invalid value for 'output_tokens' in custom dataset: "
|
||||
f"'{item['output_tokens']}'. Must be an integer."
|
||||
) from e
|
||||
|
||||
if tokenizer is None:
|
||||
prompt_len = 1
|
||||
@@ -2797,11 +2810,15 @@ class CustomImageDataset(CustomDataset):
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
num_requests: int,
|
||||
output_len: int | None = None,
|
||||
enable_multimodal_chat: bool = False,
|
||||
ensure_client_side_data: bool = False,
|
||||
request_id_prefix: str = "",
|
||||
no_oversample: bool = False,
|
||||
lora_path: str | None = None,
|
||||
max_loras: int | None = None,
|
||||
output_len: int | None = None,
|
||||
enable_multimodal_chat: bool = False,
|
||||
skip_chat_template: bool = False,
|
||||
chat_template_kwargs: dict | None = None,
|
||||
ensure_client_side_data: bool = False,
|
||||
**kwargs,
|
||||
) -> list[SampleRequest]:
|
||||
# load all data if needed
|
||||
@@ -2814,7 +2831,7 @@ class CustomImageDataset(CustomDataset):
|
||||
num_requests,
|
||||
)
|
||||
|
||||
sampled_requests = []
|
||||
sampled_requests: list[SampleRequest] = []
|
||||
for i, item in enumerate(self.data):
|
||||
if len(sampled_requests) >= num_requests:
|
||||
break
|
||||
@@ -2837,7 +2854,7 @@ class CustomImageDataset(CustomDataset):
|
||||
SampleRequest(
|
||||
prompt=prompt,
|
||||
prompt_len=prompt_len,
|
||||
expected_output_len=output_len,
|
||||
expected_output_len=output_len or 0,
|
||||
multi_modal_data=None,
|
||||
request_id=request_id_prefix + str(i),
|
||||
)
|
||||
@@ -2863,7 +2880,7 @@ class CustomImageDataset(CustomDataset):
|
||||
SampleRequest(
|
||||
prompt=prompt,
|
||||
prompt_len=prompt_len,
|
||||
expected_output_len=output_len,
|
||||
expected_output_len=output_len or 0,
|
||||
multi_modal_data=mm_content,
|
||||
request_id=request_id_prefix + str(i),
|
||||
)
|
||||
@@ -2891,17 +2908,20 @@ class CustomAudioDataset(CustomDataset):
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
num_requests: int,
|
||||
output_len: int | None = None,
|
||||
request_id_prefix: str = "",
|
||||
no_oversample: bool = False,
|
||||
skip_chat_template: bool = False,
|
||||
lora_path: str | None = None,
|
||||
max_loras: int | None = None,
|
||||
output_len: int | None = None,
|
||||
enable_multimodal_chat: bool = False,
|
||||
skip_chat_template: bool = False,
|
||||
chat_template_kwargs: dict | None = None,
|
||||
**kwargs,
|
||||
) -> list[SampleRequest]:
|
||||
self.num_available_samples = len(self.data)
|
||||
if num_requests <= 0:
|
||||
num_requests = self.num_available_samples
|
||||
sampled_requests = []
|
||||
sampled_requests: list[SampleRequest] = []
|
||||
for i, item in enumerate(self.data):
|
||||
if len(sampled_requests) >= num_requests:
|
||||
break
|
||||
@@ -2950,7 +2970,6 @@ class CustomAudioDataset(CustomDataset):
|
||||
prompt_len = (
|
||||
len(tokenizer(prompt).input_ids) if isinstance(prompt, str) else 1
|
||||
)
|
||||
new_output_len = output_len
|
||||
if output_len is None or output_len == -1:
|
||||
if "output_tokens" not in item:
|
||||
raise ValueError(
|
||||
@@ -2958,6 +2977,8 @@ class CustomAudioDataset(CustomDataset):
|
||||
"custom dataset must contain an 'output_tokens' field."
|
||||
)
|
||||
new_output_len = int(item["output_tokens"])
|
||||
else:
|
||||
new_output_len = output_len
|
||||
sampled_requests.append(
|
||||
SampleRequest(
|
||||
prompt=prompt,
|
||||
@@ -3015,10 +3036,30 @@ class SpecBench(CustomDataset):
|
||||
|
||||
def sample(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
num_requests: int,
|
||||
request_id_prefix: str = "",
|
||||
no_oversample: bool = False,
|
||||
lora_path: str | None = None,
|
||||
max_loras: int | None = None,
|
||||
output_len: int | None = None,
|
||||
enable_multimodal_chat: bool = False,
|
||||
skip_chat_template: bool = False,
|
||||
chat_template_kwargs: dict | None = None,
|
||||
**kwargs,
|
||||
) -> list[SampleRequest]:
|
||||
# leverage CustomDataset sample
|
||||
return super().sample(
|
||||
tokenizer=tokenizer,
|
||||
num_requests=num_requests,
|
||||
request_id_prefix=request_id_prefix,
|
||||
no_oversample=no_oversample,
|
||||
lora_path=lora_path,
|
||||
max_loras=max_loras,
|
||||
output_len=output_len,
|
||||
enable_multimodal_chat=enable_multimodal_chat,
|
||||
skip_chat_template=skip_chat_template,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@@ -3067,16 +3108,21 @@ class SonnetDataset(BenchmarkDataset):
|
||||
return_prompt_formatted: bool = False,
|
||||
**kwargs,
|
||||
) -> list[SampleRequest]:
|
||||
poem_lines = self.data
|
||||
assert poem_lines is not None
|
||||
# Calculate average token length for a poem line.
|
||||
tokenized_lines = [tokenizer(line).input_ids for line in self.data]
|
||||
tokenized_lines = [tokenizer(line).input_ids for line in poem_lines]
|
||||
avg_len = sum(len(tokens) for tokens in tokenized_lines) / len(tokenized_lines)
|
||||
|
||||
# Build the base prompt.
|
||||
base_prompt = "Pick as many lines as you can from these poem lines:\n"
|
||||
base_msg = [{"role": "user", "content": base_prompt}]
|
||||
base_fmt = tokenizer.apply_chat_template(
|
||||
base_msg, add_generation_prompt=True, tokenize=False
|
||||
base_msg, # type: ignore[arg-type]
|
||||
add_generation_prompt=True,
|
||||
tokenize=False,
|
||||
)
|
||||
assert isinstance(base_fmt, str)
|
||||
base_offset = len(tokenizer(base_fmt).input_ids)
|
||||
if input_len <= base_offset:
|
||||
raise ValueError(
|
||||
@@ -3085,26 +3131,31 @@ class SonnetDataset(BenchmarkDataset):
|
||||
)
|
||||
|
||||
# Determine how many poem lines to use.
|
||||
num_input_lines = round((input_len - base_offset) / avg_len)
|
||||
num_input_lines = max(round((input_len - base_offset) / avg_len), 1)
|
||||
num_prefix_lines = max(round((prefix_len - base_offset) / avg_len), 0)
|
||||
prefix_lines = self.data[:num_prefix_lines]
|
||||
prefix_lines = poem_lines[:num_prefix_lines]
|
||||
|
||||
samples: list[SampleRequest] = []
|
||||
ind = 0
|
||||
while len(samples) < num_requests:
|
||||
extra_lines = random.choices(
|
||||
self.data, k=num_input_lines - num_prefix_lines
|
||||
poem_lines, k=num_input_lines - num_prefix_lines
|
||||
)
|
||||
prompt = f"{base_prompt}{''.join(prefix_lines + extra_lines)}"
|
||||
msg = [{"role": "user", "content": prompt}]
|
||||
prompt_formatted = tokenizer.apply_chat_template(
|
||||
msg, add_generation_prompt=True, tokenize=False
|
||||
msg, # type: ignore[arg-type]
|
||||
add_generation_prompt=True,
|
||||
tokenize=False,
|
||||
)
|
||||
assert isinstance(prompt_formatted, str)
|
||||
prompt_len = len(tokenizer(prompt_formatted).input_ids)
|
||||
if prompt_len <= input_len:
|
||||
samples.append(
|
||||
SampleRequest(
|
||||
prompt=prompt_formatted if return_prompt_formatted else prompt,
|
||||
prompt=(
|
||||
prompt_formatted if return_prompt_formatted else prompt
|
||||
),
|
||||
prompt_len=prompt_len,
|
||||
expected_output_len=output_len,
|
||||
request_id=request_id_prefix + str(ind),
|
||||
@@ -3145,6 +3196,7 @@ class BurstGPTDataset(BenchmarkDataset):
|
||||
self.data = gpt4_df
|
||||
|
||||
def _sample_loaded_data(self, num_requests: int) -> list:
|
||||
assert self.data is not None, "Dataset must be loaded before sampling"
|
||||
if num_requests <= len(self.data):
|
||||
data = self.data.sample(n=num_requests, random_state=self.random_seed)
|
||||
else:
|
||||
@@ -3258,6 +3310,7 @@ class ConversationDataset(HuggingFaceDataset):
|
||||
enable_multimodal_chat: bool = False,
|
||||
**kwargs,
|
||||
) -> list[SampleRequest]:
|
||||
assert self.data is not None, "Dataset must be loaded before sampling"
|
||||
# Filter examples with at least 2 conversations
|
||||
filtered_data = self.data.filter(lambda x: len(x["conversations"]) >= 2)
|
||||
sampled_requests: list[SampleRequest] = []
|
||||
@@ -3318,6 +3371,7 @@ class MultiModalConversationDataset(HuggingFaceDataset):
|
||||
enable_multimodal_chat: bool = False,
|
||||
**kwargs,
|
||||
) -> list[SampleRequest]:
|
||||
assert self.data is not None, "Dataset must be loaded before sampling"
|
||||
# Filter examples with at least 2 conversations
|
||||
filtered_data = self.data.filter(lambda x: len(x["conversations"]) >= 2)
|
||||
sampled_requests: list[SampleRequest] = []
|
||||
@@ -3393,7 +3447,8 @@ class VisionArenaDataset(HuggingFaceDataset):
|
||||
|
||||
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
|
||||
|
||||
sampled_requests = []
|
||||
assert self.data is not None, "Dataset must be loaded before sampling"
|
||||
sampled_requests: list[SampleRequest] = []
|
||||
for i, item in enumerate(self.data):
|
||||
if len(sampled_requests) >= num_requests:
|
||||
break
|
||||
@@ -3465,7 +3520,8 @@ class MMVUDataset(HuggingFaceDataset):
|
||||
|
||||
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
|
||||
|
||||
sampled_requests = []
|
||||
assert self.data is not None, "Dataset must be loaded before sampling"
|
||||
sampled_requests: list[SampleRequest] = []
|
||||
for i, item in enumerate(self.data):
|
||||
if len(sampled_requests) >= num_requests:
|
||||
break
|
||||
@@ -3531,18 +3587,21 @@ class InstructCoderDataset(HuggingFaceDataset):
|
||||
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
|
||||
sampled_requests: list[SampleRequest] = []
|
||||
for i, prompt in enumerate(self.sample_prompts(n=num_requests)):
|
||||
prompt_text = prompt
|
||||
# apply template
|
||||
if not skip_chat_template:
|
||||
prompt = tokenizer.apply_chat_template(
|
||||
prompt_text_result = tokenizer.apply_chat_template(
|
||||
[{"role": "user", "content": prompt}],
|
||||
add_generation_prompt=True,
|
||||
tokenize=False,
|
||||
)
|
||||
assert isinstance(prompt_text_result, str)
|
||||
prompt_text = prompt_text_result
|
||||
|
||||
prompt_len = len(tokenizer(prompt).input_ids)
|
||||
prompt_len = len(tokenizer(prompt_text).input_ids)
|
||||
sampled_requests.append(
|
||||
SampleRequest(
|
||||
prompt=prompt,
|
||||
prompt=prompt_text,
|
||||
prompt_len=prompt_len,
|
||||
expected_output_len=output_len,
|
||||
request_id=request_id_prefix + str(i),
|
||||
@@ -3554,6 +3613,7 @@ class InstructCoderDataset(HuggingFaceDataset):
|
||||
return sampled_requests
|
||||
|
||||
def sample_prompts(self, n: int) -> Iterator[str]:
|
||||
assert self.data is not None, "Dataset must be loaded before sampling"
|
||||
for item in self.data.take(n):
|
||||
prompt = (
|
||||
f"{item['input']}\n\n{item['instruction']} Just output "
|
||||
@@ -3594,6 +3654,7 @@ class MTBenchDataset(HuggingFaceDataset):
|
||||
**kwargs,
|
||||
) -> list[SampleRequest]:
|
||||
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
|
||||
assert self.data is not None, "Dataset must be loaded before sampling"
|
||||
sampled_requests: list[SampleRequest] = []
|
||||
|
||||
for i, item in enumerate(self.data):
|
||||
@@ -3654,7 +3715,8 @@ class HumanEvalDataset(HuggingFaceDataset):
|
||||
**kwargs,
|
||||
) -> list[SampleRequest]:
|
||||
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
|
||||
sampled_requests = []
|
||||
sampled_requests: list[SampleRequest] = []
|
||||
assert self.data is not None, "Data must be loaded before sampling"
|
||||
|
||||
for i, item in enumerate(self.data):
|
||||
if len(sampled_requests) >= num_requests:
|
||||
@@ -3714,7 +3776,8 @@ class GSM8KDataset(HuggingFaceDataset):
|
||||
**kwargs,
|
||||
) -> list[SampleRequest]:
|
||||
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
|
||||
sampled_requests = []
|
||||
sampled_requests: list[SampleRequest] = []
|
||||
assert self.data is not None, "Data must be loaded before sampling"
|
||||
|
||||
for i, item in enumerate(self.data):
|
||||
if len(sampled_requests) >= num_requests:
|
||||
@@ -3772,17 +3835,18 @@ class BlazeditDataset(HuggingFaceDataset):
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
num_requests: int,
|
||||
output_len: int | None = None,
|
||||
skip_chat_template: bool = False,
|
||||
request_id_prefix: str = "",
|
||||
no_oversample: bool = False,
|
||||
output_len: int | None = None,
|
||||
skip_chat_template: bool = False,
|
||||
min_distance: float = 0.0,
|
||||
max_distance: float = 1.0,
|
||||
**kwargs,
|
||||
) -> list[SampleRequest]:
|
||||
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
|
||||
sampled_requests = []
|
||||
sampled_requests: list[SampleRequest] = []
|
||||
|
||||
assert self.data is not None, "Dataset must be loaded before sampling"
|
||||
for i, item in enumerate(self.data):
|
||||
if len(sampled_requests) >= num_requests:
|
||||
break
|
||||
@@ -3810,11 +3874,13 @@ Please generate the new code file in the "New file" section below.""" # noqa: E
|
||||
|
||||
# apply template
|
||||
if not skip_chat_template:
|
||||
prompt = tokenizer.apply_chat_template(
|
||||
prompt_result = tokenizer.apply_chat_template(
|
||||
[{"role": "user", "content": prompt}],
|
||||
add_generation_prompt=True,
|
||||
tokenize=False,
|
||||
)
|
||||
assert isinstance(prompt_result, str)
|
||||
prompt = prompt_result
|
||||
|
||||
prompt_len = len(tokenizer(prompt).input_ids)
|
||||
|
||||
@@ -3858,6 +3924,7 @@ class AIMODataset(HuggingFaceDataset):
|
||||
output_len: int | None = None,
|
||||
**kwargs,
|
||||
) -> list[SampleRequest]:
|
||||
assert self.data is not None, "Dataset must be loaded before sampling"
|
||||
sampled_requests: list[SampleRequest] = []
|
||||
ind = 0
|
||||
dynamic_output = output_len is None
|
||||
@@ -3970,6 +4037,7 @@ class NextEditPredictionDataset(HuggingFaceDataset):
|
||||
formatting_prompt_func = self.MAPPING_PROMPT_FUNCS.get(self.hf_name)
|
||||
if formatting_prompt_func is None:
|
||||
raise ValueError(f"Unsupported dataset path: {self.hf_name}")
|
||||
assert self.data is not None, "Dataset must be loaded before sampling"
|
||||
samples = []
|
||||
for i, sample in enumerate(self.data):
|
||||
sample = formatting_prompt_func(sample)
|
||||
@@ -4061,12 +4129,14 @@ class ASRDataset(HuggingFaceDataset):
|
||||
def _disable_audio_decode(self) -> None:
|
||||
from datasets import Audio
|
||||
|
||||
assert self.data is not None, "Dataset must be loaded before sampling"
|
||||
self.data = self.data.cast_column("audio", Audio(decode=False))
|
||||
|
||||
def _materialize_local_audio_column(self) -> None:
|
||||
local_path_root = Path(
|
||||
hf_api().snapshot_download(self.hf_name, repo_type="dataset")
|
||||
)
|
||||
assert self.data is not None, "Dataset must be loaded before sampling"
|
||||
self.data = self.data.map(
|
||||
lambda item: {
|
||||
"audio": str(local_path_root / item["url"]),
|
||||
@@ -4090,15 +4160,19 @@ class ASRDataset(HuggingFaceDataset):
|
||||
else:
|
||||
prompt = ""
|
||||
prompt_len = len(tokenizer(prompt).input_ids)
|
||||
assert self.data is not None, "Dataset must be loaded before sampling"
|
||||
sampled_requests: list[SampleRequest] = []
|
||||
ind = 0
|
||||
skipped = 0
|
||||
asr_min_audio_len_sec = kwargs.get("asr_min_audio_len_sec")
|
||||
asr_max_audio_len_sec = kwargs.get("asr_max_audio_len_sec")
|
||||
asr_min_audio_len_sec: float = float(kwargs.get("asr_min_audio_len_sec") or 0.0)
|
||||
asr_max_audio_len_sec: float = float(
|
||||
kwargs.get("asr_max_audio_len_sec") or float("inf")
|
||||
)
|
||||
durations = []
|
||||
for item in self.data:
|
||||
if len(sampled_requests) >= num_requests:
|
||||
break
|
||||
mm_content: dict[str, Any]
|
||||
audio = item["audio"]
|
||||
if (
|
||||
isinstance(audio, dict)
|
||||
@@ -4203,8 +4277,7 @@ class MLPerfDataset(HuggingFaceDataset):
|
||||
output_len: int | None = None,
|
||||
**kwargs,
|
||||
) -> list[SampleRequest]:
|
||||
# Force dynamic output length based on reference completion.
|
||||
dynamic_output = output_len is None
|
||||
assert self.data is not None, "Dataset must be loaded before sampling"
|
||||
sampled_requests: list[SampleRequest] = []
|
||||
ind = 0
|
||||
|
||||
@@ -4222,15 +4295,18 @@ class MLPerfDataset(HuggingFaceDataset):
|
||||
{"role": "user", "content": question},
|
||||
]
|
||||
prompt_formatted = tokenizer.apply_chat_template(
|
||||
messages, add_generation_prompt=True, tokenize=False
|
||||
messages, # type: ignore[arg-type]
|
||||
add_generation_prompt=True,
|
||||
tokenize=False,
|
||||
)
|
||||
assert isinstance(prompt_formatted, str)
|
||||
prompt_len = len(tokenizer(prompt_formatted).input_ids)
|
||||
|
||||
# Determine output length from reference answer tokens.
|
||||
ref_out_len = len(
|
||||
tokenizer(reference_answer, add_special_tokens=False).input_ids
|
||||
)
|
||||
expected_output_len = ref_out_len if dynamic_output else output_len
|
||||
expected_output_len = ref_out_len if output_len is None else output_len
|
||||
|
||||
# Validate sequence lengths.
|
||||
if not is_valid_sequence(prompt_len, expected_output_len):
|
||||
@@ -4371,6 +4447,7 @@ class MMStarDataset(HuggingFaceDataset):
|
||||
) -> list[SampleRequest]:
|
||||
# If --hf-output-len is not set, use the default output length.
|
||||
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
|
||||
assert self.data is not None, "Dataset must be loaded before sampling"
|
||||
sampled_requests: list[SampleRequest] = []
|
||||
|
||||
for ind, item in enumerate(self.data):
|
||||
@@ -4603,7 +4680,7 @@ class BFCLDataset(HuggingFaceDataset):
|
||||
exc_info=True,
|
||||
)
|
||||
rendered = None
|
||||
if rendered is not None:
|
||||
if rendered is not None and isinstance(rendered, str):
|
||||
prompt_len = len(tokenizer(rendered).input_ids)
|
||||
else:
|
||||
text = "\n".join(m.get("content", "") for m in messages)
|
||||
|
||||
Reference in New Issue
Block a user