Adding deterministic lora benchmarking to vLLM Bench (#36057)
Signed-off-by: Ubuntu <ubuntu@ip-172-31-43-201.ap-northeast-1.compute.internal> Signed-off-by: Ronald Xu <ronaldxu@amazon.com>
This commit is contained in:
@@ -183,6 +183,68 @@ class BenchmarkDataset(ABC):
|
||||
)
|
||||
return lora_request
|
||||
|
||||
def get_round_robin_lora_request(
|
||||
self,
|
||||
index: int,
|
||||
max_loras: int | None = None,
|
||||
lora_path: str | None = None,
|
||||
) -> LoRARequest | None:
|
||||
"""
|
||||
Optionally select a LoRA request using deterministic round-robin.
|
||||
|
||||
This method cycles through LoRA IDs in order based on the request
|
||||
index, providing reproducible LoRA assignment.
|
||||
|
||||
Args:
|
||||
index (int): The request index used for round-robin selection.
|
||||
max_loras (Optional[int]): The maximum number of LoRAs available.
|
||||
If `None`, LoRA is not used.
|
||||
lora_path (Optional[str]): Path to the LoRA parameters on disk.
|
||||
If `None`, LoRA is not used.
|
||||
|
||||
Returns:
|
||||
A new [`LoRARequest`][vllm.lora.request.LoRARequest]
|
||||
(or `None` if not applicable).
|
||||
"""
|
||||
if max_loras is None or lora_path is None:
|
||||
return None
|
||||
|
||||
# Deterministic round-robin: cycle through [1, max_loras]
|
||||
lora_id = index % max_loras + 1
|
||||
lora_request = LoRARequest(
|
||||
lora_name=str(lora_id),
|
||||
lora_int_id=lora_id,
|
||||
lora_path=lora_path_on_disk(lora_path),
|
||||
)
|
||||
return lora_request
|
||||
|
||||
def get_lora_request(
|
||||
self,
|
||||
index: int,
|
||||
max_loras: int | None = None,
|
||||
lora_path: str | None = None,
|
||||
lora_assignment: str = "random",
|
||||
) -> LoRARequest | None:
|
||||
"""
|
||||
Select a LoRA request using the specified assignment strategy.
|
||||
|
||||
Args:
|
||||
index (int): The request index (used for round-robin).
|
||||
max_loras (Optional[int]): The maximum number of LoRAs available.
|
||||
lora_path (Optional[str]): Path to the LoRA parameters on disk.
|
||||
lora_assignment (str): Strategy for LoRA selection.
|
||||
'random' (default) or 'round-robin'.
|
||||
|
||||
Returns:
|
||||
A new [`LoRARequest`][vllm.lora.request.LoRARequest]
|
||||
(or `None` if not applicable).
|
||||
"""
|
||||
if lora_assignment == "round-robin":
|
||||
return self.get_round_robin_lora_request(
|
||||
index=index, max_loras=max_loras, lora_path=lora_path
|
||||
)
|
||||
return self.get_random_lora_request(max_loras=max_loras, lora_path=lora_path)
|
||||
|
||||
@abstractmethod
|
||||
def sample(
|
||||
self,
|
||||
@@ -478,6 +540,9 @@ class RandomDataset(BenchmarkDataset):
|
||||
input_len: int = DEFAULT_INPUT_LEN,
|
||||
output_len: int = DEFAULT_OUTPUT_LEN,
|
||||
batchsize: int = 1,
|
||||
max_loras: int | None = None,
|
||||
lora_path: str | None = None,
|
||||
lora_assignment: str = "random",
|
||||
**kwargs,
|
||||
) -> list[SampleRequest]:
|
||||
# validate total input tokens (prefix + sampled) is at least 1.
|
||||
@@ -522,11 +587,18 @@ class RandomDataset(BenchmarkDataset):
|
||||
allowed_tokens=allowed_tokens,
|
||||
)
|
||||
token_mismatch_total += token_mismatch
|
||||
lora_req = self.get_lora_request(
|
||||
index=i,
|
||||
max_loras=max_loras,
|
||||
lora_path=lora_path,
|
||||
lora_assignment=lora_assignment,
|
||||
)
|
||||
requests.append(
|
||||
SampleRequest(
|
||||
prompt=prompt,
|
||||
prompt_len=total_input_len,
|
||||
expected_output_len=int(output_lens[i]),
|
||||
lora_request=lora_req,
|
||||
request_id=request_id_prefix + str(i),
|
||||
)
|
||||
)
|
||||
@@ -1263,6 +1335,7 @@ class ShareGPTDataset(BenchmarkDataset):
|
||||
enable_multimodal_chat: bool = False,
|
||||
request_id_prefix: str = "",
|
||||
no_oversample: bool = False,
|
||||
lora_assignment: str = "random",
|
||||
**kwargs,
|
||||
) -> list:
|
||||
samples: list = []
|
||||
@@ -1275,8 +1348,11 @@ class ShareGPTDataset(BenchmarkDataset):
|
||||
entry["conversations"][1]["value"],
|
||||
)
|
||||
|
||||
lora_request = self.get_random_lora_request(
|
||||
max_loras=max_loras, lora_path=lora_path
|
||||
lora_request = self.get_lora_request(
|
||||
index=ind,
|
||||
max_loras=max_loras,
|
||||
lora_path=lora_path,
|
||||
lora_assignment=lora_assignment,
|
||||
)
|
||||
prompt_ids = tokenizer(prompt).input_ids
|
||||
completion_ids = tokenizer(completion).input_ids
|
||||
@@ -2413,6 +2489,7 @@ class BurstGPTDataset(BenchmarkDataset):
|
||||
lora_path: str | None = None,
|
||||
request_id_prefix: str = "",
|
||||
no_oversample: bool = False,
|
||||
lora_assignment: str = "random",
|
||||
**kwargs,
|
||||
) -> list[SampleRequest]:
|
||||
samples = []
|
||||
@@ -2420,8 +2497,11 @@ class BurstGPTDataset(BenchmarkDataset):
|
||||
for i in range(num_requests):
|
||||
input_len = int(data[i][2])
|
||||
output_len = int(data[i][3])
|
||||
lora_req = self.get_random_lora_request(
|
||||
max_loras=max_loras, lora_path=lora_path
|
||||
lora_req = self.get_lora_request(
|
||||
index=i,
|
||||
max_loras=max_loras,
|
||||
lora_path=lora_path,
|
||||
lora_assignment=lora_assignment,
|
||||
)
|
||||
vocab_size = tokenizer.vocab_size
|
||||
# Generate a synthetic prompt: a list of token IDs computed as (i +
|
||||
|
||||
Reference in New Issue
Block a user