Adding deterministic lora benchmarking to vLLM Bench (#36057)

Signed-off-by: Ubuntu <ubuntu@ip-172-31-43-201.ap-northeast-1.compute.internal>
Signed-off-by: Ronald Xu <ronaldxu@amazon.com>
This commit is contained in:
RonaldBXu
2026-03-18 16:02:03 +00:00
committed by GitHub
parent b1169d7be8
commit c9d838fc33
3 changed files with 122 additions and 9 deletions
+84 -4
View File
@@ -183,6 +183,68 @@ class BenchmarkDataset(ABC):
)
return lora_request
def get_round_robin_lora_request(
self,
index: int,
max_loras: int | None = None,
lora_path: str | None = None,
) -> LoRARequest | None:
"""
Optionally select a LoRA request using deterministic round-robin.
This method cycles through LoRA IDs in order based on the request
index, providing reproducible LoRA assignment.
Args:
index (int): The request index used for round-robin selection.
max_loras (Optional[int]): The maximum number of LoRAs available.
If `None`, LoRA is not used.
lora_path (Optional[str]): Path to the LoRA parameters on disk.
If `None`, LoRA is not used.
Returns:
A new [`LoRARequest`][vllm.lora.request.LoRARequest]
(or `None` if not applicable).
"""
if max_loras is None or lora_path is None:
return None
# Deterministic round-robin: cycle through [1, max_loras]
lora_id = index % max_loras + 1
lora_request = LoRARequest(
lora_name=str(lora_id),
lora_int_id=lora_id,
lora_path=lora_path_on_disk(lora_path),
)
return lora_request
def get_lora_request(
self,
index: int,
max_loras: int | None = None,
lora_path: str | None = None,
lora_assignment: str = "random",
) -> LoRARequest | None:
"""
Select a LoRA request using the specified assignment strategy.
Args:
index (int): The request index (used for round-robin).
max_loras (Optional[int]): The maximum number of LoRAs available.
lora_path (Optional[str]): Path to the LoRA parameters on disk.
lora_assignment (str): Strategy for LoRA selection.
'random' (default) or 'round-robin'.
Returns:
A new [`LoRARequest`][vllm.lora.request.LoRARequest]
(or `None` if not applicable).
"""
if lora_assignment == "round-robin":
return self.get_round_robin_lora_request(
index=index, max_loras=max_loras, lora_path=lora_path
)
return self.get_random_lora_request(max_loras=max_loras, lora_path=lora_path)
@abstractmethod
def sample(
self,
@@ -478,6 +540,9 @@ class RandomDataset(BenchmarkDataset):
input_len: int = DEFAULT_INPUT_LEN,
output_len: int = DEFAULT_OUTPUT_LEN,
batchsize: int = 1,
max_loras: int | None = None,
lora_path: str | None = None,
lora_assignment: str = "random",
**kwargs,
) -> list[SampleRequest]:
# validate total input tokens (prefix + sampled) is at least 1.
@@ -522,11 +587,18 @@ class RandomDataset(BenchmarkDataset):
allowed_tokens=allowed_tokens,
)
token_mismatch_total += token_mismatch
lora_req = self.get_lora_request(
index=i,
max_loras=max_loras,
lora_path=lora_path,
lora_assignment=lora_assignment,
)
requests.append(
SampleRequest(
prompt=prompt,
prompt_len=total_input_len,
expected_output_len=int(output_lens[i]),
lora_request=lora_req,
request_id=request_id_prefix + str(i),
)
)
@@ -1263,6 +1335,7 @@ class ShareGPTDataset(BenchmarkDataset):
enable_multimodal_chat: bool = False,
request_id_prefix: str = "",
no_oversample: bool = False,
lora_assignment: str = "random",
**kwargs,
) -> list:
samples: list = []
@@ -1275,8 +1348,11 @@ class ShareGPTDataset(BenchmarkDataset):
entry["conversations"][1]["value"],
)
lora_request = self.get_random_lora_request(
max_loras=max_loras, lora_path=lora_path
lora_request = self.get_lora_request(
index=ind,
max_loras=max_loras,
lora_path=lora_path,
lora_assignment=lora_assignment,
)
prompt_ids = tokenizer(prompt).input_ids
completion_ids = tokenizer(completion).input_ids
@@ -2413,6 +2489,7 @@ class BurstGPTDataset(BenchmarkDataset):
lora_path: str | None = None,
request_id_prefix: str = "",
no_oversample: bool = False,
lora_assignment: str = "random",
**kwargs,
) -> list[SampleRequest]:
samples = []
@@ -2420,8 +2497,11 @@ class BurstGPTDataset(BenchmarkDataset):
for i in range(num_requests):
input_len = int(data[i][2])
output_len = int(data[i][3])
lora_req = self.get_random_lora_request(
max_loras=max_loras, lora_path=lora_path
lora_req = self.get_lora_request(
index=i,
max_loras=max_loras,
lora_path=lora_path,
lora_assignment=lora_assignment,
)
vocab_size = tokenizer.vocab_size
# Generate a synthetic prompt: a list of token IDs computed as (i +