Adding deterministic lora benchmarking to vLLM Bench (#36057)

Signed-off-by: Ubuntu <ubuntu@ip-172-31-43-201.ap-northeast-1.compute.internal>
Signed-off-by: Ronald Xu <ronaldxu@amazon.com>
This commit is contained in:
RonaldBXu
2026-03-18 16:02:03 +00:00
committed by GitHub
parent b1169d7be8
commit c9d838fc33
3 changed files with 122 additions and 9 deletions
+10
View File
@@ -350,6 +350,7 @@ def get_requests(args, tokenizer):
"tokenizer": tokenizer,
"lora_path": args.lora_path,
"max_loras": args.max_loras,
"lora_assignment": getattr(args, "lora_assignment", "random"),
"num_requests": args.num_prompts,
}
@@ -778,6 +779,15 @@ def add_cli_args(parser: argparse.ArgumentParser):
help="Path to the lora adapters to use. This can be an absolute path, "
"a relative path, or a Hugging Face model identifier.",
)
parser.add_argument(
"--lora-assignment",
type=str,
default="random",
choices=["random", "round-robin"],
help="Strategy for assigning LoRA adapters to requests. "
"'random' (default) selects a LoRA at random for each request. "
"'round-robin' cycles through LoRAs deterministically.",
)
parser.add_argument(
"--prefix-len",
type=int,