Dynamic scheduler delay to improve ITL performance (#3279)

Co-authored-by: Jan van Lunteren <jvl@zurich.ibm.com>
This commit is contained in:
Thomas Parnell
2024-03-22 12:28:14 -07:00
committed by GitHub
co-authored by Jan van Lunteren
parent f721096d48
commit cf2f084d56
4 changed files with 72 additions and 2 deletions
+9 -1
View File
@@ -51,6 +51,7 @@ class EngineArgs:
max_cpu_loras: Optional[int] = None
device: str = 'auto'
ray_workers_use_nsight: bool = False
scheduler_delay_factor: float = 0.0
def __post_init__(self):
if self.tokenizer is None:
@@ -305,6 +306,12 @@ class EngineArgs:
default=EngineArgs.device,
choices=["auto", "cuda", "neuron"],
help='Device type for vLLM execution.')
parser.add_argument(
'--scheduler-delay-factor',
type=float,
default=EngineArgs.scheduler_delay_factor,
help='Apply a delay (of delay factor multiplied by previous'
'prompt latency) before scheduling next prompt.')
return parser
@classmethod
@@ -342,7 +349,8 @@ class EngineArgs:
), self.ray_workers_use_nsight)
scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
self.max_num_seqs,
model_config.max_model_len)
model_config.max_model_len,
self.scheduler_delay_factor)
lora_config = LoRAConfig(
max_lora_rank=self.max_lora_rank,
max_loras=self.max_loras,