[Core] Add prefill step cadence for better non-PD DP balancing (#44558)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -599,6 +599,7 @@ class EngineArgs:
|
||||
disable_chunked_mm_input: bool = SchedulerConfig.disable_chunked_mm_input
|
||||
|
||||
scheduler_reserve_full_isl: bool = SchedulerConfig.scheduler_reserve_full_isl
|
||||
prefill_schedule_interval: int = SchedulerConfig.prefill_schedule_interval
|
||||
|
||||
watermark: float = SchedulerConfig.watermark
|
||||
|
||||
@@ -1417,6 +1418,10 @@ class EngineArgs:
|
||||
**scheduler_kwargs["scheduler_reserve_full_isl"],
|
||||
)
|
||||
scheduler_group.add_argument("--watermark", **scheduler_kwargs["watermark"])
|
||||
scheduler_group.add_argument(
|
||||
"--prefill-schedule-interval",
|
||||
**scheduler_kwargs["prefill_schedule_interval"],
|
||||
)
|
||||
scheduler_group.add_argument(
|
||||
"--disable-hybrid-kv-cache-manager",
|
||||
**scheduler_kwargs["disable_hybrid_kv_cache_manager"],
|
||||
@@ -2064,6 +2069,7 @@ class EngineArgs:
|
||||
long_prefill_token_threshold=self.long_prefill_token_threshold,
|
||||
scheduler_reserve_full_isl=self.scheduler_reserve_full_isl,
|
||||
watermark=self.watermark,
|
||||
prefill_schedule_interval=self.prefill_schedule_interval,
|
||||
disable_hybrid_kv_cache_manager=self.disable_hybrid_kv_cache_manager,
|
||||
async_scheduling=self.async_scheduling,
|
||||
stream_interval=self.stream_interval,
|
||||
|
||||
Reference in New Issue
Block a user