[Perf] Support stream interval for reducing host overhead (#27869)

Signed-off-by: elvischenv <219235043+elvischenv@users.noreply.github.com>
Co-authored-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
elvischenv
2025-11-13 13:21:25 -05:00
committed by GitHub
co-authored by Nick Hill
parent f9f3b596f3
commit 5d6ce2b960
6 changed files with 67 additions and 5 deletions
+6
View File
@@ -558,6 +558,8 @@ class EngineArgs:
async_scheduling: bool | None = SchedulerConfig.async_scheduling
stream_interval: int = SchedulerConfig.stream_interval
kv_sharing_fast_prefill: bool = CacheConfig.kv_sharing_fast_prefill
kv_offloading_size: float | None = CacheConfig.kv_offloading_size
@@ -1067,6 +1069,9 @@ class EngineArgs:
scheduler_group.add_argument(
"--async-scheduling", **scheduler_kwargs["async_scheduling"]
)
scheduler_group.add_argument(
"--stream-interval", **scheduler_kwargs["stream_interval"]
)
# Compilation arguments
compilation_kwargs = get_kwargs(CompilationConfig)
@@ -1562,6 +1567,7 @@ class EngineArgs:
long_prefill_token_threshold=self.long_prefill_token_threshold,
disable_hybrid_kv_cache_manager=self.disable_hybrid_kv_cache_manager,
async_scheduling=self.async_scheduling,
stream_interval=self.stream_interval,
)
if not model_config.is_multimodal_model and self.default_mm_loras: