[Pooling] Disable async scheduling by default for pooling models (#39592)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -764,6 +764,16 @@ class VllmConfig:
|
|||||||
elif self.scheduler_config.async_scheduling is None:
|
elif self.scheduler_config.async_scheduling is None:
|
||||||
# Enable async scheduling unless there is an incompatible option.
|
# Enable async scheduling unless there is an incompatible option.
|
||||||
if (
|
if (
|
||||||
|
self.model_config is not None
|
||||||
|
and self.model_config.runner_type == "pooling"
|
||||||
|
):
|
||||||
|
# The current implementation of asynchronous scheduling negatively
|
||||||
|
# impacts performance of pooling models, so we disable by default.
|
||||||
|
logger.debug(
|
||||||
|
"Disabling asynchronous scheduling by default for pooling model."
|
||||||
|
)
|
||||||
|
self.scheduler_config.async_scheduling = False
|
||||||
|
elif (
|
||||||
self.speculative_config is not None
|
self.speculative_config is not None
|
||||||
and self.speculative_config.method not in get_args(EagleModelTypes)
|
and self.speculative_config.method not in get_args(EagleModelTypes)
|
||||||
and self.speculative_config.method not in get_args(NgramGPUTypes)
|
and self.speculative_config.method not in get_args(NgramGPUTypes)
|
||||||
|
|||||||
Reference in New Issue
Block a user