[Core] Rework handling of async scheduling config (#28250)
Signed-off-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
@@ -4,7 +4,7 @@
|
||||
import hashlib
|
||||
from collections.abc import Callable
|
||||
from dataclasses import InitVar
|
||||
from typing import Any, Literal
|
||||
from typing import TYPE_CHECKING, Any, Literal, cast
|
||||
|
||||
from pydantic import Field, field_validator, model_validator
|
||||
from pydantic.dataclasses import dataclass
|
||||
@@ -17,6 +17,10 @@ from vllm.utils import (
|
||||
MULTIMODAL_MODEL_MAX_NUM_BATCHED_TOKENS,
|
||||
POOLING_MODEL_MAX_NUM_BATCHED_TOKENS,
|
||||
)
|
||||
from vllm.utils.import_utils import resolve_obj_by_qualname
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.core.sched.interface import SchedulerInterface
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -120,7 +124,7 @@ class SchedulerConfig:
|
||||
|
||||
# scheduler class or path. "vllm.v1.core.sched.scheduler.Scheduler"
|
||||
# (default) or "mod.custom_class".
|
||||
scheduler_cls: str | type[object] = "vllm.v1.core.sched.scheduler.Scheduler"
|
||||
scheduler_cls: str | type[object] = Field(default=None)
|
||||
"""The scheduler class to use. "vllm.v1.core.sched.scheduler.Scheduler" is
|
||||
the default scheduler. Can be a class directly or the path to a class of
|
||||
form "mod.custom_class"."""
|
||||
@@ -132,12 +136,34 @@ class SchedulerConfig:
|
||||
"""
|
||||
|
||||
async_scheduling: bool = False
|
||||
"""EXPERIMENTAL: If set to True, perform async scheduling. This may help
|
||||
reduce the CPU overheads, leading to better latency and throughput. However,
|
||||
async scheduling is currently not supported with some features such as
|
||||
structured outputs, speculative decoding, and pipeline parallelism.
|
||||
"""If set to True, perform async scheduling. This helps to avoid gaps in
|
||||
GPU utilization, leading to better latency and throughput.
|
||||
Async scheduling is currently not supported with some features such as
|
||||
speculative decoding and pipeline parallelism.
|
||||
"""
|
||||
|
||||
def get_scheduler_cls(self) -> type["SchedulerInterface"]:
|
||||
if self.scheduler_cls is None:
|
||||
if self.async_scheduling:
|
||||
from vllm.v1.core.sched.async_scheduler import AsyncScheduler
|
||||
|
||||
return AsyncScheduler
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
|
||||
return Scheduler
|
||||
|
||||
# This warning can be removed once the Scheduler interface is
|
||||
# finalized and we can maintain support for scheduler classes that
|
||||
# implement it
|
||||
logger.warning_once(
|
||||
"Using custom scheduler class %s. This scheduler interface is "
|
||||
"not public and compatibility may not be maintained.",
|
||||
self.scheduler_cls,
|
||||
)
|
||||
if not isinstance(self.scheduler_cls, str):
|
||||
return cast(type["SchedulerInterface"], self.scheduler_cls)
|
||||
return resolve_obj_by_qualname(self.scheduler_cls)
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
WARNING: Whenever a new field is added to this config,
|
||||
@@ -161,6 +187,8 @@ class SchedulerConfig:
|
||||
"max_num_seqs",
|
||||
"max_model_len",
|
||||
"enable_chunked_prefill",
|
||||
"scheduler_cls",
|
||||
"async_scheduling",
|
||||
mode="wrap",
|
||||
)
|
||||
@classmethod
|
||||
@@ -242,9 +270,6 @@ class SchedulerConfig:
|
||||
self.long_prefill_token_threshold,
|
||||
)
|
||||
|
||||
if self.async_scheduling:
|
||||
self.scheduler_cls = "vllm.v1.core.sched.async_scheduler.AsyncScheduler"
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _verify_args(self) -> Self:
|
||||
if (
|
||||
|
||||
+49
-2
@@ -353,6 +353,53 @@ class VllmConfig:
|
||||
self.model_config, self.load_config
|
||||
)
|
||||
|
||||
executor_backend = self.parallel_config.distributed_executor_backend
|
||||
executor_supports_async_sched = executor_backend in (
|
||||
"mp",
|
||||
"uni",
|
||||
"external_launcher",
|
||||
)
|
||||
|
||||
if self.scheduler_config.async_scheduling:
|
||||
# Async scheduling explicitly enabled, hard fail any incompatibilities.
|
||||
if self.parallel_config.pipeline_parallel_size > 1:
|
||||
raise ValueError(
|
||||
"Async scheduling is not yet compatible with "
|
||||
"pipeline_parallel_size > 1."
|
||||
)
|
||||
if self.speculative_config is not None:
|
||||
raise ValueError(
|
||||
"Async scheduling is not yet compatible with speculative decoding."
|
||||
)
|
||||
if not executor_supports_async_sched:
|
||||
raise ValueError(
|
||||
"Currently, async scheduling only supports `mp`, `uni`, or "
|
||||
"`external_launcher` distributed executor backend, but you chose "
|
||||
f"`{executor_backend}`."
|
||||
)
|
||||
elif self.scheduler_config.async_scheduling is None:
|
||||
# Enable async scheduling unless there is an incompatible option.
|
||||
# NOTE: we won't reach here until async scheduling is enabled by default.
|
||||
if (
|
||||
self.parallel_config.pipeline_parallel_size > 1
|
||||
or self.speculative_config is not None
|
||||
):
|
||||
logger.warning(
|
||||
"Async scheduling is not yet supported with speculative decoding "
|
||||
" or pipeline_parallel_size > 1 and will be disabled."
|
||||
)
|
||||
self.scheduler_config.async_scheduling = False
|
||||
elif not executor_supports_async_sched:
|
||||
logger.warning(
|
||||
"Async scheduling will be disabled because it is not supported "
|
||||
"with the `%s` distributed executor backend (only `mp`, `uni`, and "
|
||||
"`external_launcher` are supported).",
|
||||
executor_backend,
|
||||
)
|
||||
self.scheduler_config.async_scheduling = False
|
||||
else:
|
||||
self.scheduler_config.async_scheduling = True
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if (
|
||||
@@ -467,7 +514,7 @@ class VllmConfig:
|
||||
self.speculative_config is not None
|
||||
and self.speculative_config.use_eagle()
|
||||
):
|
||||
raise NotImplementedError(
|
||||
raise ValueError(
|
||||
"Fast prefill optimization for KV sharing is not "
|
||||
"compatible with EAGLE as EAGLE requires correct logits "
|
||||
"for all tokens while fast prefill gives incorrect logits "
|
||||
@@ -491,7 +538,7 @@ class VllmConfig:
|
||||
)
|
||||
if not getattr(self.model_config.hf_config, "is_causal", True):
|
||||
disable_chunked_prefill_reasons.append(
|
||||
"Only models using causal attention supports chunked "
|
||||
"Only models using causal attention support chunked "
|
||||
"prefill and prefix caching; disabling both."
|
||||
)
|
||||
elif self.model_config.is_encoder_decoder:
|
||||
|
||||
Reference in New Issue
Block a user