[MISC][Bugfix] Use less CPU when message queue has been empty for some time (#16226)

Signed-off-by: Povilas Kanapickas <povilas@radix.lt>
This commit is contained in:
Povilas Kanapickas
2025-06-05 16:53:08 +00:00
committed by GitHub
parent 61059bee40
commit 85e2b7bb13
3 changed files with 70 additions and 11 deletions
+6
View File
@@ -122,6 +122,7 @@ if TYPE_CHECKING:
VLLM_ALL2ALL_BACKEND: str = "naive"
VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: int = 163840
VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS: int = 1
VLLM_SLEEP_WHEN_IDLE: bool = False
def get_default_cache_root():
@@ -841,6 +842,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
# Regex timeout for use by the vLLM tool parsing plugins.
"VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS":
lambda: int(os.getenv("VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS", "1")),
# Reduce CPU usage when vLLM is idle. Enabling this will incur small
# latency penalty when a request eventually comes.
"VLLM_SLEEP_WHEN_IDLE":
lambda: bool(int(os.getenv("VLLM_SLEEP_WHEN_IDLE", "0"))),
}
# --8<-- [end:env-vars-definition]