[Perf] Enable environment cache in EngineCore to enable the feature for UniProcExecutor as well (#29289)
Signed-off-by: Jialin Ouyang <Jialin.Ouyang@gmail.com>
This commit is contained in:
@@ -1586,6 +1586,8 @@ def destroy_distributed_environment():
|
||||
|
||||
|
||||
def cleanup_dist_env_and_memory(shutdown_ray: bool = False):
|
||||
# Reset environment variable cache
|
||||
envs.disable_envs_cache()
|
||||
# Ensure all objects are not frozen before cleanup
|
||||
gc.unfreeze()
|
||||
|
||||
|
||||
@@ -1580,6 +1580,12 @@ def __getattr__(name: str):
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
|
||||
def _is_envs_cache_enabled() -> bool:
|
||||
"""Checked if __getattr__ is wrapped with functools.cache"""
|
||||
global __getattr__
|
||||
return hasattr(__getattr__, "cache_clear")
|
||||
|
||||
|
||||
def enable_envs_cache() -> None:
|
||||
"""
|
||||
Enables caching of environment variables. This is useful for performance
|
||||
@@ -1590,6 +1596,9 @@ def enable_envs_cache() -> None:
|
||||
runtime overhead. This also means that environment variables should NOT
|
||||
be updated after the service is initialized.
|
||||
"""
|
||||
if _is_envs_cache_enabled():
|
||||
# Avoid wrapping functools.cache multiple times
|
||||
return
|
||||
# Tag __getattr__ with functools.cache
|
||||
global __getattr__
|
||||
__getattr__ = functools.cache(__getattr__)
|
||||
@@ -1599,6 +1608,17 @@ def enable_envs_cache() -> None:
|
||||
__getattr__(key)
|
||||
|
||||
|
||||
def disable_envs_cache() -> None:
|
||||
"""
|
||||
Resets the environment variables cache. It could be used to isolate environments
|
||||
between unit tests.
|
||||
"""
|
||||
global __getattr__
|
||||
# If __getattr__ is wrapped by functions.cache, unwrap the caching layer.
|
||||
if _is_envs_cache_enabled():
|
||||
__getattr__ = __getattr__.__wrapped__
|
||||
|
||||
|
||||
def __dir__():
|
||||
return list(environment_variables.keys())
|
||||
|
||||
|
||||
@@ -211,6 +211,9 @@ class EngineCore:
|
||||
freeze_gc_heap()
|
||||
# If enable, attach GC debugger after static variable freeze.
|
||||
maybe_attach_gc_debug_callback()
|
||||
# Enable environment variable cache (e.g. assume no more
|
||||
# environment variable overrides after this point)
|
||||
enable_envs_cache()
|
||||
|
||||
def _initialize_kv_caches(
|
||||
self, vllm_config: VllmConfig
|
||||
@@ -672,10 +675,6 @@ class EngineCoreProc(EngineCore):
|
||||
assert addresses.coordinator_input is not None
|
||||
logger.info("Waiting for READY message from DP Coordinator...")
|
||||
|
||||
# Enable environment variable cache (e.g. assume no more
|
||||
# environment variable overrides after this point)
|
||||
enable_envs_cache()
|
||||
|
||||
@contextmanager
|
||||
def _perform_handshakes(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user