From 46e6a83ce12b5968d956279a1bd4611de16d69eb Mon Sep 17 00:00:00 2001 From: AlexHuang Date: Thu, 6 Aug 2026 19:42:13 +0800 Subject: [PATCH] [Bugfix][KV Offload] Clean up resources after initialization failure (#51227) Signed-off-by: Alex Co-authored-by: OpenAI Codex --- vllm/v1/kv_offload/cpu/spec.py | 17 ++-- vllm/v1/kv_offload/tiering/spec.py | 121 ++++++++++++++++++----------- 2 files changed, 85 insertions(+), 53 deletions(-) diff --git a/vllm/v1/kv_offload/cpu/spec.py b/vllm/v1/kv_offload/cpu/spec.py index 054bf34064..91d20c10b1 100644 --- a/vllm/v1/kv_offload/cpu/spec.py +++ b/vllm/v1/kv_offload/cpu/spec.py @@ -165,12 +165,17 @@ class CPUOffloadingSpec(OffloadingSpec): kv_bytes_per_block=self.kv_bytes_per_chunk, cpu_page_size=self.cpu_page_size_per_worker, ) - return CPUOffloadingWorker( - kv_caches=kv_caches, - blocks_per_chunk=self.blocks_per_chunk, - num_cpu_blocks=self.num_blocks, - mmap_region=mmap_region, - ) + try: + return CPUOffloadingWorker( + kv_caches=kv_caches, + blocks_per_chunk=self.blocks_per_chunk, + num_cpu_blocks=self.num_blocks, + mmap_region=mmap_region, + ) + except Exception: + if mmap_region is not None: + mmap_region.cleanup() + raise @override def get_worker(self, kv_caches: CanonicalKVCaches) -> OffloadingWorker: diff --git a/vllm/v1/kv_offload/tiering/spec.py b/vllm/v1/kv_offload/tiering/spec.py index fb2cdacb99..ccce426bfd 100644 --- a/vllm/v1/kv_offload/tiering/spec.py +++ b/vllm/v1/kv_offload/tiering/spec.py @@ -185,31 +185,38 @@ class TieringOffloadingSpec(CPUOffloadingSpec): TieringOffloadingManager instance """ if not self._manager: - # Create scheduler-side SharedOffloadRegion (rank=None) so the - # primary tier can eagerly create a memoryview over _base. - scheduler_mmap = SharedOffloadRegion( - engine_id=self._engine_id, - num_blocks=self.num_blocks, - rank=None, - kv_bytes_per_block=self.kv_bytes_per_chunk, - cpu_page_size=self.cpu_page_size_per_worker, - ) - self._scheduler_mmap = scheduler_mmap + if int(self.extra_config.get("store_threshold", 0)) >= 2: + raise ValueError( + "store_threshold is not supported for TieringOffloadingSpec" + ) - # Create primary tier (CPU-based) - primary_tier = CPUPrimaryTierOffloadingManager( - num_blocks=self.num_blocks, - cache_policy=self.eviction_policy, - cache_policy_module_path=self.cache_policy_module_path, - enable_events=self.kv_events_config.enable_kv_cache_events, - mmap_region=scheduler_mmap, - ) - - # Create secondary tiers - primary_kv_view = primary_tier.get_kv_memoryview() + scheduler_mmap: SharedOffloadRegion | None = None + primary_tier: CPUPrimaryTierOffloadingManager | None = None secondary_tiers = [] - for i, tier_config in enumerate(self.secondary_tier_configs): - try: + try: + # Create scheduler-side SharedOffloadRegion (rank=None) so the + # primary tier can eagerly create a memoryview over _base. + scheduler_mmap = SharedOffloadRegion( + engine_id=self._engine_id, + num_blocks=self.num_blocks, + rank=None, + kv_bytes_per_block=self.kv_bytes_per_chunk, + cpu_page_size=self.cpu_page_size_per_worker, + ) + self._scheduler_mmap = scheduler_mmap + + # Create primary tier (CPU-based) + primary_tier = CPUPrimaryTierOffloadingManager( + num_blocks=self.num_blocks, + cache_policy=self.eviction_policy, + cache_policy_module_path=self.cache_policy_module_path, + enable_events=self.kv_events_config.enable_kv_cache_events, + mmap_region=scheduler_mmap, + ) + + # Create secondary tiers + primary_kv_view = primary_tier.get_kv_memoryview() + for i, tier_config in enumerate(self.secondary_tier_configs): tier = SecondaryTierFactory.create_secondary_tier( tier_config, primary_kv_view, self ) @@ -219,26 +226,42 @@ class TieringOffloadingSpec(CPUOffloadingSpec): i, tier.tier_type, ) - except Exception as e: - logger.error( - "Failed to create secondary tier from config index %i: %s", - i, - e, - ) - raise - # Create TieringOffloadingManager. GPU↔CPU transfers use the inherited - # get_worker(). Secondary tier transfers are handled by the - # secondary tier managers and need no additional workers here. - tiering_manager = TieringOffloadingManager( - primary_tier=primary_tier, - secondary_tiers=secondary_tiers, - ) - if int(self.extra_config.get("store_threshold", 0)) >= 2: - raise ValueError( - "store_threshold is not supported for TieringOffloadingSpec" + # Create TieringOffloadingManager. GPU↔CPU transfers use the inherited + # get_worker(). Secondary tier transfers are handled by the + # secondary tier managers and need no additional workers here. + tiering_manager = TieringOffloadingManager( + primary_tier=primary_tier, + secondary_tiers=secondary_tiers, ) - self._manager = tiering_manager + self._manager = tiering_manager + except Exception: + for tier in reversed(secondary_tiers): + try: + tier.shutdown() + except Exception: + logger.exception( + "Failed to shut down secondary tier during " + "initialization cleanup" + ) + if primary_tier is not None: + try: + primary_tier.shutdown() + except Exception: + logger.exception( + "Failed to shut down primary tier during " + "initialization cleanup" + ) + elif scheduler_mmap is not None: + try: + scheduler_mmap.cleanup() + except Exception: + logger.exception( + "Failed to clean up scheduler mmap during " + "initialization cleanup" + ) + self._scheduler_mmap = None + raise logger.info( "Created TieringOffloadingManager with primary tier " @@ -273,9 +296,13 @@ class TieringOffloadingSpec(CPUOffloadingSpec): kv_bytes_per_block=self.kv_bytes_per_chunk, cpu_page_size=self.cpu_page_size_per_worker, ) - return CPUOffloadingWorker( - kv_caches=kv_caches, - blocks_per_chunk=self.blocks_per_chunk, - num_cpu_blocks=self.num_blocks, - mmap_region=worker_mmap, - ) + try: + return CPUOffloadingWorker( + kv_caches=kv_caches, + blocks_per_chunk=self.blocks_per_chunk, + num_cpu_blocks=self.num_blocks, + mmap_region=worker_mmap, + ) + except Exception: + worker_mmap.cleanup() + raise