[Bugfix][KV Offload] Clean up resources after initialization failure (#51227)

Signed-off-by: Alex <jihui.huang@daocloud.io>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
AlexHuang
2026-08-06 14:42:13 +03:00
committed by GitHub
co-authored by OpenAI Codex
parent 5fba75aefe
commit 46e6a83ce1
2 changed files with 85 additions and 53 deletions
+11 -6
View File
@@ -165,12 +165,17 @@ class CPUOffloadingSpec(OffloadingSpec):
kv_bytes_per_block=self.kv_bytes_per_chunk,
cpu_page_size=self.cpu_page_size_per_worker,
)
return CPUOffloadingWorker(
kv_caches=kv_caches,
blocks_per_chunk=self.blocks_per_chunk,
num_cpu_blocks=self.num_blocks,
mmap_region=mmap_region,
)
try:
return CPUOffloadingWorker(
kv_caches=kv_caches,
blocks_per_chunk=self.blocks_per_chunk,
num_cpu_blocks=self.num_blocks,
mmap_region=mmap_region,
)
except Exception:
if mmap_region is not None:
mmap_region.cleanup()
raise
@override
def get_worker(self, kv_caches: CanonicalKVCaches) -> OffloadingWorker:
+74 -47
View File
@@ -185,31 +185,38 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
TieringOffloadingManager instance
"""
if not self._manager:
# Create scheduler-side SharedOffloadRegion (rank=None) so the
# primary tier can eagerly create a memoryview over _base.
scheduler_mmap = SharedOffloadRegion(
engine_id=self._engine_id,
num_blocks=self.num_blocks,
rank=None,
kv_bytes_per_block=self.kv_bytes_per_chunk,
cpu_page_size=self.cpu_page_size_per_worker,
)
self._scheduler_mmap = scheduler_mmap
if int(self.extra_config.get("store_threshold", 0)) >= 2:
raise ValueError(
"store_threshold is not supported for TieringOffloadingSpec"
)
# Create primary tier (CPU-based)
primary_tier = CPUPrimaryTierOffloadingManager(
num_blocks=self.num_blocks,
cache_policy=self.eviction_policy,
cache_policy_module_path=self.cache_policy_module_path,
enable_events=self.kv_events_config.enable_kv_cache_events,
mmap_region=scheduler_mmap,
)
# Create secondary tiers
primary_kv_view = primary_tier.get_kv_memoryview()
scheduler_mmap: SharedOffloadRegion | None = None
primary_tier: CPUPrimaryTierOffloadingManager | None = None
secondary_tiers = []
for i, tier_config in enumerate(self.secondary_tier_configs):
try:
try:
# Create scheduler-side SharedOffloadRegion (rank=None) so the
# primary tier can eagerly create a memoryview over _base.
scheduler_mmap = SharedOffloadRegion(
engine_id=self._engine_id,
num_blocks=self.num_blocks,
rank=None,
kv_bytes_per_block=self.kv_bytes_per_chunk,
cpu_page_size=self.cpu_page_size_per_worker,
)
self._scheduler_mmap = scheduler_mmap
# Create primary tier (CPU-based)
primary_tier = CPUPrimaryTierOffloadingManager(
num_blocks=self.num_blocks,
cache_policy=self.eviction_policy,
cache_policy_module_path=self.cache_policy_module_path,
enable_events=self.kv_events_config.enable_kv_cache_events,
mmap_region=scheduler_mmap,
)
# Create secondary tiers
primary_kv_view = primary_tier.get_kv_memoryview()
for i, tier_config in enumerate(self.secondary_tier_configs):
tier = SecondaryTierFactory.create_secondary_tier(
tier_config, primary_kv_view, self
)
@@ -219,26 +226,42 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
i,
tier.tier_type,
)
except Exception as e:
logger.error(
"Failed to create secondary tier from config index %i: %s",
i,
e,
)
raise
# Create TieringOffloadingManager. GPU↔CPU transfers use the inherited
# get_worker(). Secondary tier transfers are handled by the
# secondary tier managers and need no additional workers here.
tiering_manager = TieringOffloadingManager(
primary_tier=primary_tier,
secondary_tiers=secondary_tiers,
)
if int(self.extra_config.get("store_threshold", 0)) >= 2:
raise ValueError(
"store_threshold is not supported for TieringOffloadingSpec"
# Create TieringOffloadingManager. GPU↔CPU transfers use the inherited
# get_worker(). Secondary tier transfers are handled by the
# secondary tier managers and need no additional workers here.
tiering_manager = TieringOffloadingManager(
primary_tier=primary_tier,
secondary_tiers=secondary_tiers,
)
self._manager = tiering_manager
self._manager = tiering_manager
except Exception:
for tier in reversed(secondary_tiers):
try:
tier.shutdown()
except Exception:
logger.exception(
"Failed to shut down secondary tier during "
"initialization cleanup"
)
if primary_tier is not None:
try:
primary_tier.shutdown()
except Exception:
logger.exception(
"Failed to shut down primary tier during "
"initialization cleanup"
)
elif scheduler_mmap is not None:
try:
scheduler_mmap.cleanup()
except Exception:
logger.exception(
"Failed to clean up scheduler mmap during "
"initialization cleanup"
)
self._scheduler_mmap = None
raise
logger.info(
"Created TieringOffloadingManager with primary tier "
@@ -273,9 +296,13 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
kv_bytes_per_block=self.kv_bytes_per_chunk,
cpu_page_size=self.cpu_page_size_per_worker,
)
return CPUOffloadingWorker(
kv_caches=kv_caches,
blocks_per_chunk=self.blocks_per_chunk,
num_cpu_blocks=self.num_blocks,
mmap_region=worker_mmap,
)
try:
return CPUOffloadingWorker(
kv_caches=kv_caches,
blocks_per_chunk=self.blocks_per_chunk,
num_cpu_blocks=self.num_blocks,
mmap_region=worker_mmap,
)
except Exception:
worker_mmap.cleanup()
raise