[Bugfix][KV Offload] Clean up resources after initialization failure (#51227)
Signed-off-by: Alex <jihui.huang@daocloud.io> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
5fba75aefe
commit
46e6a83ce1
@@ -165,12 +165,17 @@ class CPUOffloadingSpec(OffloadingSpec):
|
||||
kv_bytes_per_block=self.kv_bytes_per_chunk,
|
||||
cpu_page_size=self.cpu_page_size_per_worker,
|
||||
)
|
||||
return CPUOffloadingWorker(
|
||||
kv_caches=kv_caches,
|
||||
blocks_per_chunk=self.blocks_per_chunk,
|
||||
num_cpu_blocks=self.num_blocks,
|
||||
mmap_region=mmap_region,
|
||||
)
|
||||
try:
|
||||
return CPUOffloadingWorker(
|
||||
kv_caches=kv_caches,
|
||||
blocks_per_chunk=self.blocks_per_chunk,
|
||||
num_cpu_blocks=self.num_blocks,
|
||||
mmap_region=mmap_region,
|
||||
)
|
||||
except Exception:
|
||||
if mmap_region is not None:
|
||||
mmap_region.cleanup()
|
||||
raise
|
||||
|
||||
@override
|
||||
def get_worker(self, kv_caches: CanonicalKVCaches) -> OffloadingWorker:
|
||||
|
||||
@@ -185,31 +185,38 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
TieringOffloadingManager instance
|
||||
"""
|
||||
if not self._manager:
|
||||
# Create scheduler-side SharedOffloadRegion (rank=None) so the
|
||||
# primary tier can eagerly create a memoryview over _base.
|
||||
scheduler_mmap = SharedOffloadRegion(
|
||||
engine_id=self._engine_id,
|
||||
num_blocks=self.num_blocks,
|
||||
rank=None,
|
||||
kv_bytes_per_block=self.kv_bytes_per_chunk,
|
||||
cpu_page_size=self.cpu_page_size_per_worker,
|
||||
)
|
||||
self._scheduler_mmap = scheduler_mmap
|
||||
if int(self.extra_config.get("store_threshold", 0)) >= 2:
|
||||
raise ValueError(
|
||||
"store_threshold is not supported for TieringOffloadingSpec"
|
||||
)
|
||||
|
||||
# Create primary tier (CPU-based)
|
||||
primary_tier = CPUPrimaryTierOffloadingManager(
|
||||
num_blocks=self.num_blocks,
|
||||
cache_policy=self.eviction_policy,
|
||||
cache_policy_module_path=self.cache_policy_module_path,
|
||||
enable_events=self.kv_events_config.enable_kv_cache_events,
|
||||
mmap_region=scheduler_mmap,
|
||||
)
|
||||
|
||||
# Create secondary tiers
|
||||
primary_kv_view = primary_tier.get_kv_memoryview()
|
||||
scheduler_mmap: SharedOffloadRegion | None = None
|
||||
primary_tier: CPUPrimaryTierOffloadingManager | None = None
|
||||
secondary_tiers = []
|
||||
for i, tier_config in enumerate(self.secondary_tier_configs):
|
||||
try:
|
||||
try:
|
||||
# Create scheduler-side SharedOffloadRegion (rank=None) so the
|
||||
# primary tier can eagerly create a memoryview over _base.
|
||||
scheduler_mmap = SharedOffloadRegion(
|
||||
engine_id=self._engine_id,
|
||||
num_blocks=self.num_blocks,
|
||||
rank=None,
|
||||
kv_bytes_per_block=self.kv_bytes_per_chunk,
|
||||
cpu_page_size=self.cpu_page_size_per_worker,
|
||||
)
|
||||
self._scheduler_mmap = scheduler_mmap
|
||||
|
||||
# Create primary tier (CPU-based)
|
||||
primary_tier = CPUPrimaryTierOffloadingManager(
|
||||
num_blocks=self.num_blocks,
|
||||
cache_policy=self.eviction_policy,
|
||||
cache_policy_module_path=self.cache_policy_module_path,
|
||||
enable_events=self.kv_events_config.enable_kv_cache_events,
|
||||
mmap_region=scheduler_mmap,
|
||||
)
|
||||
|
||||
# Create secondary tiers
|
||||
primary_kv_view = primary_tier.get_kv_memoryview()
|
||||
for i, tier_config in enumerate(self.secondary_tier_configs):
|
||||
tier = SecondaryTierFactory.create_secondary_tier(
|
||||
tier_config, primary_kv_view, self
|
||||
)
|
||||
@@ -219,26 +226,42 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
i,
|
||||
tier.tier_type,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"Failed to create secondary tier from config index %i: %s",
|
||||
i,
|
||||
e,
|
||||
)
|
||||
raise
|
||||
|
||||
# Create TieringOffloadingManager. GPU↔CPU transfers use the inherited
|
||||
# get_worker(). Secondary tier transfers are handled by the
|
||||
# secondary tier managers and need no additional workers here.
|
||||
tiering_manager = TieringOffloadingManager(
|
||||
primary_tier=primary_tier,
|
||||
secondary_tiers=secondary_tiers,
|
||||
)
|
||||
if int(self.extra_config.get("store_threshold", 0)) >= 2:
|
||||
raise ValueError(
|
||||
"store_threshold is not supported for TieringOffloadingSpec"
|
||||
# Create TieringOffloadingManager. GPU↔CPU transfers use the inherited
|
||||
# get_worker(). Secondary tier transfers are handled by the
|
||||
# secondary tier managers and need no additional workers here.
|
||||
tiering_manager = TieringOffloadingManager(
|
||||
primary_tier=primary_tier,
|
||||
secondary_tiers=secondary_tiers,
|
||||
)
|
||||
self._manager = tiering_manager
|
||||
self._manager = tiering_manager
|
||||
except Exception:
|
||||
for tier in reversed(secondary_tiers):
|
||||
try:
|
||||
tier.shutdown()
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to shut down secondary tier during "
|
||||
"initialization cleanup"
|
||||
)
|
||||
if primary_tier is not None:
|
||||
try:
|
||||
primary_tier.shutdown()
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to shut down primary tier during "
|
||||
"initialization cleanup"
|
||||
)
|
||||
elif scheduler_mmap is not None:
|
||||
try:
|
||||
scheduler_mmap.cleanup()
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to clean up scheduler mmap during "
|
||||
"initialization cleanup"
|
||||
)
|
||||
self._scheduler_mmap = None
|
||||
raise
|
||||
|
||||
logger.info(
|
||||
"Created TieringOffloadingManager with primary tier "
|
||||
@@ -273,9 +296,13 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
kv_bytes_per_block=self.kv_bytes_per_chunk,
|
||||
cpu_page_size=self.cpu_page_size_per_worker,
|
||||
)
|
||||
return CPUOffloadingWorker(
|
||||
kv_caches=kv_caches,
|
||||
blocks_per_chunk=self.blocks_per_chunk,
|
||||
num_cpu_blocks=self.num_blocks,
|
||||
mmap_region=worker_mmap,
|
||||
)
|
||||
try:
|
||||
return CPUOffloadingWorker(
|
||||
kv_caches=kv_caches,
|
||||
blocks_per_chunk=self.blocks_per_chunk,
|
||||
num_cpu_blocks=self.num_blocks,
|
||||
mmap_region=worker_mmap,
|
||||
)
|
||||
except Exception:
|
||||
worker_mmap.cleanup()
|
||||
raise
|
||||
|
||||
Reference in New Issue
Block a user