[XPU] Optimize XPU worker shutdown logic to prevent resource leak (#46433)
Signed-off-by: Chaojun Zhang <chaojun.zhang@intel.com>
This commit is contained in:
@@ -23,4 +23,5 @@ steps:
|
||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||
'cd tests &&
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
|
||||
pytest -v -s basic_correctness/test_cpu_offload.py &&
|
||||
pytest -v -s basic_correctness/test_mem.py::test_end_to_end'
|
||||
|
||||
@@ -128,10 +128,10 @@ steps:
|
||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||
'cd tests &&
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
|
||||
(pytest -v -s lora/test_mixtral.py --deselect="tests/lora/test_mixtral.py::test_mixtral_lora[4]" || true) &&
|
||||
pytest -v -s lora/test_quant_model.py --deselect="tests/lora/test_quant_model.py::test_quant_model_lora[model0]" --deselect="tests/lora/test_quant_model.py::test_quant_model_lora[model1]" --deselect="tests/lora/test_quant_model.py::test_quant_model_tp_equality[model0]" &&
|
||||
pytest -v -s lora/test_transformers_model.py &&
|
||||
pytest -v -s lora/test_chatglm3_tp.py &&
|
||||
pytest -v -s lora/test_llama_tp.py::test_llama_lora &&
|
||||
pytest -s -v lora/test_minicpmv_tp.py'
|
||||
|
||||
- label: LoRA Multimodal
|
||||
|
||||
@@ -604,6 +604,13 @@ class RemoteVLLMServer:
|
||||
mem_info = nvmlDeviceGetMemoryInfo(handle)
|
||||
total_used += mem_info.used
|
||||
return total_used
|
||||
elif current_platform.is_xpu():
|
||||
total_used = 0
|
||||
device_count = current_platform.device_count()
|
||||
for i in range(device_count):
|
||||
free, total = torch.xpu.mem_get_info(i)
|
||||
total_used += total - free
|
||||
return total_used
|
||||
except Exception as e:
|
||||
print(f"[RemoteOpenAIServer] Could not query GPU memory: {e}")
|
||||
return None
|
||||
|
||||
@@ -243,6 +243,16 @@ class XPUPlatform(Platform):
|
||||
if "VLLM_WORKER_MULTIPROC_METHOD" not in os.environ:
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
# XPU requires graceful shutdown to allow oneCCL/Level Zero resources
|
||||
# to be properly released. Without this, subsequent server startups on
|
||||
# the same devices may hang during CCL initialization.
|
||||
if vllm_config.shutdown_timeout == 0:
|
||||
vllm_config.shutdown_timeout = 5
|
||||
logger.info(
|
||||
"XPU platform: set server shutdown_timeout=%d.",
|
||||
vllm_config.shutdown_timeout,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def update_block_size_for_backend(cls, vllm_config: "VllmConfig") -> None:
|
||||
super().update_block_size_for_backend(vllm_config)
|
||||
|
||||
@@ -6385,7 +6385,7 @@ class GPUModelRunner(
|
||||
_ROPE_DICT.clear()
|
||||
|
||||
reset_workspace_manager()
|
||||
if current_platform.is_rocm():
|
||||
if current_platform.is_rocm() or current_platform.is_xpu():
|
||||
gc.collect()
|
||||
torch.accelerator.empty_cache()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
@@ -1294,10 +1294,11 @@ class Worker(WorkerBase):
|
||||
# Release kept-alive cumem pools while the pluggable allocator wrappers
|
||||
# and callbacks are still alive, so MemPool teardown is not deferred to
|
||||
# interpreter finalization (pytorch/pytorch#145168).
|
||||
from vllm.device_allocator.cumem import CuMemAllocator
|
||||
if current_platform.is_cuda_alike():
|
||||
from vllm.device_allocator.cumem import CuMemAllocator
|
||||
|
||||
if CuMemAllocator.instance is not None:
|
||||
CuMemAllocator.instance.release_pools()
|
||||
if CuMemAllocator.instance is not None:
|
||||
CuMemAllocator.instance.release_pools()
|
||||
|
||||
def elastic_ep_execute(self, execute_method: str, *args, **kwargs):
|
||||
return self.elastic_ep_executor.execute(execute_method, *args, **kwargs)
|
||||
|
||||
@@ -162,3 +162,20 @@ class XPUWorker(Worker):
|
||||
logger.debug("Starting torch profiler with trace name: %s", trace_name)
|
||||
|
||||
super().profile(is_start=is_start, profile_prefix=profile_prefix)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
logger.info(
|
||||
"XPUWorker shutdown: cleaning up (rank=%d, local_rank=%d)",
|
||||
self.rank,
|
||||
self.local_rank,
|
||||
)
|
||||
super().shutdown()
|
||||
from vllm.device_allocator.xpumem import XpuMemAllocator
|
||||
|
||||
if XpuMemAllocator.instance is not None:
|
||||
XpuMemAllocator.instance.release_pools()
|
||||
logger.info(
|
||||
"XPUWorker shutdown: done (rank=%d, local_rank=%d)",
|
||||
self.rank,
|
||||
self.local_rank,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user