[XPU] Optimize XPU worker shutdown logic to prevent resource leak (#46433)

Signed-off-by: Chaojun Zhang <chaojun.zhang@intel.com>
This commit is contained in:
Chaojun Zhang
2026-06-30 15:27:21 +08:00
committed by GitHub
parent ba22cb6765
commit 8cc242335d
7 changed files with 41 additions and 5 deletions
@@ -23,4 +23,5 @@ steps:
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'cd tests &&
export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
pytest -v -s basic_correctness/test_cpu_offload.py &&
pytest -v -s basic_correctness/test_mem.py::test_end_to_end'
+1 -1
View File
@@ -128,10 +128,10 @@ steps:
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'cd tests &&
export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
(pytest -v -s lora/test_mixtral.py --deselect="tests/lora/test_mixtral.py::test_mixtral_lora[4]" || true) &&
pytest -v -s lora/test_quant_model.py --deselect="tests/lora/test_quant_model.py::test_quant_model_lora[model0]" --deselect="tests/lora/test_quant_model.py::test_quant_model_lora[model1]" --deselect="tests/lora/test_quant_model.py::test_quant_model_tp_equality[model0]" &&
pytest -v -s lora/test_transformers_model.py &&
pytest -v -s lora/test_chatglm3_tp.py &&
pytest -v -s lora/test_llama_tp.py::test_llama_lora &&
pytest -s -v lora/test_minicpmv_tp.py'
- label: LoRA Multimodal
+7
View File
@@ -604,6 +604,13 @@ class RemoteVLLMServer:
mem_info = nvmlDeviceGetMemoryInfo(handle)
total_used += mem_info.used
return total_used
elif current_platform.is_xpu():
total_used = 0
device_count = current_platform.device_count()
for i in range(device_count):
free, total = torch.xpu.mem_get_info(i)
total_used += total - free
return total_used
except Exception as e:
print(f"[RemoteOpenAIServer] Could not query GPU memory: {e}")
return None
+10
View File
@@ -243,6 +243,16 @@ class XPUPlatform(Platform):
if "VLLM_WORKER_MULTIPROC_METHOD" not in os.environ:
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
# XPU requires graceful shutdown to allow oneCCL/Level Zero resources
# to be properly released. Without this, subsequent server startups on
# the same devices may hang during CCL initialization.
if vllm_config.shutdown_timeout == 0:
vllm_config.shutdown_timeout = 5
logger.info(
"XPU platform: set server shutdown_timeout=%d.",
vllm_config.shutdown_timeout,
)
@classmethod
def update_block_size_for_backend(cls, vllm_config: "VllmConfig") -> None:
super().update_block_size_for_backend(vllm_config)
+1 -1
View File
@@ -6385,7 +6385,7 @@ class GPUModelRunner(
_ROPE_DICT.clear()
reset_workspace_manager()
if current_platform.is_rocm():
if current_platform.is_rocm() or current_platform.is_xpu():
gc.collect()
torch.accelerator.empty_cache()
torch.accelerator.synchronize()
+4 -3
View File
@@ -1294,10 +1294,11 @@ class Worker(WorkerBase):
# Release kept-alive cumem pools while the pluggable allocator wrappers
# and callbacks are still alive, so MemPool teardown is not deferred to
# interpreter finalization (pytorch/pytorch#145168).
from vllm.device_allocator.cumem import CuMemAllocator
if current_platform.is_cuda_alike():
from vllm.device_allocator.cumem import CuMemAllocator
if CuMemAllocator.instance is not None:
CuMemAllocator.instance.release_pools()
if CuMemAllocator.instance is not None:
CuMemAllocator.instance.release_pools()
def elastic_ep_execute(self, execute_method: str, *args, **kwargs):
return self.elastic_ep_executor.execute(execute_method, *args, **kwargs)
+17
View File
@@ -162,3 +162,20 @@ class XPUWorker(Worker):
logger.debug("Starting torch profiler with trace name: %s", trace_name)
super().profile(is_start=is_start, profile_prefix=profile_prefix)
def shutdown(self) -> None:
logger.info(
"XPUWorker shutdown: cleaning up (rank=%d, local_rank=%d)",
self.rank,
self.local_rank,
)
super().shutdown()
from vllm.device_allocator.xpumem import XpuMemAllocator
if XpuMemAllocator.instance is not None:
XpuMemAllocator.instance.release_pools()
logger.info(
"XPUWorker shutdown: done (rank=%d, local_rank=%d)",
self.rank,
self.local_rank,
)