From 8cc242335de805cac390580f0dcd9e69b6ed86c0 Mon Sep 17 00:00:00 2001 From: Chaojun Zhang Date: Tue, 30 Jun 2026 15:27:21 +0800 Subject: [PATCH] [XPU] Optimize XPU worker shutdown logic to prevent resource leak (#46433) Signed-off-by: Chaojun Zhang --- .buildkite/intel_jobs/basic_correctness.yaml | 1 + .buildkite/intel_jobs/lora_intel.yaml | 2 +- tests/utils.py | 7 +++++++ vllm/platforms/xpu.py | 10 ++++++++++ vllm/v1/worker/gpu_model_runner.py | 2 +- vllm/v1/worker/gpu_worker.py | 7 ++++--- vllm/v1/worker/xpu_worker.py | 17 +++++++++++++++++ 7 files changed, 41 insertions(+), 5 deletions(-) diff --git a/.buildkite/intel_jobs/basic_correctness.yaml b/.buildkite/intel_jobs/basic_correctness.yaml index 1a4a0915ac..fa472a7d3b 100644 --- a/.buildkite/intel_jobs/basic_correctness.yaml +++ b/.buildkite/intel_jobs/basic_correctness.yaml @@ -23,4 +23,5 @@ steps: bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && export VLLM_WORKER_MULTIPROC_METHOD=spawn && + pytest -v -s basic_correctness/test_cpu_offload.py && pytest -v -s basic_correctness/test_mem.py::test_end_to_end' diff --git a/.buildkite/intel_jobs/lora_intel.yaml b/.buildkite/intel_jobs/lora_intel.yaml index bdfb38b0bd..93385ecc8a 100644 --- a/.buildkite/intel_jobs/lora_intel.yaml +++ b/.buildkite/intel_jobs/lora_intel.yaml @@ -128,10 +128,10 @@ steps: bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && export VLLM_WORKER_MULTIPROC_METHOD=spawn && - (pytest -v -s lora/test_mixtral.py --deselect="tests/lora/test_mixtral.py::test_mixtral_lora[4]" || true) && pytest -v -s lora/test_quant_model.py --deselect="tests/lora/test_quant_model.py::test_quant_model_lora[model0]" --deselect="tests/lora/test_quant_model.py::test_quant_model_lora[model1]" --deselect="tests/lora/test_quant_model.py::test_quant_model_tp_equality[model0]" && pytest -v -s lora/test_transformers_model.py && pytest -v -s lora/test_chatglm3_tp.py && + pytest -v -s lora/test_llama_tp.py::test_llama_lora && pytest -s -v lora/test_minicpmv_tp.py' - label: LoRA Multimodal diff --git a/tests/utils.py b/tests/utils.py index 07cda56ce0..08579f99e4 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -604,6 +604,13 @@ class RemoteVLLMServer: mem_info = nvmlDeviceGetMemoryInfo(handle) total_used += mem_info.used return total_used + elif current_platform.is_xpu(): + total_used = 0 + device_count = current_platform.device_count() + for i in range(device_count): + free, total = torch.xpu.mem_get_info(i) + total_used += total - free + return total_used except Exception as e: print(f"[RemoteOpenAIServer] Could not query GPU memory: {e}") return None diff --git a/vllm/platforms/xpu.py b/vllm/platforms/xpu.py index 94f5e8e5a8..7f5709f5cf 100644 --- a/vllm/platforms/xpu.py +++ b/vllm/platforms/xpu.py @@ -243,6 +243,16 @@ class XPUPlatform(Platform): if "VLLM_WORKER_MULTIPROC_METHOD" not in os.environ: os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + # XPU requires graceful shutdown to allow oneCCL/Level Zero resources + # to be properly released. Without this, subsequent server startups on + # the same devices may hang during CCL initialization. + if vllm_config.shutdown_timeout == 0: + vllm_config.shutdown_timeout = 5 + logger.info( + "XPU platform: set server shutdown_timeout=%d.", + vllm_config.shutdown_timeout, + ) + @classmethod def update_block_size_for_backend(cls, vllm_config: "VllmConfig") -> None: super().update_block_size_for_backend(vllm_config) diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index a772a897a1..68173eaef0 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -6385,7 +6385,7 @@ class GPUModelRunner( _ROPE_DICT.clear() reset_workspace_manager() - if current_platform.is_rocm(): + if current_platform.is_rocm() or current_platform.is_xpu(): gc.collect() torch.accelerator.empty_cache() torch.accelerator.synchronize() diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 2947101d97..0c5512d5e1 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -1294,10 +1294,11 @@ class Worker(WorkerBase): # Release kept-alive cumem pools while the pluggable allocator wrappers # and callbacks are still alive, so MemPool teardown is not deferred to # interpreter finalization (pytorch/pytorch#145168). - from vllm.device_allocator.cumem import CuMemAllocator + if current_platform.is_cuda_alike(): + from vllm.device_allocator.cumem import CuMemAllocator - if CuMemAllocator.instance is not None: - CuMemAllocator.instance.release_pools() + if CuMemAllocator.instance is not None: + CuMemAllocator.instance.release_pools() def elastic_ep_execute(self, execute_method: str, *args, **kwargs): return self.elastic_ep_executor.execute(execute_method, *args, **kwargs) diff --git a/vllm/v1/worker/xpu_worker.py b/vllm/v1/worker/xpu_worker.py index 555c602278..e669365890 100644 --- a/vllm/v1/worker/xpu_worker.py +++ b/vllm/v1/worker/xpu_worker.py @@ -162,3 +162,20 @@ class XPUWorker(Worker): logger.debug("Starting torch profiler with trace name: %s", trace_name) super().profile(is_start=is_start, profile_prefix=profile_prefix) + + def shutdown(self) -> None: + logger.info( + "XPUWorker shutdown: cleaning up (rank=%d, local_rank=%d)", + self.rank, + self.local_rank, + ) + super().shutdown() + from vllm.device_allocator.xpumem import XpuMemAllocator + + if XpuMemAllocator.instance is not None: + XpuMemAllocator.instance.release_pools() + logger.info( + "XPUWorker shutdown: done (rank=%d, local_rank=%d)", + self.rank, + self.local_rank, + )