From b337523c118d1cdb211732199f0a7d5b1e942bb9 Mon Sep 17 00:00:00 2001 From: Chaojun Zhang Date: Tue, 13 Oct 2026 13:12:10 +0800 Subject: [PATCH] [XPU] [Test] Extend spawn-safe sitecustomize monkeypatch to XPU (#57911) Signed-off-by: Chaojun Zhang Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: xiangdong <40376367+zxd1997066@users.noreply.github.com> --- .buildkite/intel_jobs/distributed_intel.yaml | 33 ++++++++++++++++++++ tests/v1/shutdown/conftest.py | 15 +++++++-- tests/v1/shutdown/test_forward_error.py | 8 ++--- tests/v1/shutdown/test_startup_error.py | 8 ++--- 4 files changed, 53 insertions(+), 11 deletions(-) create mode 100644 .buildkite/intel_jobs/distributed_intel.yaml diff --git a/.buildkite/intel_jobs/distributed_intel.yaml b/.buildkite/intel_jobs/distributed_intel.yaml new file mode 100644 index 0000000000..54800b4103 --- /dev/null +++ b/.buildkite/intel_jobs/distributed_intel.yaml @@ -0,0 +1,33 @@ +group: Distributed Intel +depends_on: + - image-build-xpu +steps: +- label: ":intel: (BMG) Distributed Torchrun + Shutdown" + key: distributed-torchrun-shutdown-tests-2-gpus + timeout_in_minutes: 50 + device: intel_gpu + agent_tags: + label: production + gpu: 2+ + mem: 24+ + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/distributed/ + - "!vllm/distributed/kv_transfer/" + - vllm/engine/ + - vllm/executor/ + - vllm/worker/worker_base.py + - vllm/v1/engine/ + - vllm/v1/worker/ + - tests/distributed/ + - tests/v1/shutdown + - tests/v1/worker/test_worker_memory_snapshot.py + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'cd tests && pytest -v -s v1/shutdown' diff --git a/tests/v1/shutdown/conftest.py b/tests/v1/shutdown/conftest.py index b276d0fab2..ef54b188fe 100644 --- a/tests/v1/shutdown/conftest.py +++ b/tests/v1/shutdown/conftest.py @@ -10,9 +10,18 @@ from vllm.platforms import current_platform @pytest.fixture -def rocm_sitecustomize_factory(monkeypatch, tmp_path: Path): - """Return a function that installs a given sitecustomize payload.""" - if not current_platform.is_rocm(): +def spawn_sitecustomize_factory(monkeypatch, tmp_path: Path): + """Return a function that installs a given sitecustomize payload. + + Both ROCm and XPU force the `spawn` multiprocessing start method (see + `vllm.platforms.rocm`/`vllm.platforms.xpu`), which re-imports modules in + child processes instead of inheriting them via `fork`'s copy-on-write + memory. A plain `monkeypatch.setattr` in the parent test process is + therefore invisible to spawned EngineCore/worker subprocesses on these + platforms, so the payload must be installed via `sitecustomize.py` on + `PYTHONPATH` instead. + """ + if not (current_platform.is_rocm() or current_platform.is_xpu()): return lambda _: None def install(lines: Iterable[str]) -> None: diff --git a/tests/v1/shutdown/test_forward_error.py b/tests/v1/shutdown/test_forward_error.py index 8bc09a64ca..c40a41cc3b 100644 --- a/tests/v1/shutdown/test_forward_error.py +++ b/tests/v1/shutdown/test_forward_error.py @@ -41,21 +41,21 @@ def evil_forward(self, *args, **kwargs): @pytest.fixture -def rocm_evil_forward(rocm_sitecustomize_factory): +def evil_forward_fixture(spawn_sitecustomize_factory): lines = [ "from vllm.distributed import get_tensor_model_parallel_rank", "from vllm.model_executor.models.llama import LlamaForCausalLM", inspect.getsource(evil_forward), f"LlamaForCausalLM.forward = {evil_forward.__name__}", ] - rocm_sitecustomize_factory(lines) + spawn_sitecustomize_factory(lines) @pytest.mark.asyncio @pytest.mark.parametrize("tensor_parallel_size", [2, 1]) @pytest.mark.parametrize("model", MODELS) async def test_async_llm_model_error( - monkeypatch, rocm_evil_forward, tensor_parallel_size: int, model: str + monkeypatch, evil_forward_fixture, tensor_parallel_size: int, model: str ) -> None: """Test that AsyncLLM propagates a forward pass error and frees memory. @@ -118,7 +118,7 @@ async def test_async_llm_model_error( @pytest.mark.parametrize("model", MODELS) def test_llm_model_error( monkeypatch, - rocm_evil_forward, + evil_forward_fixture, tensor_parallel_size: int, enable_multiprocessing: bool, model: str, diff --git a/tests/v1/shutdown/test_startup_error.py b/tests/v1/shutdown/test_startup_error.py index 6d6d0d91e7..d70f69409e 100644 --- a/tests/v1/shutdown/test_startup_error.py +++ b/tests/v1/shutdown/test_startup_error.py @@ -30,7 +30,7 @@ def evil_method(self, *args, **kwargs): @pytest.fixture -def rocm_evil_method(rocm_sitecustomize_factory, request): +def evil_method_fixture(spawn_sitecustomize_factory, request): failing_method = request.getfixturevalue("failing_method") lines = [ "from vllm.distributed import get_tensor_model_parallel_rank", @@ -38,7 +38,7 @@ def rocm_evil_method(rocm_sitecustomize_factory, request): inspect.getsource(evil_method), f"LlamaForCausalLM.{failing_method} = {evil_method.__name__}", ] - rocm_sitecustomize_factory(lines) + spawn_sitecustomize_factory(lines) @pytest.mark.timeout(SHUTDOWN_TEST_TIMEOUT_SEC) @@ -47,7 +47,7 @@ def rocm_evil_method(rocm_sitecustomize_factory, request): @pytest.mark.parametrize("failing_method", ["forward", "load_weights"]) def test_async_llm_startup_error( monkeypatch, - rocm_evil_method, + evil_method_fixture, model: str, tensor_parallel_size: int, failing_method: str, @@ -84,7 +84,7 @@ def test_async_llm_startup_error( @pytest.mark.parametrize("failing_method", ["forward", "load_weights"]) def test_llm_startup_error( monkeypatch, - rocm_evil_method, + evil_method_fixture, model: str, tensor_parallel_size: int, enable_multiprocessing: bool,