[XPU] [Test] Extend spawn-safe sitecustomize monkeypatch to XPU (#57911)
Signed-off-by: Chaojun Zhang <chaojun.zhang@intel.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: xiangdong <40376367+zxd1997066@users.noreply.github.com>
This commit is contained in:
co-authored by
Copilot
xiangdong
parent
5555a2d8f7
commit
b337523c11
@@ -0,0 +1,33 @@
|
||||
group: Distributed Intel
|
||||
depends_on:
|
||||
- image-build-xpu
|
||||
steps:
|
||||
- label: ":intel: (BMG) Distributed Torchrun + Shutdown"
|
||||
key: distributed-torchrun-shutdown-tests-2-gpus
|
||||
timeout_in_minutes: 50
|
||||
device: intel_gpu
|
||||
agent_tags:
|
||||
label: production
|
||||
gpu: 2+
|
||||
mem: 24+
|
||||
no_plugin: true
|
||||
working_dir: "."
|
||||
env:
|
||||
REGISTRY: "public.ecr.aws/q9t5s3a7"
|
||||
REPO: "vllm-ci-test-repo"
|
||||
VLLM_TEST_DEVICE: "xpu"
|
||||
source_file_dependencies:
|
||||
- vllm/distributed/
|
||||
- "!vllm/distributed/kv_transfer/"
|
||||
- vllm/engine/
|
||||
- vllm/executor/
|
||||
- vllm/worker/worker_base.py
|
||||
- vllm/v1/engine/
|
||||
- vllm/v1/worker/
|
||||
- tests/distributed/
|
||||
- tests/v1/shutdown
|
||||
- tests/v1/worker/test_worker_memory_snapshot.py
|
||||
commands:
|
||||
- >-
|
||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||
'cd tests && pytest -v -s v1/shutdown'
|
||||
@@ -10,9 +10,18 @@ from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def rocm_sitecustomize_factory(monkeypatch, tmp_path: Path):
|
||||
"""Return a function that installs a given sitecustomize payload."""
|
||||
if not current_platform.is_rocm():
|
||||
def spawn_sitecustomize_factory(monkeypatch, tmp_path: Path):
|
||||
"""Return a function that installs a given sitecustomize payload.
|
||||
|
||||
Both ROCm and XPU force the `spawn` multiprocessing start method (see
|
||||
`vllm.platforms.rocm`/`vllm.platforms.xpu`), which re-imports modules in
|
||||
child processes instead of inheriting them via `fork`'s copy-on-write
|
||||
memory. A plain `monkeypatch.setattr` in the parent test process is
|
||||
therefore invisible to spawned EngineCore/worker subprocesses on these
|
||||
platforms, so the payload must be installed via `sitecustomize.py` on
|
||||
`PYTHONPATH` instead.
|
||||
"""
|
||||
if not (current_platform.is_rocm() or current_platform.is_xpu()):
|
||||
return lambda _: None
|
||||
|
||||
def install(lines: Iterable[str]) -> None:
|
||||
|
||||
@@ -41,21 +41,21 @@ def evil_forward(self, *args, **kwargs):
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def rocm_evil_forward(rocm_sitecustomize_factory):
|
||||
def evil_forward_fixture(spawn_sitecustomize_factory):
|
||||
lines = [
|
||||
"from vllm.distributed import get_tensor_model_parallel_rank",
|
||||
"from vllm.model_executor.models.llama import LlamaForCausalLM",
|
||||
inspect.getsource(evil_forward),
|
||||
f"LlamaForCausalLM.forward = {evil_forward.__name__}",
|
||||
]
|
||||
rocm_sitecustomize_factory(lines)
|
||||
spawn_sitecustomize_factory(lines)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("tensor_parallel_size", [2, 1])
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
async def test_async_llm_model_error(
|
||||
monkeypatch, rocm_evil_forward, tensor_parallel_size: int, model: str
|
||||
monkeypatch, evil_forward_fixture, tensor_parallel_size: int, model: str
|
||||
) -> None:
|
||||
"""Test that AsyncLLM propagates a forward pass error and frees memory.
|
||||
|
||||
@@ -118,7 +118,7 @@ async def test_async_llm_model_error(
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
def test_llm_model_error(
|
||||
monkeypatch,
|
||||
rocm_evil_forward,
|
||||
evil_forward_fixture,
|
||||
tensor_parallel_size: int,
|
||||
enable_multiprocessing: bool,
|
||||
model: str,
|
||||
|
||||
@@ -30,7 +30,7 @@ def evil_method(self, *args, **kwargs):
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def rocm_evil_method(rocm_sitecustomize_factory, request):
|
||||
def evil_method_fixture(spawn_sitecustomize_factory, request):
|
||||
failing_method = request.getfixturevalue("failing_method")
|
||||
lines = [
|
||||
"from vllm.distributed import get_tensor_model_parallel_rank",
|
||||
@@ -38,7 +38,7 @@ def rocm_evil_method(rocm_sitecustomize_factory, request):
|
||||
inspect.getsource(evil_method),
|
||||
f"LlamaForCausalLM.{failing_method} = {evil_method.__name__}",
|
||||
]
|
||||
rocm_sitecustomize_factory(lines)
|
||||
spawn_sitecustomize_factory(lines)
|
||||
|
||||
|
||||
@pytest.mark.timeout(SHUTDOWN_TEST_TIMEOUT_SEC)
|
||||
@@ -47,7 +47,7 @@ def rocm_evil_method(rocm_sitecustomize_factory, request):
|
||||
@pytest.mark.parametrize("failing_method", ["forward", "load_weights"])
|
||||
def test_async_llm_startup_error(
|
||||
monkeypatch,
|
||||
rocm_evil_method,
|
||||
evil_method_fixture,
|
||||
model: str,
|
||||
tensor_parallel_size: int,
|
||||
failing_method: str,
|
||||
@@ -84,7 +84,7 @@ def test_async_llm_startup_error(
|
||||
@pytest.mark.parametrize("failing_method", ["forward", "load_weights"])
|
||||
def test_llm_startup_error(
|
||||
monkeypatch,
|
||||
rocm_evil_method,
|
||||
evil_method_fixture,
|
||||
model: str,
|
||||
tensor_parallel_size: int,
|
||||
enable_multiprocessing: bool,
|
||||
|
||||
Reference in New Issue
Block a user