From 82a42234be76edb07b509006de7809475d878b71 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Wed, 10 Jun 2026 03:56:11 -0500 Subject: [PATCH] [ROCm][CI] Defer AITER sampler import and isolate server test PYTHONPATH (#44823) Signed-off-by: Andreas Karatzas --- tests/utils.py | 50 +++++++++++++++++++++-- tests/v1/sample/test_topk_topp_sampler.py | 45 ++++++++++++++++++++ vllm/v1/sample/ops/topk_topp_sampler.py | 39 +++++++++++------- 3 files changed, 117 insertions(+), 17 deletions(-) diff --git a/tests/utils.py b/tests/utils.py index 5b18435398..db5905b927 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -18,7 +18,7 @@ import tempfile import threading import time import warnings -from collections.abc import Callable, Iterable, Sequence +from collections.abc import Callable, Iterable, MutableMapping, Sequence from contextlib import ExitStack, contextmanager from multiprocessing import Process, get_context from pathlib import Path @@ -149,6 +149,46 @@ ROCM_ENGINE_KWARGS: dict = ( if current_platform.is_rocm() else {} ) +_TILELANG_TVM_PYTHONPATH_FRAGMENT = os.path.join( + "tilelang", "3rdparty", "tvm", "python" +) + + +def _sanitize_pythonpath_value(pythonpath: str | None) -> str: + if not pythonpath: + return "" + entries = [] + for entry in pythonpath.split(os.pathsep): + normalized = entry.replace(os.sep, "/") + if _TILELANG_TVM_PYTHONPATH_FRAGMENT.replace(os.sep, "/") in normalized: + continue + entries.append(entry) + return os.pathsep.join(entries) + + +def _sanitize_pythonpath_env(env: MutableMapping[str, str]) -> None: + cleaned = _sanitize_pythonpath_value(env.get("PYTHONPATH")) + if cleaned: + env["PYTHONPATH"] = cleaned + else: + env.pop("PYTHONPATH", None) + + +def _sanitize_current_pythonpath_env() -> None: + _sanitize_pythonpath_env(os.environ) + + +@contextmanager +def _temporarily_sanitized_pythonpath_env(): + original = os.environ.get("PYTHONPATH") + _sanitize_current_pythonpath_env() + try: + yield + finally: + if original is None: + os.environ.pop("PYTHONPATH", None) + else: + os.environ["PYTHONPATH"] = original def requires_spawn_multiprocessing() -> bool: @@ -253,7 +293,8 @@ class RemoteVLLMServer: getattr(args, "show_hidden_metrics_for_version", None) is not None ) - self._pre_download_model(model, args) + with _temporarily_sanitized_pythonpath_env(): + self._pre_download_model(model, args) self._shutdown_complete = False # Record GPU memory before server start so we know what @@ -727,6 +768,7 @@ class RemoteOpenAIServer(RemoteVLLMServer): env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" if env_dict is not None: env.update(env_dict) + _sanitize_pythonpath_env(env) serve_cmd = ["vllm", "serve", model, *vllm_serve_args] print(f"Launching RemoteOpenAIServer with: {' '.join(serve_cmd)}") print(f"Environment variables: {env}") @@ -754,6 +796,7 @@ class RemoteLaunchRenderServer(RemoteVLLMServer): env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" if env_dict is not None: env.update(env_dict) + _sanitize_pythonpath_env(env) serve_cmd = ["vllm", "launch", "render", model, *vllm_serve_args] print(f"Launching RemoteLaunchRenderServer with: {' '.join(serve_cmd)}") self.proc: subprocess.Popen = subprocess.Popen( @@ -795,7 +838,8 @@ class RemoteOpenAIServerCustom(RemoteOpenAIServer): target=_run_in_new_process_group, args=(self.child_process_fxn, env_dict, model, vllm_serve_args), ) # type: ignore[assignment] - self.proc.start() + with _temporarily_sanitized_pythonpath_env(): + self.proc.start() def __init__( self, diff --git a/tests/v1/sample/test_topk_topp_sampler.py b/tests/v1/sample/test_topk_topp_sampler.py index 554649b5e1..047e2b754e 100644 --- a/tests/v1/sample/test_topk_topp_sampler.py +++ b/tests/v1/sample/test_topk_topp_sampler.py @@ -64,6 +64,51 @@ def test_sampler_threads_fp64_gumbel_to_topk_topp_sampler(): assert sampler.topk_topp_sampler.use_fp64_gumbel +def test_rocm_aiter_sampler_defers_import_when_generators_force_native( + monkeypatch: pytest.MonkeyPatch, +): + from vllm.v1.sample.ops import topk_topp_sampler + + class MockPlatform: + @staticmethod + def is_cuda(): + return False + + @staticmethod + def is_cpu(): + return False + + @staticmethod + def is_xpu(): + return False + + class MockRocmAiterOps: + @staticmethod + def is_enabled(): + return True + + real_import = __import__ + + def guard_aiter_sampling_import(name, *args, **kwargs): + if name == "aiter.ops.sampling": + raise AssertionError("aiter sampling import should be deferred") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(topk_topp_sampler, "current_platform", MockPlatform()) + monkeypatch.setattr(topk_topp_sampler, "rocm_aiter_ops", MockRocmAiterOps()) + monkeypatch.setattr("builtins.__import__", guard_aiter_sampling_import) + + sampler = topk_topp_sampler.TopKTopPSampler() + logits = torch.randn(2, 8) + k = torch.full((2,), 2, dtype=torch.int32) + generators = {0: torch.Generator(device=logits.device).manual_seed(0)} + + token_ids, logits_to_return = sampler(logits, generators, k, None) + + assert token_ids.shape == (2,) + assert logits_to_return is None + + def test_random_sample_uses_fp64_exponential_race_when_requested(): torch.set_default_device(DEVICE_TYPE) probs = torch.tensor( diff --git a/vllm/v1/sample/ops/topk_topp_sampler.py b/vllm/v1/sample/ops/topk_topp_sampler.py index 6f324ce985..69b35830ad 100644 --- a/vllm/v1/sample/ops/topk_topp_sampler.py +++ b/vllm/v1/sample/ops/topk_topp_sampler.py @@ -111,20 +111,12 @@ class TopKTopPSampler(nn.Module): logprobs_mode not in ("processed_logits", "processed_logprobs") and rocm_aiter_ops.is_enabled() ): - try: - import aiter.ops.sampling # noqa: F401 - - self.aiter_ops = torch.ops.aiter - logger.info_once( - "Using aiter sampler on ROCm (lazy import, sampling-only)." - ) - self.forward = self.forward_hip - except ImportError: - logger.warning_once( - "aiter.ops.sampling is not available on ROCm. " - "Falling back to forward_native implementation." - ) - self.forward = self.forward_native + self.aiter_ops = None + self._aiter_ops_import_failed = False + logger.info_once( + "Using aiter sampler on ROCm (lazy import, sampling-only)." + ) + self.forward = self.forward_hip else: self.forward = self.forward_native @@ -211,6 +203,22 @@ class TopKTopPSampler(nn.Module): return sample_with_exponential_noise(probs, q), logits_to_return + def _init_aiter_ops(self) -> bool: + if self._aiter_ops_import_failed: + return False + try: + import aiter.ops.sampling # noqa: F401 + except ImportError: + self._aiter_ops_import_failed = True + self.forward = self.forward_native + logger.warning_once( + "aiter.ops.sampling is not available on ROCm. " + "Falling back to PyTorch-native implementation." + ) + return False + self.aiter_ops = torch.ops.aiter + return True + def forward_hip( self, logits: torch.Tensor, @@ -232,6 +240,8 @@ class TopKTopPSampler(nn.Module): "processed_logits", "processed_logprobs", ), "aiter sampler does not support returning logits/logprobs." + if self.aiter_ops is None and not self._init_aiter_ops(): + return self.forward_native(logits, generators, k, p) return self.aiter_sample(logits, k, p, generators), None def aiter_sample( @@ -242,6 +252,7 @@ class TopKTopPSampler(nn.Module): generators: dict[int, torch.Generator], ) -> torch.Tensor: """Sample from logits using aiter ops.""" + assert self.aiter_ops is not None use_top_k = k is not None use_top_p = p is not None # Joint k+p path