[ROCm][CI] Defer AITER sampler import and isolate server test PYTHONPATH (#44823)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
2026-06-10 08:56:11 +00:00
committed by GitHub
parent af9f583344
commit 82a42234be
3 changed files with 117 additions and 17 deletions
+47 -3
View File
@@ -18,7 +18,7 @@ import tempfile
import threading
import time
import warnings
from collections.abc import Callable, Iterable, Sequence
from collections.abc import Callable, Iterable, MutableMapping, Sequence
from contextlib import ExitStack, contextmanager
from multiprocessing import Process, get_context
from pathlib import Path
@@ -149,6 +149,46 @@ ROCM_ENGINE_KWARGS: dict = (
if current_platform.is_rocm()
else {}
)
_TILELANG_TVM_PYTHONPATH_FRAGMENT = os.path.join(
"tilelang", "3rdparty", "tvm", "python"
)
def _sanitize_pythonpath_value(pythonpath: str | None) -> str:
if not pythonpath:
return ""
entries = []
for entry in pythonpath.split(os.pathsep):
normalized = entry.replace(os.sep, "/")
if _TILELANG_TVM_PYTHONPATH_FRAGMENT.replace(os.sep, "/") in normalized:
continue
entries.append(entry)
return os.pathsep.join(entries)
def _sanitize_pythonpath_env(env: MutableMapping[str, str]) -> None:
cleaned = _sanitize_pythonpath_value(env.get("PYTHONPATH"))
if cleaned:
env["PYTHONPATH"] = cleaned
else:
env.pop("PYTHONPATH", None)
def _sanitize_current_pythonpath_env() -> None:
_sanitize_pythonpath_env(os.environ)
@contextmanager
def _temporarily_sanitized_pythonpath_env():
original = os.environ.get("PYTHONPATH")
_sanitize_current_pythonpath_env()
try:
yield
finally:
if original is None:
os.environ.pop("PYTHONPATH", None)
else:
os.environ["PYTHONPATH"] = original
def requires_spawn_multiprocessing() -> bool:
@@ -253,7 +293,8 @@ class RemoteVLLMServer:
getattr(args, "show_hidden_metrics_for_version", None) is not None
)
self._pre_download_model(model, args)
with _temporarily_sanitized_pythonpath_env():
self._pre_download_model(model, args)
self._shutdown_complete = False
# Record GPU memory before server start so we know what
@@ -727,6 +768,7 @@ class RemoteOpenAIServer(RemoteVLLMServer):
env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
if env_dict is not None:
env.update(env_dict)
_sanitize_pythonpath_env(env)
serve_cmd = ["vllm", "serve", model, *vllm_serve_args]
print(f"Launching RemoteOpenAIServer with: {' '.join(serve_cmd)}")
print(f"Environment variables: {env}")
@@ -754,6 +796,7 @@ class RemoteLaunchRenderServer(RemoteVLLMServer):
env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
if env_dict is not None:
env.update(env_dict)
_sanitize_pythonpath_env(env)
serve_cmd = ["vllm", "launch", "render", model, *vllm_serve_args]
print(f"Launching RemoteLaunchRenderServer with: {' '.join(serve_cmd)}")
self.proc: subprocess.Popen = subprocess.Popen(
@@ -795,7 +838,8 @@ class RemoteOpenAIServerCustom(RemoteOpenAIServer):
target=_run_in_new_process_group,
args=(self.child_process_fxn, env_dict, model, vllm_serve_args),
) # type: ignore[assignment]
self.proc.start()
with _temporarily_sanitized_pythonpath_env():
self.proc.start()
def __init__(
self,
+45
View File
@@ -64,6 +64,51 @@ def test_sampler_threads_fp64_gumbel_to_topk_topp_sampler():
assert sampler.topk_topp_sampler.use_fp64_gumbel
def test_rocm_aiter_sampler_defers_import_when_generators_force_native(
monkeypatch: pytest.MonkeyPatch,
):
from vllm.v1.sample.ops import topk_topp_sampler
class MockPlatform:
@staticmethod
def is_cuda():
return False
@staticmethod
def is_cpu():
return False
@staticmethod
def is_xpu():
return False
class MockRocmAiterOps:
@staticmethod
def is_enabled():
return True
real_import = __import__
def guard_aiter_sampling_import(name, *args, **kwargs):
if name == "aiter.ops.sampling":
raise AssertionError("aiter sampling import should be deferred")
return real_import(name, *args, **kwargs)
monkeypatch.setattr(topk_topp_sampler, "current_platform", MockPlatform())
monkeypatch.setattr(topk_topp_sampler, "rocm_aiter_ops", MockRocmAiterOps())
monkeypatch.setattr("builtins.__import__", guard_aiter_sampling_import)
sampler = topk_topp_sampler.TopKTopPSampler()
logits = torch.randn(2, 8)
k = torch.full((2,), 2, dtype=torch.int32)
generators = {0: torch.Generator(device=logits.device).manual_seed(0)}
token_ids, logits_to_return = sampler(logits, generators, k, None)
assert token_ids.shape == (2,)
assert logits_to_return is None
def test_random_sample_uses_fp64_exponential_race_when_requested():
torch.set_default_device(DEVICE_TYPE)
probs = torch.tensor(
+25 -14
View File
@@ -111,20 +111,12 @@ class TopKTopPSampler(nn.Module):
logprobs_mode not in ("processed_logits", "processed_logprobs")
and rocm_aiter_ops.is_enabled()
):
try:
import aiter.ops.sampling # noqa: F401
self.aiter_ops = torch.ops.aiter
logger.info_once(
"Using aiter sampler on ROCm (lazy import, sampling-only)."
)
self.forward = self.forward_hip
except ImportError:
logger.warning_once(
"aiter.ops.sampling is not available on ROCm. "
"Falling back to forward_native implementation."
)
self.forward = self.forward_native
self.aiter_ops = None
self._aiter_ops_import_failed = False
logger.info_once(
"Using aiter sampler on ROCm (lazy import, sampling-only)."
)
self.forward = self.forward_hip
else:
self.forward = self.forward_native
@@ -211,6 +203,22 @@ class TopKTopPSampler(nn.Module):
return sample_with_exponential_noise(probs, q), logits_to_return
def _init_aiter_ops(self) -> bool:
if self._aiter_ops_import_failed:
return False
try:
import aiter.ops.sampling # noqa: F401
except ImportError:
self._aiter_ops_import_failed = True
self.forward = self.forward_native
logger.warning_once(
"aiter.ops.sampling is not available on ROCm. "
"Falling back to PyTorch-native implementation."
)
return False
self.aiter_ops = torch.ops.aiter
return True
def forward_hip(
self,
logits: torch.Tensor,
@@ -232,6 +240,8 @@ class TopKTopPSampler(nn.Module):
"processed_logits",
"processed_logprobs",
), "aiter sampler does not support returning logits/logprobs."
if self.aiter_ops is None and not self._init_aiter_ops():
return self.forward_native(logits, generators, k, p)
return self.aiter_sample(logits, k, p, generators), None
def aiter_sample(
@@ -242,6 +252,7 @@ class TopKTopPSampler(nn.Module):
generators: dict[int, torch.Generator],
) -> torch.Tensor:
"""Sample from logits using aiter ops."""
assert self.aiter_ops is not None
use_top_k = k is not None
use_top_p = p is not None
# Joint k+p path