[ROCm][CI] Defer AITER sampler import and isolate server test PYTHONPATH (#44823)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
+47
-3
@@ -18,7 +18,7 @@ import tempfile
|
||||
import threading
|
||||
import time
|
||||
import warnings
|
||||
from collections.abc import Callable, Iterable, Sequence
|
||||
from collections.abc import Callable, Iterable, MutableMapping, Sequence
|
||||
from contextlib import ExitStack, contextmanager
|
||||
from multiprocessing import Process, get_context
|
||||
from pathlib import Path
|
||||
@@ -149,6 +149,46 @@ ROCM_ENGINE_KWARGS: dict = (
|
||||
if current_platform.is_rocm()
|
||||
else {}
|
||||
)
|
||||
_TILELANG_TVM_PYTHONPATH_FRAGMENT = os.path.join(
|
||||
"tilelang", "3rdparty", "tvm", "python"
|
||||
)
|
||||
|
||||
|
||||
def _sanitize_pythonpath_value(pythonpath: str | None) -> str:
|
||||
if not pythonpath:
|
||||
return ""
|
||||
entries = []
|
||||
for entry in pythonpath.split(os.pathsep):
|
||||
normalized = entry.replace(os.sep, "/")
|
||||
if _TILELANG_TVM_PYTHONPATH_FRAGMENT.replace(os.sep, "/") in normalized:
|
||||
continue
|
||||
entries.append(entry)
|
||||
return os.pathsep.join(entries)
|
||||
|
||||
|
||||
def _sanitize_pythonpath_env(env: MutableMapping[str, str]) -> None:
|
||||
cleaned = _sanitize_pythonpath_value(env.get("PYTHONPATH"))
|
||||
if cleaned:
|
||||
env["PYTHONPATH"] = cleaned
|
||||
else:
|
||||
env.pop("PYTHONPATH", None)
|
||||
|
||||
|
||||
def _sanitize_current_pythonpath_env() -> None:
|
||||
_sanitize_pythonpath_env(os.environ)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _temporarily_sanitized_pythonpath_env():
|
||||
original = os.environ.get("PYTHONPATH")
|
||||
_sanitize_current_pythonpath_env()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
if original is None:
|
||||
os.environ.pop("PYTHONPATH", None)
|
||||
else:
|
||||
os.environ["PYTHONPATH"] = original
|
||||
|
||||
|
||||
def requires_spawn_multiprocessing() -> bool:
|
||||
@@ -253,7 +293,8 @@ class RemoteVLLMServer:
|
||||
getattr(args, "show_hidden_metrics_for_version", None) is not None
|
||||
)
|
||||
|
||||
self._pre_download_model(model, args)
|
||||
with _temporarily_sanitized_pythonpath_env():
|
||||
self._pre_download_model(model, args)
|
||||
self._shutdown_complete = False
|
||||
|
||||
# Record GPU memory before server start so we know what
|
||||
@@ -727,6 +768,7 @@ class RemoteOpenAIServer(RemoteVLLMServer):
|
||||
env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
if env_dict is not None:
|
||||
env.update(env_dict)
|
||||
_sanitize_pythonpath_env(env)
|
||||
serve_cmd = ["vllm", "serve", model, *vllm_serve_args]
|
||||
print(f"Launching RemoteOpenAIServer with: {' '.join(serve_cmd)}")
|
||||
print(f"Environment variables: {env}")
|
||||
@@ -754,6 +796,7 @@ class RemoteLaunchRenderServer(RemoteVLLMServer):
|
||||
env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
if env_dict is not None:
|
||||
env.update(env_dict)
|
||||
_sanitize_pythonpath_env(env)
|
||||
serve_cmd = ["vllm", "launch", "render", model, *vllm_serve_args]
|
||||
print(f"Launching RemoteLaunchRenderServer with: {' '.join(serve_cmd)}")
|
||||
self.proc: subprocess.Popen = subprocess.Popen(
|
||||
@@ -795,7 +838,8 @@ class RemoteOpenAIServerCustom(RemoteOpenAIServer):
|
||||
target=_run_in_new_process_group,
|
||||
args=(self.child_process_fxn, env_dict, model, vllm_serve_args),
|
||||
) # type: ignore[assignment]
|
||||
self.proc.start()
|
||||
with _temporarily_sanitized_pythonpath_env():
|
||||
self.proc.start()
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -64,6 +64,51 @@ def test_sampler_threads_fp64_gumbel_to_topk_topp_sampler():
|
||||
assert sampler.topk_topp_sampler.use_fp64_gumbel
|
||||
|
||||
|
||||
def test_rocm_aiter_sampler_defers_import_when_generators_force_native(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
from vllm.v1.sample.ops import topk_topp_sampler
|
||||
|
||||
class MockPlatform:
|
||||
@staticmethod
|
||||
def is_cuda():
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def is_cpu():
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def is_xpu():
|
||||
return False
|
||||
|
||||
class MockRocmAiterOps:
|
||||
@staticmethod
|
||||
def is_enabled():
|
||||
return True
|
||||
|
||||
real_import = __import__
|
||||
|
||||
def guard_aiter_sampling_import(name, *args, **kwargs):
|
||||
if name == "aiter.ops.sampling":
|
||||
raise AssertionError("aiter sampling import should be deferred")
|
||||
return real_import(name, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(topk_topp_sampler, "current_platform", MockPlatform())
|
||||
monkeypatch.setattr(topk_topp_sampler, "rocm_aiter_ops", MockRocmAiterOps())
|
||||
monkeypatch.setattr("builtins.__import__", guard_aiter_sampling_import)
|
||||
|
||||
sampler = topk_topp_sampler.TopKTopPSampler()
|
||||
logits = torch.randn(2, 8)
|
||||
k = torch.full((2,), 2, dtype=torch.int32)
|
||||
generators = {0: torch.Generator(device=logits.device).manual_seed(0)}
|
||||
|
||||
token_ids, logits_to_return = sampler(logits, generators, k, None)
|
||||
|
||||
assert token_ids.shape == (2,)
|
||||
assert logits_to_return is None
|
||||
|
||||
|
||||
def test_random_sample_uses_fp64_exponential_race_when_requested():
|
||||
torch.set_default_device(DEVICE_TYPE)
|
||||
probs = torch.tensor(
|
||||
|
||||
@@ -111,20 +111,12 @@ class TopKTopPSampler(nn.Module):
|
||||
logprobs_mode not in ("processed_logits", "processed_logprobs")
|
||||
and rocm_aiter_ops.is_enabled()
|
||||
):
|
||||
try:
|
||||
import aiter.ops.sampling # noqa: F401
|
||||
|
||||
self.aiter_ops = torch.ops.aiter
|
||||
logger.info_once(
|
||||
"Using aiter sampler on ROCm (lazy import, sampling-only)."
|
||||
)
|
||||
self.forward = self.forward_hip
|
||||
except ImportError:
|
||||
logger.warning_once(
|
||||
"aiter.ops.sampling is not available on ROCm. "
|
||||
"Falling back to forward_native implementation."
|
||||
)
|
||||
self.forward = self.forward_native
|
||||
self.aiter_ops = None
|
||||
self._aiter_ops_import_failed = False
|
||||
logger.info_once(
|
||||
"Using aiter sampler on ROCm (lazy import, sampling-only)."
|
||||
)
|
||||
self.forward = self.forward_hip
|
||||
else:
|
||||
self.forward = self.forward_native
|
||||
|
||||
@@ -211,6 +203,22 @@ class TopKTopPSampler(nn.Module):
|
||||
|
||||
return sample_with_exponential_noise(probs, q), logits_to_return
|
||||
|
||||
def _init_aiter_ops(self) -> bool:
|
||||
if self._aiter_ops_import_failed:
|
||||
return False
|
||||
try:
|
||||
import aiter.ops.sampling # noqa: F401
|
||||
except ImportError:
|
||||
self._aiter_ops_import_failed = True
|
||||
self.forward = self.forward_native
|
||||
logger.warning_once(
|
||||
"aiter.ops.sampling is not available on ROCm. "
|
||||
"Falling back to PyTorch-native implementation."
|
||||
)
|
||||
return False
|
||||
self.aiter_ops = torch.ops.aiter
|
||||
return True
|
||||
|
||||
def forward_hip(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
@@ -232,6 +240,8 @@ class TopKTopPSampler(nn.Module):
|
||||
"processed_logits",
|
||||
"processed_logprobs",
|
||||
), "aiter sampler does not support returning logits/logprobs."
|
||||
if self.aiter_ops is None and not self._init_aiter_ops():
|
||||
return self.forward_native(logits, generators, k, p)
|
||||
return self.aiter_sample(logits, k, p, generators), None
|
||||
|
||||
def aiter_sample(
|
||||
@@ -242,6 +252,7 @@ class TopKTopPSampler(nn.Module):
|
||||
generators: dict[int, torch.Generator],
|
||||
) -> torch.Tensor:
|
||||
"""Sample from logits using aiter ops."""
|
||||
assert self.aiter_ops is not None
|
||||
use_top_k = k is not None
|
||||
use_top_p = p is not None
|
||||
# Joint k+p path
|
||||
|
||||
Reference in New Issue
Block a user