diff --git a/tests/test_jit_monitor.py b/tests/test_jit_monitor.py index 8d3a241612..9f3285ddec 100644 --- a/tests/test_jit_monitor.py +++ b/tests/test_jit_monitor.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import inspect import os import sys from contextlib import contextmanager @@ -19,11 +20,15 @@ def _reset_monitor(): jit_monitor._mode = "warn" jit_monitor._verbose = False jit_monitor._cutedsl_hook_installed = False + jit_monitor._tilelang_hook_installed = False + jit_monitor._tilelang_jitimpl_compile_depth = 0 yield jit_monitor._active = False jit_monitor._mode = "warn" jit_monitor._verbose = False jit_monitor._cutedsl_hook_installed = False + jit_monitor._tilelang_hook_installed = False + jit_monitor._tilelang_jitimpl_compile_depth = 0 # ------------------------------------------------------------------ @@ -55,6 +60,45 @@ def _fake_cute_compile(*args, **kwargs): return "compiled" +def _fake_tilelang_import_modules(): + """Fake Python's TileLang modules touched by ``jit_monitor.activate``.""" + + class FakeJITKernel: + def __init__(self, *args, **kwargs): + pass + + class FakeJITImpl: + def __init__(self, func, signature): + self.func = func + self.signature = signature + self.mode = "lazy" + self._kernel_cache = {} + + def __call__(self, *args, **kwargs): + key, _ = self.func.parse_args(*args, **kwargs) + kernel = self._kernel_cache.get(key) + if kernel is None: + kernel = "compiled" + self._kernel_cache[key] = kernel + return kernel + + fake_kernel = cast(Any, ModuleType("tilelang.jit.kernel")) + fake_kernel.JITKernel = FakeJITKernel + + fake_jit = cast(Any, ModuleType("tilelang.jit")) + fake_jit.JITImpl = FakeJITImpl + fake_jit.kernel = fake_kernel + + fake_tilelang = cast(Any, ModuleType("tilelang")) + fake_tilelang.jit = fake_jit + + return { + "tilelang": fake_tilelang, + "tilelang.jit": fake_jit, + "tilelang.jit.kernel": fake_kernel, + } + + @contextmanager def _patch_jit_modules(fake_knobs, *, cute_compile=_fake_cute_compile): """Patch the Triton and CuTeDSL imports touched by ``jit_monitor.activate``.""" @@ -66,6 +110,7 @@ def _patch_jit_modules(fake_knobs, *, cute_compile=_fake_cute_compile): { "triton": fake_triton, **_fake_cute_import_modules(cute_compile), + **_fake_tilelang_import_modules(), }, ), mock.patch.object(jit_monitor, "HAS_TRITON", True), @@ -276,6 +321,116 @@ class TestCuTeDSLHook: cute.compile(lambda: None, "arg", option=True) +class TestTileLangHook: + def test_jit_kernel_logs_warning(self): + with _patch_jit_modules(_make_fake_knobs()): + from tilelang.jit.kernel import JITKernel + + func = SimpleNamespace(attrs={"global_symbol": "tl_kernel"}) + jit_monitor.activate() + with mock.patch.object(jit_monitor.logger, "warning_once") as warning_once: + JITKernel(func=func, out_idx=None, execution_backend="tvm_ffi") + + warning_once.assert_called_once() + msg = warning_once.call_args[0][0] % warning_once.call_args[0][1:] + assert "TileLang JIT compilation during inference" in msg + assert "tl_kernel" in msg + + def test_jit_impl_logs_warning(self): + with _patch_jit_modules(_make_fake_knobs()): + from tilelang.jit import JITImpl + + def tilelang_fn( + gemm_out_mul, + hidden_size: int, + n_splits: int = 1, + hc_mult: int = 4, + ): + return None + + class FakeFunc: + orig_func = tilelang_fn + + def parse_args(self, *args, **kwargs): + return ( + ( + "tilelang_key", + kwargs["hidden_size"], + kwargs.get("n_splits", 1), + ), + {}, + ) + + def set_mode(self, mode): + self.mode = mode + + tensor = SimpleNamespace( + shape=(2, 16, 24), + dtype="float32", + device="cuda:0", + ) + impl = JITImpl(FakeFunc(), inspect.signature(tilelang_fn)) + + jit_monitor.activate() + with ( + mock.patch.object(jit_monitor.logger, "warning_once") as warning_once, + mock.patch.object(jit_monitor.logger, "warning") as warning, + ): + impl(tensor, hidden_size=7168, n_splits=2) + + warning_once.assert_called_once() + warning.assert_not_called() + msg = warning_once.call_args[0][0] % warning_once.call_args[0][1:] + assert "TileLang JIT compilation during inference" in msg + assert "tilelang_fn" in msg + + def test_jit_impl_does_not_log_on_cache_hit(self): + with _patch_jit_modules(_make_fake_knobs()): + from tilelang.jit import JITImpl + + def tilelang_fn(gemm_out_mul, n_splits: int = 1): + return None + + class FakeFunc: + orig_func = tilelang_fn + + def parse_args(self, *args, **kwargs): + return (("tilelang_key", kwargs.get("n_splits", 1)), {}) + + def set_mode(self, mode): + self.mode = mode + + tensor = SimpleNamespace(shape=(2, 16, 24), dtype="float32") + impl = JITImpl(FakeFunc(), inspect.signature(tilelang_fn)) + + jit_monitor.activate() + with mock.patch.object(jit_monitor.logger, "warning_once") as warning_once: + impl(tensor, n_splits=2) + impl(tensor, n_splits=2) + + warning_once.assert_called_once() + + def test_from_database_does_not_log(self): + with _patch_jit_modules(_make_fake_knobs()): + from tilelang.jit.kernel import JITKernel + + func = SimpleNamespace(attrs={"global_symbol": "cached_tl_kernel"}) + jit_monitor.activate() + with mock.patch.object(jit_monitor.logger, "warning_once") as warning_once: + JITKernel(func=func, from_database=True) + + warning_once.assert_not_called() + + def test_error_mode_raises(self): + with _patch_jit_modules(_make_fake_knobs()): + from tilelang.jit.kernel import JITKernel + + func = SimpleNamespace(attrs={"global_symbol": "error_tl_kernel"}) + jit_monitor.activate(mode="error") + with pytest.raises(RuntimeError, match="TileLang JIT compilation"): + JITKernel(func=func) + + # ------------------------------------------------------------------ # Integration tests (real Triton + GPU) # ------------------------------------------------------------------ diff --git a/vllm/utils/jit_monitor.py b/vllm/utils/jit_monitor.py index 7ba8ecde65..8228e24c1c 100644 --- a/vllm/utils/jit_monitor.py +++ b/vllm/utils/jit_monitor.py @@ -18,13 +18,16 @@ Currently monitors: - Triton ``@triton.autotune`` cache misses (via ``knobs.autotuning.print``) - Triton ``@triton.jit`` first-time compilations (via ``knobs.runtime.jit_post_compile_hook``) +- TileLang ``@tilelang.jit`` first-time compilations """ import contextlib import functools +import importlib import os from collections.abc import Iterator, Mapping -from typing import Literal +from contextlib import suppress +from typing import Any, Literal, cast from vllm.logger import init_logger from vllm.triton_utils.importing import HAS_TRITON @@ -36,6 +39,8 @@ _active: bool = False _mode: JitMonitorMode = "warn" _verbose: bool = False _cutedsl_hook_installed: bool = False +_tilelang_hook_installed: bool = False +_tilelang_jitimpl_compile_depth: int = 0 def is_active() -> bool: @@ -69,6 +74,7 @@ def activate(*, mode: JitMonitorMode = "warn", verbose: bool = False) -> None: _setup_triton_autotuning_print() _setup_triton_jit_hook() _setup_cutedsl_jit_hook() + _setup_tilelang_jit_hook() logger.info( "Kernel JIT monitor activated; monitored JIT compilations during " @@ -290,6 +296,211 @@ def _setup_cutedsl_jit_hook() -> None: _cutedsl_hook_installed = True +# ------------------------------------------------------------------ +# TileLang JIT compilation hook +# ------------------------------------------------------------------ + + +def _tilelang_arg( + args: tuple[object, ...], + kwargs: Mapping[str, object], + index: int, + name: str, + default: object = None, +) -> object: + if len(args) > index: + return args[index] + return kwargs.get(name, default) + + +def _tilelang_kernel_name(func: object) -> str: + attrs = getattr(func, "attrs", None) + global_symbol = None + get_attr = getattr(attrs, "get", None) + if callable(get_attr): + try: + global_symbol = get_attr("global_symbol") + except Exception: + global_symbol = None + elif isinstance(attrs, Mapping): + global_symbol = attrs.get("global_symbol") + + if global_symbol is not None: + return str(global_symbol) + + name = getattr(func, "__name__", None) + if name is not None: + return str(name) + return func.__class__.__name__ if func is not None else "" + + +def _tilelang_call_kwargs(kwargs: Mapping[str, object]) -> dict[str, object]: + call_kwargs = dict(kwargs) + tune_params = call_kwargs.pop("__tune_params", {}) + if isinstance(tune_params, Mapping): + call_kwargs.update(tune_params) + return call_kwargs + + +def _tilelang_cache_miss_key( + jit_impl: object, + args: tuple[object, ...], + kwargs: Mapping[str, object], +) -> object | None: + if kwargs.get("__return_compile_arguments", False): + return None + + call_kwargs = _tilelang_call_kwargs(kwargs) + func = getattr(jit_impl, "func", None) + parse_args = getattr(func, "parse_args", None) + if not callable(parse_args): + return None + + try: + if getattr(jit_impl, "mode", None) == "auto": + impl = cast(Any, jit_impl) + mode = impl._infer_jit_mode(*args, **call_kwargs) + impl.mode = mode + if func is not None: + func.set_mode(mode) + key, _ = parse_args(*args, **call_kwargs) + except Exception: + return None + + cache = getattr(jit_impl, "_kernel_cache", {}) + if isinstance(cache, Mapping): + return key if key not in cache else None + + get = getattr(cache, "get", None) + if not callable(get): + return key + try: + return key if get(key) is None else None + except Exception: + return key + + +def _format_tilelang_runtime_shapes( + jit_impl: object, + args: tuple[object, ...], + kwargs: Mapping[str, object], +) -> str: + signature = getattr(jit_impl, "signature", None) + bind_partial = getattr(signature, "bind_partial", None) + if not callable(bind_partial): + return "{}" + + try: + bound = bind_partial(*args, **_tilelang_call_kwargs(kwargs)) + bound.apply_defaults() + except Exception: + return "{}" + + items = [] + for name, value in bound.arguments.items(): + if str(name).startswith("__"): + continue + if not (hasattr(value, "shape") and hasattr(value, "dtype")): + continue + shape = getattr(value, "shape", None) + if shape is not None: + with suppress(TypeError): + shape = tuple(shape) + items.append(f"{name}={_safe_repr(shape, max_len=80)}") + return "{" + ", ".join(items) + "}" + + +def _format_verbose_tilelang_compile_details( + jit_impl: object, + args: tuple[object, ...], + kwargs: Mapping[str, object], + cache_key: object, +) -> str: + return ( + f"cache_key={_safe_repr(cache_key, max_len=400)}; " + f"runtime_shapes={_format_tilelang_runtime_shapes(jit_impl, args, kwargs)}" + ) + + +def _log_tilelang_jit_compile( + fn_name: str, + detail: str | None = None, +) -> None: + _handle_jit_event( + backend="TileLang", + event="JIT compilation", + fn_name=fn_name, + detail=detail, + ) + + +def _setup_tilelang_jit_hook() -> None: + """Wrap TileLang JIT entry points to warn on compilation.""" + global _tilelang_hook_installed + if _tilelang_hook_installed: + return + + try: + tilelang_kernel = importlib.import_module("tilelang.jit.kernel") + except Exception: + logger.debug("TileLang is not available; skipping TileLang JIT monitor.") + return + + jit_kernel_cls = getattr(tilelang_kernel, "JITKernel", None) + if jit_kernel_cls is None: + logger.debug( + "TileLang JITKernel is unavailable; skipping TileLang JIT monitor." + ) + return + + try: + tilelang_jit = importlib.import_module("tilelang.jit") + except Exception: + tilelang_jit = None + jit_impl_cls = getattr(tilelang_jit, "JITImpl", None) + original_init = jit_kernel_cls.__init__ + + @functools.wraps(original_init) + def _init_with_monitor(self, *args, **kwargs): + from_database = bool(_tilelang_arg(args, kwargs, 7, "from_database", False)) + if not from_database and _tilelang_jitimpl_compile_depth == 0: + func = _tilelang_arg(args, kwargs, 0, "func") + _log_tilelang_jit_compile(_tilelang_kernel_name(func)) + return original_init(self, *args, **kwargs) + + jit_kernel_cls.__init__ = _init_with_monitor + + if jit_impl_cls is not None: + original_call = jit_impl_cls.__call__ + + @functools.wraps(original_call) + def _call_with_monitor(self, *args, **kwargs): + global _tilelang_jitimpl_compile_depth + cache_key = _tilelang_cache_miss_key(self, args, kwargs) + if cache_key is None: + return original_call(self, *args, **kwargs) + + _tilelang_jitimpl_compile_depth += 1 + try: + detail = None + if _verbose: + detail = _format_verbose_tilelang_compile_details( + self, args, kwargs, cache_key + ) + func = getattr(self, "func", None) + orig_func = getattr(func, "orig_func", None) + _log_tilelang_jit_compile( + _tilelang_kernel_name(orig_func or func), detail + ) + return original_call(self, *args, **kwargs) + finally: + _tilelang_jitimpl_compile_depth -= 1 + + jit_impl_cls.__call__ = _call_with_monitor + + _tilelang_hook_installed = True + + @contextlib.contextmanager def numba_workqueue_threading_layer() -> Iterator[None]: """Force numba's fork-safe `workqueue` threading layer for this block.