[Feat] Add runtime monitor for post-warmup TileLang compilation (#46718)

Signed-off-by: LopezCastroRoberto <rocastro@redhat.com>
Signed-off-by: Roberto L. Castro <38211239+LopezCastroRoberto@users.noreply.github.com>
This commit is contained in:
Roberto L. Castro
2026-07-08 22:11:28 +00:00
committed by GitHub
parent dcdd756d75
commit 5f85975624
2 changed files with 367 additions and 1 deletions
+155
View File
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import inspect
import os
import sys
from contextlib import contextmanager
@@ -19,11 +20,15 @@ def _reset_monitor():
jit_monitor._mode = "warn"
jit_monitor._verbose = False
jit_monitor._cutedsl_hook_installed = False
jit_monitor._tilelang_hook_installed = False
jit_monitor._tilelang_jitimpl_compile_depth = 0
yield
jit_monitor._active = False
jit_monitor._mode = "warn"
jit_monitor._verbose = False
jit_monitor._cutedsl_hook_installed = False
jit_monitor._tilelang_hook_installed = False
jit_monitor._tilelang_jitimpl_compile_depth = 0
# ------------------------------------------------------------------
@@ -55,6 +60,45 @@ def _fake_cute_compile(*args, **kwargs):
return "compiled"
def _fake_tilelang_import_modules():
"""Fake Python's TileLang modules touched by ``jit_monitor.activate``."""
class FakeJITKernel:
def __init__(self, *args, **kwargs):
pass
class FakeJITImpl:
def __init__(self, func, signature):
self.func = func
self.signature = signature
self.mode = "lazy"
self._kernel_cache = {}
def __call__(self, *args, **kwargs):
key, _ = self.func.parse_args(*args, **kwargs)
kernel = self._kernel_cache.get(key)
if kernel is None:
kernel = "compiled"
self._kernel_cache[key] = kernel
return kernel
fake_kernel = cast(Any, ModuleType("tilelang.jit.kernel"))
fake_kernel.JITKernel = FakeJITKernel
fake_jit = cast(Any, ModuleType("tilelang.jit"))
fake_jit.JITImpl = FakeJITImpl
fake_jit.kernel = fake_kernel
fake_tilelang = cast(Any, ModuleType("tilelang"))
fake_tilelang.jit = fake_jit
return {
"tilelang": fake_tilelang,
"tilelang.jit": fake_jit,
"tilelang.jit.kernel": fake_kernel,
}
@contextmanager
def _patch_jit_modules(fake_knobs, *, cute_compile=_fake_cute_compile):
"""Patch the Triton and CuTeDSL imports touched by ``jit_monitor.activate``."""
@@ -66,6 +110,7 @@ def _patch_jit_modules(fake_knobs, *, cute_compile=_fake_cute_compile):
{
"triton": fake_triton,
**_fake_cute_import_modules(cute_compile),
**_fake_tilelang_import_modules(),
},
),
mock.patch.object(jit_monitor, "HAS_TRITON", True),
@@ -276,6 +321,116 @@ class TestCuTeDSLHook:
cute.compile(lambda: None, "arg", option=True)
class TestTileLangHook:
def test_jit_kernel_logs_warning(self):
with _patch_jit_modules(_make_fake_knobs()):
from tilelang.jit.kernel import JITKernel
func = SimpleNamespace(attrs={"global_symbol": "tl_kernel"})
jit_monitor.activate()
with mock.patch.object(jit_monitor.logger, "warning_once") as warning_once:
JITKernel(func=func, out_idx=None, execution_backend="tvm_ffi")
warning_once.assert_called_once()
msg = warning_once.call_args[0][0] % warning_once.call_args[0][1:]
assert "TileLang JIT compilation during inference" in msg
assert "tl_kernel" in msg
def test_jit_impl_logs_warning(self):
with _patch_jit_modules(_make_fake_knobs()):
from tilelang.jit import JITImpl
def tilelang_fn(
gemm_out_mul,
hidden_size: int,
n_splits: int = 1,
hc_mult: int = 4,
):
return None
class FakeFunc:
orig_func = tilelang_fn
def parse_args(self, *args, **kwargs):
return (
(
"tilelang_key",
kwargs["hidden_size"],
kwargs.get("n_splits", 1),
),
{},
)
def set_mode(self, mode):
self.mode = mode
tensor = SimpleNamespace(
shape=(2, 16, 24),
dtype="float32",
device="cuda:0",
)
impl = JITImpl(FakeFunc(), inspect.signature(tilelang_fn))
jit_monitor.activate()
with (
mock.patch.object(jit_monitor.logger, "warning_once") as warning_once,
mock.patch.object(jit_monitor.logger, "warning") as warning,
):
impl(tensor, hidden_size=7168, n_splits=2)
warning_once.assert_called_once()
warning.assert_not_called()
msg = warning_once.call_args[0][0] % warning_once.call_args[0][1:]
assert "TileLang JIT compilation during inference" in msg
assert "tilelang_fn" in msg
def test_jit_impl_does_not_log_on_cache_hit(self):
with _patch_jit_modules(_make_fake_knobs()):
from tilelang.jit import JITImpl
def tilelang_fn(gemm_out_mul, n_splits: int = 1):
return None
class FakeFunc:
orig_func = tilelang_fn
def parse_args(self, *args, **kwargs):
return (("tilelang_key", kwargs.get("n_splits", 1)), {})
def set_mode(self, mode):
self.mode = mode
tensor = SimpleNamespace(shape=(2, 16, 24), dtype="float32")
impl = JITImpl(FakeFunc(), inspect.signature(tilelang_fn))
jit_monitor.activate()
with mock.patch.object(jit_monitor.logger, "warning_once") as warning_once:
impl(tensor, n_splits=2)
impl(tensor, n_splits=2)
warning_once.assert_called_once()
def test_from_database_does_not_log(self):
with _patch_jit_modules(_make_fake_knobs()):
from tilelang.jit.kernel import JITKernel
func = SimpleNamespace(attrs={"global_symbol": "cached_tl_kernel"})
jit_monitor.activate()
with mock.patch.object(jit_monitor.logger, "warning_once") as warning_once:
JITKernel(func=func, from_database=True)
warning_once.assert_not_called()
def test_error_mode_raises(self):
with _patch_jit_modules(_make_fake_knobs()):
from tilelang.jit.kernel import JITKernel
func = SimpleNamespace(attrs={"global_symbol": "error_tl_kernel"})
jit_monitor.activate(mode="error")
with pytest.raises(RuntimeError, match="TileLang JIT compilation"):
JITKernel(func=func)
# ------------------------------------------------------------------
# Integration tests (real Triton + GPU)
# ------------------------------------------------------------------
+212 -1
View File
@@ -18,13 +18,16 @@ Currently monitors:
- Triton ``@triton.autotune`` cache misses (via ``knobs.autotuning.print``)
- Triton ``@triton.jit`` first-time compilations
(via ``knobs.runtime.jit_post_compile_hook``)
- TileLang ``@tilelang.jit`` first-time compilations
"""
import contextlib
import functools
import importlib
import os
from collections.abc import Iterator, Mapping
from typing import Literal
from contextlib import suppress
from typing import Any, Literal, cast
from vllm.logger import init_logger
from vllm.triton_utils.importing import HAS_TRITON
@@ -36,6 +39,8 @@ _active: bool = False
_mode: JitMonitorMode = "warn"
_verbose: bool = False
_cutedsl_hook_installed: bool = False
_tilelang_hook_installed: bool = False
_tilelang_jitimpl_compile_depth: int = 0
def is_active() -> bool:
@@ -69,6 +74,7 @@ def activate(*, mode: JitMonitorMode = "warn", verbose: bool = False) -> None:
_setup_triton_autotuning_print()
_setup_triton_jit_hook()
_setup_cutedsl_jit_hook()
_setup_tilelang_jit_hook()
logger.info(
"Kernel JIT monitor activated; monitored JIT compilations during "
@@ -290,6 +296,211 @@ def _setup_cutedsl_jit_hook() -> None:
_cutedsl_hook_installed = True
# ------------------------------------------------------------------
# TileLang JIT compilation hook
# ------------------------------------------------------------------
def _tilelang_arg(
args: tuple[object, ...],
kwargs: Mapping[str, object],
index: int,
name: str,
default: object = None,
) -> object:
if len(args) > index:
return args[index]
return kwargs.get(name, default)
def _tilelang_kernel_name(func: object) -> str:
attrs = getattr(func, "attrs", None)
global_symbol = None
get_attr = getattr(attrs, "get", None)
if callable(get_attr):
try:
global_symbol = get_attr("global_symbol")
except Exception:
global_symbol = None
elif isinstance(attrs, Mapping):
global_symbol = attrs.get("global_symbol")
if global_symbol is not None:
return str(global_symbol)
name = getattr(func, "__name__", None)
if name is not None:
return str(name)
return func.__class__.__name__ if func is not None else "<unknown>"
def _tilelang_call_kwargs(kwargs: Mapping[str, object]) -> dict[str, object]:
call_kwargs = dict(kwargs)
tune_params = call_kwargs.pop("__tune_params", {})
if isinstance(tune_params, Mapping):
call_kwargs.update(tune_params)
return call_kwargs
def _tilelang_cache_miss_key(
jit_impl: object,
args: tuple[object, ...],
kwargs: Mapping[str, object],
) -> object | None:
if kwargs.get("__return_compile_arguments", False):
return None
call_kwargs = _tilelang_call_kwargs(kwargs)
func = getattr(jit_impl, "func", None)
parse_args = getattr(func, "parse_args", None)
if not callable(parse_args):
return None
try:
if getattr(jit_impl, "mode", None) == "auto":
impl = cast(Any, jit_impl)
mode = impl._infer_jit_mode(*args, **call_kwargs)
impl.mode = mode
if func is not None:
func.set_mode(mode)
key, _ = parse_args(*args, **call_kwargs)
except Exception:
return None
cache = getattr(jit_impl, "_kernel_cache", {})
if isinstance(cache, Mapping):
return key if key not in cache else None
get = getattr(cache, "get", None)
if not callable(get):
return key
try:
return key if get(key) is None else None
except Exception:
return key
def _format_tilelang_runtime_shapes(
jit_impl: object,
args: tuple[object, ...],
kwargs: Mapping[str, object],
) -> str:
signature = getattr(jit_impl, "signature", None)
bind_partial = getattr(signature, "bind_partial", None)
if not callable(bind_partial):
return "{}"
try:
bound = bind_partial(*args, **_tilelang_call_kwargs(kwargs))
bound.apply_defaults()
except Exception:
return "{}"
items = []
for name, value in bound.arguments.items():
if str(name).startswith("__"):
continue
if not (hasattr(value, "shape") and hasattr(value, "dtype")):
continue
shape = getattr(value, "shape", None)
if shape is not None:
with suppress(TypeError):
shape = tuple(shape)
items.append(f"{name}={_safe_repr(shape, max_len=80)}")
return "{" + ", ".join(items) + "}"
def _format_verbose_tilelang_compile_details(
jit_impl: object,
args: tuple[object, ...],
kwargs: Mapping[str, object],
cache_key: object,
) -> str:
return (
f"cache_key={_safe_repr(cache_key, max_len=400)}; "
f"runtime_shapes={_format_tilelang_runtime_shapes(jit_impl, args, kwargs)}"
)
def _log_tilelang_jit_compile(
fn_name: str,
detail: str | None = None,
) -> None:
_handle_jit_event(
backend="TileLang",
event="JIT compilation",
fn_name=fn_name,
detail=detail,
)
def _setup_tilelang_jit_hook() -> None:
"""Wrap TileLang JIT entry points to warn on compilation."""
global _tilelang_hook_installed
if _tilelang_hook_installed:
return
try:
tilelang_kernel = importlib.import_module("tilelang.jit.kernel")
except Exception:
logger.debug("TileLang is not available; skipping TileLang JIT monitor.")
return
jit_kernel_cls = getattr(tilelang_kernel, "JITKernel", None)
if jit_kernel_cls is None:
logger.debug(
"TileLang JITKernel is unavailable; skipping TileLang JIT monitor."
)
return
try:
tilelang_jit = importlib.import_module("tilelang.jit")
except Exception:
tilelang_jit = None
jit_impl_cls = getattr(tilelang_jit, "JITImpl", None)
original_init = jit_kernel_cls.__init__
@functools.wraps(original_init)
def _init_with_monitor(self, *args, **kwargs):
from_database = bool(_tilelang_arg(args, kwargs, 7, "from_database", False))
if not from_database and _tilelang_jitimpl_compile_depth == 0:
func = _tilelang_arg(args, kwargs, 0, "func")
_log_tilelang_jit_compile(_tilelang_kernel_name(func))
return original_init(self, *args, **kwargs)
jit_kernel_cls.__init__ = _init_with_monitor
if jit_impl_cls is not None:
original_call = jit_impl_cls.__call__
@functools.wraps(original_call)
def _call_with_monitor(self, *args, **kwargs):
global _tilelang_jitimpl_compile_depth
cache_key = _tilelang_cache_miss_key(self, args, kwargs)
if cache_key is None:
return original_call(self, *args, **kwargs)
_tilelang_jitimpl_compile_depth += 1
try:
detail = None
if _verbose:
detail = _format_verbose_tilelang_compile_details(
self, args, kwargs, cache_key
)
func = getattr(self, "func", None)
orig_func = getattr(func, "orig_func", None)
_log_tilelang_jit_compile(
_tilelang_kernel_name(orig_func or func), detail
)
return original_call(self, *args, **kwargs)
finally:
_tilelang_jitimpl_compile_depth -= 1
jit_impl_cls.__call__ = _call_with_monitor
_tilelang_hook_installed = True
@contextlib.contextmanager
def numba_workqueue_threading_layer() -> Iterator[None]:
"""Force numba's fork-safe `workqueue` threading layer for this block.