[Feat] Add runtime monitor for post-warmup CuTeDSL compilation (#46167)

This commit is contained in:
Roberto L. Castro
2026-06-23 23:33:17 +00:00
committed by GitHub
parent 80e511772f
commit d86c66c981
7 changed files with 366 additions and 172 deletions
+11
View File
@@ -214,6 +214,17 @@ def test_jit_monitor_verbose_arg():
assert EngineArgs(model="test", jit_monitor_verbose=True).jit_monitor_verbose
@pytest.mark.parametrize("mode", ["warn", "error"])
def test_jit_monitor_mode_arg(mode):
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
args = parser.parse_args(["--jit-monitor-mode", mode])
assert args.jit_monitor_mode == mode
engine_args = EngineArgs(model="test", jit_monitor_mode=mode)
assert engine_args.jit_monitor_mode == mode
assert engine_args.create_observability_config().jit_monitor_mode == mode
def test_hf_token_get_kwargs():
kwargs = get_kwargs(ModelConfig)["hf_token"]
+113 -18
View File
@@ -3,26 +3,31 @@
import os
import sys
from contextlib import contextmanager
from types import SimpleNamespace
from types import ModuleType, SimpleNamespace
from typing import Any, cast
from unittest import mock
import pytest
from vllm.triton_utils import jit_monitor
from vllm.utils import jit_monitor
@pytest.fixture(autouse=True)
def _reset_monitor():
"""Reset global monitor state between tests."""
jit_monitor._active = False
jit_monitor._mode = "warn"
jit_monitor._verbose = False
jit_monitor._cutedsl_hook_installed = False
yield
jit_monitor._active = False
jit_monitor._mode = "warn"
jit_monitor._verbose = False
jit_monitor._cutedsl_hook_installed = False
# ------------------------------------------------------------------
# Helpers — lightweight stand-ins for triton.knobs
# Helpers — lightweight stand-ins for the modules ``activate()`` patches
# ------------------------------------------------------------------
@@ -33,12 +38,36 @@ def _make_fake_knobs(*, autotuning_print=False, jit_hook=None):
return SimpleNamespace(autotuning=autotuning, runtime=runtime)
def _fake_cute_import_modules(compile_fn):
"""Fake Python's parent package + submodule for ``import cutlass.cute``."""
fake_cute = cast(Any, ModuleType("cutlass.cute"))
fake_cute.compile = compile_fn
fake_parent_package = cast(Any, ModuleType("cutlass"))
fake_parent_package.__path__ = []
fake_parent_package.cute = fake_cute
return {
"cutlass": fake_parent_package,
"cutlass.cute": fake_cute,
}
def _fake_cute_compile(*args, **kwargs):
return "compiled"
@contextmanager
def _patch_triton_knobs(fake_knobs):
"""Context manager that makes ``from triton import knobs`` return *fake_knobs*."""
fake_triton = SimpleNamespace(knobs=fake_knobs)
def _patch_jit_modules(fake_knobs, *, cute_compile=_fake_cute_compile):
"""Patch the Triton and CuTeDSL imports touched by ``jit_monitor.activate``."""
fake_triton = cast(Any, ModuleType("triton"))
fake_triton.knobs = fake_knobs
with (
mock.patch.dict(sys.modules, {"triton": fake_triton}),
mock.patch.dict(
sys.modules,
{
"triton": fake_triton,
**_fake_cute_import_modules(cute_compile),
},
),
mock.patch.object(jit_monitor, "HAS_TRITON", True),
):
yield
@@ -52,13 +81,13 @@ def _patch_triton_knobs(fake_knobs):
class TestActivateBasic:
def test_sets_active(self):
assert not jit_monitor.is_active()
with _patch_triton_knobs(_make_fake_knobs()):
with _patch_jit_modules(_make_fake_knobs()):
jit_monitor.activate()
assert jit_monitor.is_active()
def test_idempotent(self):
fake = _make_fake_knobs()
with _patch_triton_knobs(fake):
with _patch_jit_modules(fake):
jit_monitor.activate()
first_hook = fake.runtime.jit_post_compile_hook
jit_monitor.activate()
@@ -67,17 +96,21 @@ class TestActivateBasic:
def test_logs_info_on_activation(self):
with (
mock.patch.object(jit_monitor.logger, "info") as m,
_patch_triton_knobs(_make_fake_knobs()),
_patch_jit_modules(_make_fake_knobs()),
):
jit_monitor.activate()
m.assert_called_once()
assert "Kernel JIT monitor activated" in m.call_args[0][0]
def test_rejects_unknown_mode(self):
with pytest.raises(ValueError, match="Unsupported JIT monitor mode"):
jit_monitor.activate(mode="panic") # type: ignore[arg-type]
class TestAutotuningPrint:
def test_enables_autotuning_print(self):
fake = _make_fake_knobs(autotuning_print=False)
with _patch_triton_knobs(fake):
with _patch_jit_modules(fake):
jit_monitor.activate()
assert fake.autotuning.print is True
@@ -85,7 +118,7 @@ class TestAutotuningPrint:
fake = _make_fake_knobs(autotuning_print=False)
with (
mock.patch.dict(os.environ, {"TRITON_PRINT_AUTOTUNING": "0"}),
_patch_triton_knobs(fake),
_patch_jit_modules(fake),
):
jit_monitor.activate()
assert fake.autotuning.print is False
@@ -94,23 +127,23 @@ class TestAutotuningPrint:
fake = _make_fake_knobs(autotuning_print=True)
with (
mock.patch.dict(os.environ, {"TRITON_PRINT_AUTOTUNING": "1"}),
_patch_triton_knobs(fake),
_patch_jit_modules(fake),
):
jit_monitor.activate()
assert fake.autotuning.print is True
class TestJitHook:
class TestTritonJitHook:
def test_hook_registered(self):
fake = _make_fake_knobs()
assert fake.runtime.jit_post_compile_hook is None
with _patch_triton_knobs(fake):
with _patch_jit_modules(fake):
jit_monitor.activate()
assert fake.runtime.jit_post_compile_hook is not None
def test_hook_logs_warning(self):
fake = _make_fake_knobs()
with _patch_triton_knobs(fake):
with _patch_jit_modules(fake):
jit_monitor.activate()
hook = fake.runtime.jit_post_compile_hook
@@ -138,7 +171,7 @@ class TestJitHook:
def test_hook_chains_existing_hook(self):
existing = mock.MagicMock(return_value="existing_result")
fake = _make_fake_knobs(jit_hook=existing)
with _patch_triton_knobs(fake):
with _patch_jit_modules(fake):
jit_monitor.activate()
hook = fake.runtime.jit_post_compile_hook
@@ -158,7 +191,7 @@ class TestJitHook:
def test_hook_works_without_existing_hook(self):
fake = _make_fake_knobs(jit_hook=None)
with _patch_triton_knobs(fake):
with _patch_jit_modules(fake):
jit_monitor.activate()
hook = fake.runtime.jit_post_compile_hook
@@ -173,6 +206,23 @@ class TestJitHook:
)
assert result is None
def test_error_mode_raises(self):
fake = _make_fake_knobs()
with _patch_jit_modules(fake):
jit_monitor.activate(mode="error")
hook = fake.runtime.jit_post_compile_hook
mock_fn = SimpleNamespace(name="error_kernel")
with pytest.raises(RuntimeError, match="Triton kernel JIT compilation"):
hook(
key="k",
repr="r",
fn=mock_fn,
compile=lambda: None,
is_manual_warmup=False,
already_compiled=False,
)
class TestNoTritonFallback:
def test_activate_without_triton(self):
@@ -181,6 +231,51 @@ class TestNoTritonFallback:
assert jit_monitor.is_active()
class TestCuTeDSLHook:
def test_compile_logs_warning(self):
def compile_fn(*args, **kwargs):
return "compiled"
with _patch_jit_modules(_make_fake_knobs(), cute_compile=compile_fn):
import cutlass.cute as cute
jit_monitor.activate()
with mock.patch.object(jit_monitor.logger, "warning_once") as warning_once:
result = cute.compile(lambda: None, "arg", option=True)
assert result == "compiled"
warning_once.assert_called_once()
msg = warning_once.call_args[0][0] % warning_once.call_args[0][1:]
assert "CuTeDSL JIT compilation during inference" in msg
def test_compile_logs_verbose_warning(self):
def compile_fn(*args, **kwargs):
return "compiled"
with _patch_jit_modules(_make_fake_knobs(), cute_compile=compile_fn):
import cutlass.cute as cute
jit_monitor.activate(verbose=True)
with mock.patch.object(jit_monitor.logger, "warning") as warning:
result = cute.compile(lambda: None, "arg", option=True)
assert result == "compiled"
warning.assert_called_once()
msg = warning.call_args[0][0] % warning.call_args[0][1:]
assert "CuTeDSL JIT compilation during inference" in msg
def test_error_mode_raises(self):
def compile_fn(*args, **kwargs):
return "compiled"
with _patch_jit_modules(_make_fake_knobs(), cute_compile=compile_fn):
import cutlass.cute as cute
jit_monitor.activate(mode="error")
with pytest.raises(RuntimeError, match="CuTeDSL JIT compilation"):
cute.compile(lambda: None, "arg", option=True)
# ------------------------------------------------------------------
# Integration tests (real Triton + GPU)
# ------------------------------------------------------------------
+4 -1
View File
@@ -76,8 +76,11 @@ class ObservabilityConfig:
This includes number of context/generation requests and tokens
and the elapsed cpu time for the iteration."""
jit_monitor_mode: Literal["warn", "error"] = "warn"
"""How to handle post-warmup JIT compilation events."""
jit_monitor_verbose: bool = False
"""Log every Triton JIT compile with its dispatch key. This can emit many
"""Log every monitored JIT compile with runtime details. This can emit many
logs and add overhead, so it is intended for debugging."""
@cached_property
+22 -13
View File
@@ -643,6 +643,7 @@ class EngineArgs:
enable_logging_iteration_details: bool = (
ObservabilityConfig.enable_logging_iteration_details
)
jit_monitor_mode: Literal["warn", "error"] = ObservabilityConfig.jit_monitor_mode
jit_monitor_verbose: bool = ObservabilityConfig.jit_monitor_verbose
enable_mm_processor_stats: bool = ObservabilityConfig.enable_mm_processor_stats
scheduling_policy: SchedulerPolicy = SchedulerConfig.policy
@@ -1378,6 +1379,10 @@ class EngineArgs:
"--enable-logging-iteration-details",
**observability_kwargs["enable_logging_iteration_details"],
)
observability_group.add_argument(
"--jit-monitor-mode",
**observability_kwargs["jit_monitor_mode"],
)
observability_group.add_argument(
"--jit-monitor-verbose",
**observability_kwargs["jit_monitor_verbose"],
@@ -1781,6 +1786,22 @@ class EngineArgs:
cfg = json.loads(cfg)
return DiffusionConfig(**cfg)
def create_observability_config(self) -> ObservabilityConfig:
return ObservabilityConfig(
show_hidden_metrics_for_version=self.show_hidden_metrics_for_version,
otlp_traces_endpoint=self.otlp_traces_endpoint,
collect_detailed_traces=self.collect_detailed_traces,
kv_cache_metrics=self.kv_cache_metrics,
kv_cache_metrics_sample=self.kv_cache_metrics_sample,
cudagraph_metrics=self.cudagraph_metrics,
enable_layerwise_nvtx_tracing=self.enable_layerwise_nvtx_tracing,
enable_mfu_metrics=self.enable_mfu_metrics,
enable_mm_processor_stats=self.enable_mm_processor_stats,
enable_logging_iteration_details=self.enable_logging_iteration_details,
jit_monitor_mode=self.jit_monitor_mode,
jit_monitor_verbose=self.jit_monitor_verbose,
)
def create_engine_config(
self,
usage_context: UsageContext | None = None,
@@ -2263,19 +2284,7 @@ class EngineArgs:
self.reasoning_parser_plugin
)
observability_config = ObservabilityConfig(
show_hidden_metrics_for_version=self.show_hidden_metrics_for_version,
otlp_traces_endpoint=self.otlp_traces_endpoint,
collect_detailed_traces=self.collect_detailed_traces,
kv_cache_metrics=self.kv_cache_metrics,
kv_cache_metrics_sample=self.kv_cache_metrics_sample,
cudagraph_metrics=self.cudagraph_metrics,
enable_layerwise_nvtx_tracing=self.enable_layerwise_nvtx_tracing,
enable_mfu_metrics=self.enable_mfu_metrics,
enable_mm_processor_stats=self.enable_mm_processor_stats,
enable_logging_iteration_details=self.enable_logging_iteration_details,
jit_monitor_verbose=self.jit_monitor_verbose,
)
observability_config = self.create_observability_config()
# Compilation config overrides
compilation_config = copy.deepcopy(self.compilation_config)
-135
View File
@@ -1,135 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Monitor unexpected Triton kernel JIT compilation during inference.
After server warmup completes, any Triton JIT compilation or autotuning
event indicates a cache miss or unexpected input shape that causes a
latency spike. This module registers hooks in the Triton runtime to
detect and log such events so they can be investigated.
Set ``--jit-monitor-verbose`` to log every Triton JIT compile with its
dispatch key. This is intentionally opt-in because it can emit many logs and
add overhead.
Currently monitors:
- Triton ``@triton.autotune`` cache misses (via ``knobs.autotuning.print``)
- Triton ``@triton.jit`` first-time compilations
(via ``knobs.runtime.jit_post_compile_hook``)
"""
import os
from vllm.logger import init_logger
from vllm.triton_utils.importing import HAS_TRITON
logger = init_logger(__name__)
_active: bool = False
_verbose: bool = False
def is_active() -> bool:
"""Return whether the JIT compilation monitor is currently active."""
return _active
def activate(*, verbose: bool = False) -> None:
"""Enable JIT compilation monitoring after warmup.
Call once per worker process at the end of
:func:`compile_or_warm_up_model`. After activation every Triton
kernel compilation or autotuning benchmark that happens during
inference will be logged as a warning.
Safe to call multiple times — subsequent calls are no-ops.
If the user has explicitly set ``TRITON_PRINT_AUTOTUNING=0`` in
their environment, autotuning printing is left disabled; the JIT
compilation hook is still registered regardless.
"""
global _active, _verbose
if _active:
return
_active = True
_verbose = verbose
_setup_triton_autotuning_print()
_setup_triton_jit_hook()
logger.info(
"Kernel JIT monitor activated — Triton JIT compilations "
"during inference will be logged as warnings."
)
# ------------------------------------------------------------------
# Triton autotuning print
# ------------------------------------------------------------------
def _setup_triton_autotuning_print() -> None:
"""Enable ``TRITON_PRINT_AUTOTUNING`` unless the user opted out."""
if not HAS_TRITON:
return
from triton import knobs # type: ignore[import-untyped]
user_val = os.environ.get("TRITON_PRINT_AUTOTUNING")
if user_val == "0":
logger.debug(
"TRITON_PRINT_AUTOTUNING=0 set by user — "
"autotuning messages will stay suppressed."
)
return
knobs.autotuning.print = True
# ------------------------------------------------------------------
# Triton JIT compilation hook
# ------------------------------------------------------------------
def _log_jit_compile(fn_name: str, kwargs) -> None:
if _verbose:
compile_info = kwargs.get("compile")
if not isinstance(compile_info, dict):
compile_info = {}
logger.warning(
"Triton %sJIT compilation during inference: %s (key=%s).",
"autotune/warmup candidate " if kwargs.get("warmup") else "kernel ",
fn_name,
compile_info.get("key") or kwargs.get("key"),
)
return
logger.warning_once(
"Triton kernel JIT compilation during inference: %s. "
"This causes a latency spike; consider extending warmup "
"to cover this shape/config.",
fn_name,
)
def _setup_triton_jit_hook() -> None:
"""Register a ``jit_post_compile_hook`` that warns on compilation."""
if not HAS_TRITON:
return
from triton import knobs # type: ignore[import-untyped]
existing_hook = knobs.runtime.jit_post_compile_hook
def _on_jit_compile(**kwargs):
# `jit_post_compile_hook` is Triton internal API and its
# signature has changed across releases (kwargs added/renamed).
# Accept **kwargs so an upstream change cannot crash this hook
# with TypeError, and forward the full kwarg set to any
# pre-existing hook unchanged.
fn = kwargs.get("fn")
fn_name = getattr(fn, "name", "<unknown>")
_log_jit_compile(fn_name, kwargs)
if existing_hook is not None:
return existing_hook(**kwargs)
return None
knobs.runtime.jit_post_compile_hook = _on_jit_compile
+212
View File
@@ -0,0 +1,212 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Monitor unexpected kernel JIT compilation during inference.
After server warmup completes, any kernel JIT compilation or autotuning event
indicates a cache miss or unexpected input shape that causes a latency spike.
This module registers hooks in supported runtimes to detect such events so
they can be investigated.
Set ``--jit-monitor-mode=error`` to fail fast on unexpected runtime
compilation. Set ``--jit-monitor-verbose`` to log every JIT compile with
additional runtime details. Verbose logging is intentionally opt-in because it
can emit many logs and add overhead.
Currently monitors:
- CuTeDSL cute.compile calls
- Triton ``@triton.autotune`` cache misses (via ``knobs.autotuning.print``)
- Triton ``@triton.jit`` first-time compilations
(via ``knobs.runtime.jit_post_compile_hook``)
"""
import functools
import os
from typing import Literal
from vllm.logger import init_logger
from vllm.triton_utils.importing import HAS_TRITON
logger = init_logger(__name__)
JitMonitorMode = Literal["warn", "error"]
_active: bool = False
_mode: JitMonitorMode = "warn"
_verbose: bool = False
_cutedsl_hook_installed: bool = False
def is_active() -> bool:
"""Return whether the JIT compilation monitor is currently active."""
return _active
def activate(*, mode: JitMonitorMode = "warn", verbose: bool = False) -> None:
"""Enable JIT compilation monitoring after warmup.
Call once per worker process at the end of
:func:`compile_or_warm_up_model`. After activation every monitored kernel
compilation or autotuning benchmark that happens during inference will be
logged as a warning or raised as an error, depending on ``mode``.
Safe to call multiple times; subsequent calls are no-ops.
If the user has explicitly set ``TRITON_PRINT_AUTOTUNING=0`` in
their environment, autotuning printing is left disabled; the JIT
compilation hook is still registered regardless.
"""
global _active, _mode, _verbose
if _active:
return
if mode not in ("warn", "error"):
raise ValueError(f"Unsupported JIT monitor mode: {mode!r}")
_active = True
_mode = mode
_verbose = verbose
_setup_triton_autotuning_print()
_setup_triton_jit_hook()
_setup_cutedsl_jit_hook()
logger.info(
"Kernel JIT monitor activated; monitored JIT compilations during "
"inference will use mode=%s.",
mode,
)
# ------------------------------------------------------------------
# Triton autotuning print
# ------------------------------------------------------------------
def _setup_triton_autotuning_print() -> None:
"""Enable ``TRITON_PRINT_AUTOTUNING`` unless the user opted out."""
if not HAS_TRITON:
return
from triton import knobs # type: ignore[import-untyped]
user_val = os.environ.get("TRITON_PRINT_AUTOTUNING")
if user_val == "0":
logger.debug(
"TRITON_PRINT_AUTOTUNING=0 set by user; "
"autotuning messages will stay suppressed."
)
return
knobs.autotuning.print = True
# ------------------------------------------------------------------
# Triton JIT compilation hook
# ------------------------------------------------------------------
def _handle_jit_event(
*,
backend: str,
event: str,
fn_name: str,
detail: str | None = None,
) -> None:
message = (
"%s %s during inference: %s%s. "
"This causes a latency spike; consider extending warmup "
"to cover this shape/config."
)
detail_suffix = f" ({detail})" if detail else ""
args = (backend, event, fn_name, detail_suffix)
if _mode == "error":
raise RuntimeError(message % args)
if _verbose:
logger.warning(message, *args)
return
logger.warning_once(message, *args)
def _log_triton_jit_compile(fn_name: str, kwargs) -> None:
compile_info = kwargs.get("compile")
if not isinstance(compile_info, dict):
compile_info = {}
key = compile_info.get("key") or kwargs.get("key")
detail = f"key={key}" if _verbose and key is not None else None
event = (
"autotune/warmup candidate JIT compilation"
if kwargs.get("warmup")
else "kernel JIT compilation"
)
_handle_jit_event(
backend="Triton",
event=event,
fn_name=fn_name,
detail=detail,
)
def _setup_triton_jit_hook() -> None:
"""Register a ``jit_post_compile_hook`` that warns on compilation."""
if not HAS_TRITON:
return
from triton import knobs # type: ignore[import-untyped]
existing_hook = knobs.runtime.jit_post_compile_hook
def _on_jit_compile(**kwargs):
# `jit_post_compile_hook` is Triton internal API and its
# signature has changed across releases (kwargs added/renamed).
# Accept **kwargs so an upstream change cannot crash this hook
# with TypeError, and forward the full kwarg set to any
# pre-existing hook unchanged.
fn = kwargs.get("fn")
fn_name = getattr(fn, "name", "<unknown>")
_log_triton_jit_compile(fn_name, kwargs)
if existing_hook is not None:
return existing_hook(**kwargs)
return None
knobs.runtime.jit_post_compile_hook = _on_jit_compile
# ------------------------------------------------------------------
# CuTeDSL JIT compilation hook
# ------------------------------------------------------------------
def _log_cutedsl_jit_compile(fn_name: str) -> None:
_handle_jit_event(
backend="CuTeDSL",
event="JIT compilation",
fn_name=fn_name,
)
def _setup_cutedsl_jit_hook() -> None:
"""Wrap ``cutlass.cute.compile`` to warn on compilation."""
global _cutedsl_hook_installed
if _cutedsl_hook_installed:
return
try:
import cutlass.cute as cute
except Exception:
logger.debug("CuTeDSL is not available; skipping CuTeDSL JIT monitor.")
return
original_compile = cute.compile
@functools.wraps(original_compile)
def _compile_with_monitor(*args, **kwargs):
kernel = args[0] if args else kwargs.get("function")
kernel_name = getattr(kernel, "__name__", None)
if kernel_name is None:
kernel_name = (
kernel.__class__.__name__ if kernel is not None else "<unknown>"
)
_log_cutedsl_jit_compile(kernel_name)
return original_compile(*args, **kwargs)
cute.compile = _compile_with_monitor
_cutedsl_hook_installed = True
+4 -5
View File
@@ -761,12 +761,11 @@ class Worker(WorkerBase):
# All warmup is done — start monitoring for unexpected JIT
# compilations that would cause latency spikes during inference.
from vllm.triton_utils.jit_monitor import (
activate as activate_triton_jit_monitor,
)
from vllm.utils.jit_monitor import activate as activate_jit_monitor
activate_triton_jit_monitor(
verbose=self.observability_config.jit_monitor_verbose
activate_jit_monitor(
mode=self.observability_config.jit_monitor_mode,
verbose=self.observability_config.jit_monitor_verbose,
)
# Freeze the worker heap so the GC won't scan static objects