diff --git a/tests/engine/test_arg_utils.py b/tests/engine/test_arg_utils.py index a35f445302..7da3aa66c9 100644 --- a/tests/engine/test_arg_utils.py +++ b/tests/engine/test_arg_utils.py @@ -214,6 +214,17 @@ def test_jit_monitor_verbose_arg(): assert EngineArgs(model="test", jit_monitor_verbose=True).jit_monitor_verbose +@pytest.mark.parametrize("mode", ["warn", "error"]) +def test_jit_monitor_mode_arg(mode): + parser = EngineArgs.add_cli_args(FlexibleArgumentParser()) + args = parser.parse_args(["--jit-monitor-mode", mode]) + + assert args.jit_monitor_mode == mode + engine_args = EngineArgs(model="test", jit_monitor_mode=mode) + assert engine_args.jit_monitor_mode == mode + assert engine_args.create_observability_config().jit_monitor_mode == mode + + def test_hf_token_get_kwargs(): kwargs = get_kwargs(ModelConfig)["hf_token"] diff --git a/tests/test_jit_monitor.py b/tests/test_jit_monitor.py index 8dd778d52f..8d3a241612 100644 --- a/tests/test_jit_monitor.py +++ b/tests/test_jit_monitor.py @@ -3,26 +3,31 @@ import os import sys from contextlib import contextmanager -from types import SimpleNamespace +from types import ModuleType, SimpleNamespace +from typing import Any, cast from unittest import mock import pytest -from vllm.triton_utils import jit_monitor +from vllm.utils import jit_monitor @pytest.fixture(autouse=True) def _reset_monitor(): """Reset global monitor state between tests.""" jit_monitor._active = False + jit_monitor._mode = "warn" jit_monitor._verbose = False + jit_monitor._cutedsl_hook_installed = False yield jit_monitor._active = False + jit_monitor._mode = "warn" jit_monitor._verbose = False + jit_monitor._cutedsl_hook_installed = False # ------------------------------------------------------------------ -# Helpers — lightweight stand-ins for triton.knobs +# Helpers — lightweight stand-ins for the modules ``activate()`` patches # ------------------------------------------------------------------ @@ -33,12 +38,36 @@ def _make_fake_knobs(*, autotuning_print=False, jit_hook=None): return SimpleNamespace(autotuning=autotuning, runtime=runtime) +def _fake_cute_import_modules(compile_fn): + """Fake Python's parent package + submodule for ``import cutlass.cute``.""" + fake_cute = cast(Any, ModuleType("cutlass.cute")) + fake_cute.compile = compile_fn + fake_parent_package = cast(Any, ModuleType("cutlass")) + fake_parent_package.__path__ = [] + fake_parent_package.cute = fake_cute + return { + "cutlass": fake_parent_package, + "cutlass.cute": fake_cute, + } + + +def _fake_cute_compile(*args, **kwargs): + return "compiled" + + @contextmanager -def _patch_triton_knobs(fake_knobs): - """Context manager that makes ``from triton import knobs`` return *fake_knobs*.""" - fake_triton = SimpleNamespace(knobs=fake_knobs) +def _patch_jit_modules(fake_knobs, *, cute_compile=_fake_cute_compile): + """Patch the Triton and CuTeDSL imports touched by ``jit_monitor.activate``.""" + fake_triton = cast(Any, ModuleType("triton")) + fake_triton.knobs = fake_knobs with ( - mock.patch.dict(sys.modules, {"triton": fake_triton}), + mock.patch.dict( + sys.modules, + { + "triton": fake_triton, + **_fake_cute_import_modules(cute_compile), + }, + ), mock.patch.object(jit_monitor, "HAS_TRITON", True), ): yield @@ -52,13 +81,13 @@ def _patch_triton_knobs(fake_knobs): class TestActivateBasic: def test_sets_active(self): assert not jit_monitor.is_active() - with _patch_triton_knobs(_make_fake_knobs()): + with _patch_jit_modules(_make_fake_knobs()): jit_monitor.activate() assert jit_monitor.is_active() def test_idempotent(self): fake = _make_fake_knobs() - with _patch_triton_knobs(fake): + with _patch_jit_modules(fake): jit_monitor.activate() first_hook = fake.runtime.jit_post_compile_hook jit_monitor.activate() @@ -67,17 +96,21 @@ class TestActivateBasic: def test_logs_info_on_activation(self): with ( mock.patch.object(jit_monitor.logger, "info") as m, - _patch_triton_knobs(_make_fake_knobs()), + _patch_jit_modules(_make_fake_knobs()), ): jit_monitor.activate() m.assert_called_once() assert "Kernel JIT monitor activated" in m.call_args[0][0] + def test_rejects_unknown_mode(self): + with pytest.raises(ValueError, match="Unsupported JIT monitor mode"): + jit_monitor.activate(mode="panic") # type: ignore[arg-type] + class TestAutotuningPrint: def test_enables_autotuning_print(self): fake = _make_fake_knobs(autotuning_print=False) - with _patch_triton_knobs(fake): + with _patch_jit_modules(fake): jit_monitor.activate() assert fake.autotuning.print is True @@ -85,7 +118,7 @@ class TestAutotuningPrint: fake = _make_fake_knobs(autotuning_print=False) with ( mock.patch.dict(os.environ, {"TRITON_PRINT_AUTOTUNING": "0"}), - _patch_triton_knobs(fake), + _patch_jit_modules(fake), ): jit_monitor.activate() assert fake.autotuning.print is False @@ -94,23 +127,23 @@ class TestAutotuningPrint: fake = _make_fake_knobs(autotuning_print=True) with ( mock.patch.dict(os.environ, {"TRITON_PRINT_AUTOTUNING": "1"}), - _patch_triton_knobs(fake), + _patch_jit_modules(fake), ): jit_monitor.activate() assert fake.autotuning.print is True -class TestJitHook: +class TestTritonJitHook: def test_hook_registered(self): fake = _make_fake_knobs() assert fake.runtime.jit_post_compile_hook is None - with _patch_triton_knobs(fake): + with _patch_jit_modules(fake): jit_monitor.activate() assert fake.runtime.jit_post_compile_hook is not None def test_hook_logs_warning(self): fake = _make_fake_knobs() - with _patch_triton_knobs(fake): + with _patch_jit_modules(fake): jit_monitor.activate() hook = fake.runtime.jit_post_compile_hook @@ -138,7 +171,7 @@ class TestJitHook: def test_hook_chains_existing_hook(self): existing = mock.MagicMock(return_value="existing_result") fake = _make_fake_knobs(jit_hook=existing) - with _patch_triton_knobs(fake): + with _patch_jit_modules(fake): jit_monitor.activate() hook = fake.runtime.jit_post_compile_hook @@ -158,7 +191,7 @@ class TestJitHook: def test_hook_works_without_existing_hook(self): fake = _make_fake_knobs(jit_hook=None) - with _patch_triton_knobs(fake): + with _patch_jit_modules(fake): jit_monitor.activate() hook = fake.runtime.jit_post_compile_hook @@ -173,6 +206,23 @@ class TestJitHook: ) assert result is None + def test_error_mode_raises(self): + fake = _make_fake_knobs() + with _patch_jit_modules(fake): + jit_monitor.activate(mode="error") + + hook = fake.runtime.jit_post_compile_hook + mock_fn = SimpleNamespace(name="error_kernel") + with pytest.raises(RuntimeError, match="Triton kernel JIT compilation"): + hook( + key="k", + repr="r", + fn=mock_fn, + compile=lambda: None, + is_manual_warmup=False, + already_compiled=False, + ) + class TestNoTritonFallback: def test_activate_without_triton(self): @@ -181,6 +231,51 @@ class TestNoTritonFallback: assert jit_monitor.is_active() +class TestCuTeDSLHook: + def test_compile_logs_warning(self): + def compile_fn(*args, **kwargs): + return "compiled" + + with _patch_jit_modules(_make_fake_knobs(), cute_compile=compile_fn): + import cutlass.cute as cute + + jit_monitor.activate() + with mock.patch.object(jit_monitor.logger, "warning_once") as warning_once: + result = cute.compile(lambda: None, "arg", option=True) + + assert result == "compiled" + warning_once.assert_called_once() + msg = warning_once.call_args[0][0] % warning_once.call_args[0][1:] + assert "CuTeDSL JIT compilation during inference" in msg + + def test_compile_logs_verbose_warning(self): + def compile_fn(*args, **kwargs): + return "compiled" + + with _patch_jit_modules(_make_fake_knobs(), cute_compile=compile_fn): + import cutlass.cute as cute + + jit_monitor.activate(verbose=True) + with mock.patch.object(jit_monitor.logger, "warning") as warning: + result = cute.compile(lambda: None, "arg", option=True) + + assert result == "compiled" + warning.assert_called_once() + msg = warning.call_args[0][0] % warning.call_args[0][1:] + assert "CuTeDSL JIT compilation during inference" in msg + + def test_error_mode_raises(self): + def compile_fn(*args, **kwargs): + return "compiled" + + with _patch_jit_modules(_make_fake_knobs(), cute_compile=compile_fn): + import cutlass.cute as cute + + jit_monitor.activate(mode="error") + with pytest.raises(RuntimeError, match="CuTeDSL JIT compilation"): + cute.compile(lambda: None, "arg", option=True) + + # ------------------------------------------------------------------ # Integration tests (real Triton + GPU) # ------------------------------------------------------------------ diff --git a/vllm/config/observability.py b/vllm/config/observability.py index b35ec6ce74..093ed2f684 100644 --- a/vllm/config/observability.py +++ b/vllm/config/observability.py @@ -76,8 +76,11 @@ class ObservabilityConfig: This includes number of context/generation requests and tokens and the elapsed cpu time for the iteration.""" + jit_monitor_mode: Literal["warn", "error"] = "warn" + """How to handle post-warmup JIT compilation events.""" + jit_monitor_verbose: bool = False - """Log every Triton JIT compile with its dispatch key. This can emit many + """Log every monitored JIT compile with runtime details. This can emit many logs and add overhead, so it is intended for debugging.""" @cached_property diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 921f31466b..5f96a62a87 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -643,6 +643,7 @@ class EngineArgs: enable_logging_iteration_details: bool = ( ObservabilityConfig.enable_logging_iteration_details ) + jit_monitor_mode: Literal["warn", "error"] = ObservabilityConfig.jit_monitor_mode jit_monitor_verbose: bool = ObservabilityConfig.jit_monitor_verbose enable_mm_processor_stats: bool = ObservabilityConfig.enable_mm_processor_stats scheduling_policy: SchedulerPolicy = SchedulerConfig.policy @@ -1378,6 +1379,10 @@ class EngineArgs: "--enable-logging-iteration-details", **observability_kwargs["enable_logging_iteration_details"], ) + observability_group.add_argument( + "--jit-monitor-mode", + **observability_kwargs["jit_monitor_mode"], + ) observability_group.add_argument( "--jit-monitor-verbose", **observability_kwargs["jit_monitor_verbose"], @@ -1781,6 +1786,22 @@ class EngineArgs: cfg = json.loads(cfg) return DiffusionConfig(**cfg) + def create_observability_config(self) -> ObservabilityConfig: + return ObservabilityConfig( + show_hidden_metrics_for_version=self.show_hidden_metrics_for_version, + otlp_traces_endpoint=self.otlp_traces_endpoint, + collect_detailed_traces=self.collect_detailed_traces, + kv_cache_metrics=self.kv_cache_metrics, + kv_cache_metrics_sample=self.kv_cache_metrics_sample, + cudagraph_metrics=self.cudagraph_metrics, + enable_layerwise_nvtx_tracing=self.enable_layerwise_nvtx_tracing, + enable_mfu_metrics=self.enable_mfu_metrics, + enable_mm_processor_stats=self.enable_mm_processor_stats, + enable_logging_iteration_details=self.enable_logging_iteration_details, + jit_monitor_mode=self.jit_monitor_mode, + jit_monitor_verbose=self.jit_monitor_verbose, + ) + def create_engine_config( self, usage_context: UsageContext | None = None, @@ -2263,19 +2284,7 @@ class EngineArgs: self.reasoning_parser_plugin ) - observability_config = ObservabilityConfig( - show_hidden_metrics_for_version=self.show_hidden_metrics_for_version, - otlp_traces_endpoint=self.otlp_traces_endpoint, - collect_detailed_traces=self.collect_detailed_traces, - kv_cache_metrics=self.kv_cache_metrics, - kv_cache_metrics_sample=self.kv_cache_metrics_sample, - cudagraph_metrics=self.cudagraph_metrics, - enable_layerwise_nvtx_tracing=self.enable_layerwise_nvtx_tracing, - enable_mfu_metrics=self.enable_mfu_metrics, - enable_mm_processor_stats=self.enable_mm_processor_stats, - enable_logging_iteration_details=self.enable_logging_iteration_details, - jit_monitor_verbose=self.jit_monitor_verbose, - ) + observability_config = self.create_observability_config() # Compilation config overrides compilation_config = copy.deepcopy(self.compilation_config) diff --git a/vllm/triton_utils/jit_monitor.py b/vllm/triton_utils/jit_monitor.py deleted file mode 100644 index 9a7b1695af..0000000000 --- a/vllm/triton_utils/jit_monitor.py +++ /dev/null @@ -1,135 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Monitor unexpected Triton kernel JIT compilation during inference. - -After server warmup completes, any Triton JIT compilation or autotuning -event indicates a cache miss or unexpected input shape that causes a -latency spike. This module registers hooks in the Triton runtime to -detect and log such events so they can be investigated. - -Set ``--jit-monitor-verbose`` to log every Triton JIT compile with its -dispatch key. This is intentionally opt-in because it can emit many logs and -add overhead. - -Currently monitors: -- Triton ``@triton.autotune`` cache misses (via ``knobs.autotuning.print``) -- Triton ``@triton.jit`` first-time compilations - (via ``knobs.runtime.jit_post_compile_hook``) -""" - -import os - -from vllm.logger import init_logger -from vllm.triton_utils.importing import HAS_TRITON - -logger = init_logger(__name__) - -_active: bool = False -_verbose: bool = False - - -def is_active() -> bool: - """Return whether the JIT compilation monitor is currently active.""" - return _active - - -def activate(*, verbose: bool = False) -> None: - """Enable JIT compilation monitoring after warmup. - - Call once per worker process at the end of - :func:`compile_or_warm_up_model`. After activation every Triton - kernel compilation or autotuning benchmark that happens during - inference will be logged as a warning. - - Safe to call multiple times — subsequent calls are no-ops. - - If the user has explicitly set ``TRITON_PRINT_AUTOTUNING=0`` in - their environment, autotuning printing is left disabled; the JIT - compilation hook is still registered regardless. - """ - global _active, _verbose - if _active: - return - _active = True - _verbose = verbose - - _setup_triton_autotuning_print() - _setup_triton_jit_hook() - - logger.info( - "Kernel JIT monitor activated — Triton JIT compilations " - "during inference will be logged as warnings." - ) - - -# ------------------------------------------------------------------ -# Triton autotuning print -# ------------------------------------------------------------------ - - -def _setup_triton_autotuning_print() -> None: - """Enable ``TRITON_PRINT_AUTOTUNING`` unless the user opted out.""" - if not HAS_TRITON: - return - from triton import knobs # type: ignore[import-untyped] - - user_val = os.environ.get("TRITON_PRINT_AUTOTUNING") - if user_val == "0": - logger.debug( - "TRITON_PRINT_AUTOTUNING=0 set by user — " - "autotuning messages will stay suppressed." - ) - return - - knobs.autotuning.print = True - - -# ------------------------------------------------------------------ -# Triton JIT compilation hook -# ------------------------------------------------------------------ - - -def _log_jit_compile(fn_name: str, kwargs) -> None: - if _verbose: - compile_info = kwargs.get("compile") - if not isinstance(compile_info, dict): - compile_info = {} - logger.warning( - "Triton %sJIT compilation during inference: %s (key=%s).", - "autotune/warmup candidate " if kwargs.get("warmup") else "kernel ", - fn_name, - compile_info.get("key") or kwargs.get("key"), - ) - return - - logger.warning_once( - "Triton kernel JIT compilation during inference: %s. " - "This causes a latency spike; consider extending warmup " - "to cover this shape/config.", - fn_name, - ) - - -def _setup_triton_jit_hook() -> None: - """Register a ``jit_post_compile_hook`` that warns on compilation.""" - if not HAS_TRITON: - return - from triton import knobs # type: ignore[import-untyped] - - existing_hook = knobs.runtime.jit_post_compile_hook - - def _on_jit_compile(**kwargs): - # `jit_post_compile_hook` is Triton internal API and its - # signature has changed across releases (kwargs added/renamed). - # Accept **kwargs so an upstream change cannot crash this hook - # with TypeError, and forward the full kwarg set to any - # pre-existing hook unchanged. - fn = kwargs.get("fn") - fn_name = getattr(fn, "name", "") - _log_jit_compile(fn_name, kwargs) - if existing_hook is not None: - return existing_hook(**kwargs) - return None - - knobs.runtime.jit_post_compile_hook = _on_jit_compile diff --git a/vllm/utils/jit_monitor.py b/vllm/utils/jit_monitor.py new file mode 100644 index 0000000000..23a8037c57 --- /dev/null +++ b/vllm/utils/jit_monitor.py @@ -0,0 +1,212 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +Monitor unexpected kernel JIT compilation during inference. + +After server warmup completes, any kernel JIT compilation or autotuning event +indicates a cache miss or unexpected input shape that causes a latency spike. +This module registers hooks in supported runtimes to detect such events so +they can be investigated. + +Set ``--jit-monitor-mode=error`` to fail fast on unexpected runtime +compilation. Set ``--jit-monitor-verbose`` to log every JIT compile with +additional runtime details. Verbose logging is intentionally opt-in because it +can emit many logs and add overhead. + +Currently monitors: +- CuTeDSL cute.compile calls +- Triton ``@triton.autotune`` cache misses (via ``knobs.autotuning.print``) +- Triton ``@triton.jit`` first-time compilations + (via ``knobs.runtime.jit_post_compile_hook``) +""" + +import functools +import os +from typing import Literal + +from vllm.logger import init_logger +from vllm.triton_utils.importing import HAS_TRITON + +logger = init_logger(__name__) +JitMonitorMode = Literal["warn", "error"] + +_active: bool = False +_mode: JitMonitorMode = "warn" +_verbose: bool = False +_cutedsl_hook_installed: bool = False + + +def is_active() -> bool: + """Return whether the JIT compilation monitor is currently active.""" + return _active + + +def activate(*, mode: JitMonitorMode = "warn", verbose: bool = False) -> None: + """Enable JIT compilation monitoring after warmup. + + Call once per worker process at the end of + :func:`compile_or_warm_up_model`. After activation every monitored kernel + compilation or autotuning benchmark that happens during inference will be + logged as a warning or raised as an error, depending on ``mode``. + + Safe to call multiple times; subsequent calls are no-ops. + + If the user has explicitly set ``TRITON_PRINT_AUTOTUNING=0`` in + their environment, autotuning printing is left disabled; the JIT + compilation hook is still registered regardless. + """ + global _active, _mode, _verbose + if _active: + return + if mode not in ("warn", "error"): + raise ValueError(f"Unsupported JIT monitor mode: {mode!r}") + _active = True + _mode = mode + _verbose = verbose + + _setup_triton_autotuning_print() + _setup_triton_jit_hook() + _setup_cutedsl_jit_hook() + + logger.info( + "Kernel JIT monitor activated; monitored JIT compilations during " + "inference will use mode=%s.", + mode, + ) + + +# ------------------------------------------------------------------ +# Triton autotuning print +# ------------------------------------------------------------------ + + +def _setup_triton_autotuning_print() -> None: + """Enable ``TRITON_PRINT_AUTOTUNING`` unless the user opted out.""" + if not HAS_TRITON: + return + from triton import knobs # type: ignore[import-untyped] + + user_val = os.environ.get("TRITON_PRINT_AUTOTUNING") + if user_val == "0": + logger.debug( + "TRITON_PRINT_AUTOTUNING=0 set by user; " + "autotuning messages will stay suppressed." + ) + return + + knobs.autotuning.print = True + + +# ------------------------------------------------------------------ +# Triton JIT compilation hook +# ------------------------------------------------------------------ + + +def _handle_jit_event( + *, + backend: str, + event: str, + fn_name: str, + detail: str | None = None, +) -> None: + message = ( + "%s %s during inference: %s%s. " + "This causes a latency spike; consider extending warmup " + "to cover this shape/config." + ) + detail_suffix = f" ({detail})" if detail else "" + args = (backend, event, fn_name, detail_suffix) + + if _mode == "error": + raise RuntimeError(message % args) + + if _verbose: + logger.warning(message, *args) + return + + logger.warning_once(message, *args) + + +def _log_triton_jit_compile(fn_name: str, kwargs) -> None: + compile_info = kwargs.get("compile") + if not isinstance(compile_info, dict): + compile_info = {} + key = compile_info.get("key") or kwargs.get("key") + detail = f"key={key}" if _verbose and key is not None else None + event = ( + "autotune/warmup candidate JIT compilation" + if kwargs.get("warmup") + else "kernel JIT compilation" + ) + _handle_jit_event( + backend="Triton", + event=event, + fn_name=fn_name, + detail=detail, + ) + + +def _setup_triton_jit_hook() -> None: + """Register a ``jit_post_compile_hook`` that warns on compilation.""" + if not HAS_TRITON: + return + from triton import knobs # type: ignore[import-untyped] + + existing_hook = knobs.runtime.jit_post_compile_hook + + def _on_jit_compile(**kwargs): + # `jit_post_compile_hook` is Triton internal API and its + # signature has changed across releases (kwargs added/renamed). + # Accept **kwargs so an upstream change cannot crash this hook + # with TypeError, and forward the full kwarg set to any + # pre-existing hook unchanged. + fn = kwargs.get("fn") + fn_name = getattr(fn, "name", "") + _log_triton_jit_compile(fn_name, kwargs) + if existing_hook is not None: + return existing_hook(**kwargs) + return None + + knobs.runtime.jit_post_compile_hook = _on_jit_compile + + +# ------------------------------------------------------------------ +# CuTeDSL JIT compilation hook +# ------------------------------------------------------------------ + + +def _log_cutedsl_jit_compile(fn_name: str) -> None: + _handle_jit_event( + backend="CuTeDSL", + event="JIT compilation", + fn_name=fn_name, + ) + + +def _setup_cutedsl_jit_hook() -> None: + """Wrap ``cutlass.cute.compile`` to warn on compilation.""" + global _cutedsl_hook_installed + if _cutedsl_hook_installed: + return + + try: + import cutlass.cute as cute + except Exception: + logger.debug("CuTeDSL is not available; skipping CuTeDSL JIT monitor.") + return + + original_compile = cute.compile + + @functools.wraps(original_compile) + def _compile_with_monitor(*args, **kwargs): + kernel = args[0] if args else kwargs.get("function") + kernel_name = getattr(kernel, "__name__", None) + if kernel_name is None: + kernel_name = ( + kernel.__class__.__name__ if kernel is not None else "" + ) + _log_cutedsl_jit_compile(kernel_name) + return original_compile(*args, **kwargs) + + cute.compile = _compile_with_monitor + _cutedsl_hook_installed = True diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 87bfba6db2..4aa8ca5ca3 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -761,12 +761,11 @@ class Worker(WorkerBase): # All warmup is done — start monitoring for unexpected JIT # compilations that would cause latency spikes during inference. - from vllm.triton_utils.jit_monitor import ( - activate as activate_triton_jit_monitor, - ) + from vllm.utils.jit_monitor import activate as activate_jit_monitor - activate_triton_jit_monitor( - verbose=self.observability_config.jit_monitor_verbose + activate_jit_monitor( + mode=self.observability_config.jit_monitor_mode, + verbose=self.observability_config.jit_monitor_verbose, ) # Freeze the worker heap so the GC won't scan static objects