From ae4f59f0ece88ca25ba1064ae6d3b512c3bfc606 Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Thu, 14 May 2026 13:02:33 -0400 Subject: [PATCH] [Model Runner v2] Oracle for model runner v2 - qwen3 dense model by default [1/N] (#39337) Signed-off-by: yewentao256 Signed-off-by: Nick Hill Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: Nick Hill --- tests/test_config.py | 99 ++++++++++++++++++++ vllm/config/vllm.py | 114 ++++++++++++++++++++--- vllm/envs.py | 8 +- vllm/v1/attention/backends/flashinfer.py | 2 +- vllm/v1/core/sched/scheduler.py | 3 +- vllm/v1/worker/gpu_worker.py | 2 +- 6 files changed, 208 insertions(+), 20 deletions(-) diff --git a/tests/test_config.py b/tests/test_config.py index 8dd9216ec0..5c01d652a1 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -12,6 +12,7 @@ import pytest from pydantic import ValidationError import vllm.config.vllm as vllm_config_module +import vllm.envs as envs from vllm.compilation.backends import VllmBackend from vllm.config import ( CompilationConfig, @@ -49,6 +50,104 @@ def test_compile_config_repr_succeeds(): assert "inductor_passes" in val +@pytest.mark.parametrize( + ("env_value", "expected"), + [ + (None, None), + ("0", False), + ("1", True), + ], +) +def test_v2_model_runner_env_tri_state(monkeypatch, env_value, expected): + if env_value is None: + monkeypatch.delenv("VLLM_USE_V2_MODEL_RUNNER", raising=False) + else: + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", env_value) + + assert envs.VLLM_USE_V2_MODEL_RUNNER is expected + + +@pytest.mark.parametrize( + ("model_config", "expected"), + [ + ( + SimpleNamespace( + model="Qwen/Qwen3-1.7B-Base", + architectures=["Qwen3ForCausalLM"], + runner_type="generate", + is_moe=False, + is_quantized=False, + ), + True, + ), + ( + SimpleNamespace( + model="Qwen/Qwen3-32B", + architectures=["Qwen3ForCausalLM"], + runner_type="generate", + is_moe=False, + is_quantized=False, + ), + True, + ), + ( + SimpleNamespace( + model="facebook/opt-125m", + architectures=["OPTForCausalLM"], + runner_type="generate", + is_moe=False, + is_quantized=False, + ), + False, + ), + ( + SimpleNamespace( + model="Qwen/Qwen3-30B-A3B", + architectures=["Qwen3MoeForCausalLM"], + runner_type="generate", + is_moe=True, + is_quantized=False, + ), + False, + ), + ( + SimpleNamespace( + model="Qwen/Qwen3-1.7B-FP8", + architectures=["Qwen3ForCausalLM"], + runner_type="generate", + is_moe=False, + is_quantized=True, + ), + False, + ), + ( + SimpleNamespace( + model="Qwen/Qwen3.5-4B", + architectures=["Qwen3_5ForConditionalGeneration"], + runner_type="generate", + is_moe=False, + is_quantized=False, + ), + False, + ), + ( + SimpleNamespace( + model="Qwen/Qwen3-Embedding-0.6B", + architectures=["Qwen3ForCausalLM"], + runner_type="pooling", + is_moe=False, + is_quantized=False, + ), + False, + ), + ], +) +def test_is_default_v2_model_runner_model(model_config, expected): + config = SimpleNamespace(model_config=model_config) + + assert VllmConfig._is_default_v2_model_runner_model(config) is expected + + @pytest.mark.skip_global_cleanup def test_with_hf_config_populates_missing_architectures_from_causal_lm_mapping( monkeypatch, diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 9f93ab7944..f9090ff588 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -24,6 +24,7 @@ from pydantic import ConfigDict, Field, model_validator import vllm.envs as envs from vllm.logger import enable_trace_function_call, init_logger from vllm.transformers_utils.runai_utils import is_runai_obj_uri +from vllm.triton_utils import HAS_TRITON from vllm.utils import random_uuid from vllm.utils.hashing import safe_hash @@ -64,6 +65,8 @@ else: logger = init_logger(__name__) +DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES = frozenset({"Qwen3ForCausalLM"}) + class OptimizationLevel(IntEnum): """Optimization level enum.""" @@ -489,6 +492,48 @@ class VllmConfig: return self.speculative_config.num_speculative_tokens return 0 + @property + def use_v2_model_runner(self) -> bool: + use_v2_model_runner = envs.VLLM_USE_V2_MODEL_RUNNER + if use_v2_model_runner is not None: + return use_v2_model_runner + + if not self._is_default_v2_model_runner_model(): + return False + + if not HAS_TRITON: + logger.warning_once( + "Model runner v2 requires Triton; using the v1 model runner instead." + ) + return False + + unsupported = self._get_v2_model_runner_unsupported_features() + if unsupported: + logger.warning_once( + "Model runner v2 does not yet support %s; using the v1 model " + "runner instead.", + ", ".join(unsupported), + ) + return False + + return True + + def _is_default_v2_model_runner_model(self) -> bool: + model_config = self.model_config + if model_config is None: + return False + + if model_config.runner_type != "generate": + return False + + architectures = getattr(model_config, "architectures", []) + if not any( + arch in DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES for arch in architectures + ): + return False + + return not model_config.is_moe and not model_config.is_quantized + @property def needs_dp_coordinator(self) -> bool: """ @@ -1233,7 +1278,7 @@ class VllmConfig: ) current_platform.check_and_update_config(self) - if envs.VLLM_USE_V2_MODEL_RUNNER: + if self.use_v2_model_runner: self._validate_v2_model_runner() # Re-compute compile ranges after platform-specific config updates @@ -1908,35 +1953,72 @@ class VllmConfig: f"kernel_config={self.kernel_config!r}" ) - def _validate_v2_model_runner(self) -> None: - """Check for features not yet supported by the V2 model runner.""" + def _get_v2_model_runner_unsupported_features(self) -> list[str]: + """Collect features not yet supported by the V2 model runner.""" unsupported: list[str] = [] + model_config = self.model_config + speculative_config = self.speculative_config - if self.model_config is not None and self.model_config.has_inner_state: + if model_config is not None and model_config.has_inner_state: unsupported.append("hybrid/mamba models") if self.parallel_config.prefill_context_parallel_size > 1: unsupported.append("prefill context parallelism") + if self.compilation_config.mode == CompilationMode.STOCK_TORCH_COMPILE: + unsupported.append("stock torch.compile") + if ( - self.speculative_config is not None - and self.speculative_config.method not in ("eagle", "eagle3", "mtp") + self.compilation_config.pass_config.enable_sp + and self.parallel_config.tensor_parallel_size > 1 ): - unsupported.append(f"speculative method '{self.speculative_config.method}'") + unsupported.append("sequence parallelism") + + if speculative_config is not None: + # TODO: ngram / ngram_gpu are not supported by the v2 model runner yet + if speculative_config.method in ("ngram", "ngram_gpu"): + unsupported.append("ngram/ngram_gpu speculative decoding") + elif speculative_config.method not in ("eagle", "eagle3", "mtp"): + unsupported.append(f"speculative method '{speculative_config.method}'") + + if ( + speculative_config.method == "eagle3" + and self.parallel_config.pipeline_parallel_size > 1 + ): + unsupported.append("EAGLE3 with pipeline parallelism") + + if self.reasoning_config is not None: + # TODO: add reasoning budget enforcement to ModelRunnerV2. + unsupported.append("reasoning budget enforcement") if self.parallel_config.enable_dbo: unsupported.append("dual batch overlap") - if ( - self.model_config is not None - and self.model_config.enable_return_routed_experts - ): + if model_config is not None and model_config.enable_return_routed_experts: # Will be added by https://github.com/vllm-project/vllm/pull/38163 unsupported.append("routed experts capture") - if self.model_config is not None and self.model_config.logits_processors: + has_logitsproc_plugins = False + if model_config is not None: + from importlib.metadata import entry_points + + has_logitsproc_plugins = bool(entry_points(group="vllm.logits_processors")) + + if model_config is not None and ( + model_config.logits_processors or has_logitsproc_plugins + ): unsupported.append("custom logits processors") + if model_config is not None and model_config.enable_prompt_embeds: + unsupported.append("prompt embeds") + + if ( + model_config is not None + and model_config.runner_type == "generate" + and model_config.logprobs_mode in ("raw_logits", "processed_logits") + ): + unsupported.append(f"logprobs mode '{model_config.logprobs_mode}'") + if self.cache_config.kv_sharing_fast_prefill: # Will be added by https://github.com/vllm-project/vllm/pull/35045 unsupported.append("KV sharing fast prefill") @@ -1945,6 +2027,14 @@ class VllmConfig: # Will be added by https://github.com/vllm-project/vllm/pull/38390 unsupported.append("EC transfer") + return unsupported + + def _validate_v2_model_runner(self) -> None: + """Check for features not yet supported by the V2 model runner.""" + if not HAS_TRITON: + raise ValueError("VLLM_USE_V2_MODEL_RUNNER requires Triton.") + + unsupported = self._get_v2_model_runner_unsupported_features() if unsupported: raise ValueError( "VLLM_USE_V2_MODEL_RUNNER does not yet support: " diff --git a/vllm/envs.py b/vllm/envs.py index 851f32a095..7c9ee0ea9d 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -248,7 +248,7 @@ if TYPE_CHECKING: VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD: int = 256 VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD: int = 1024 VLLM_COMPILE_CACHE_SAVE_FORMAT: Literal["binary", "unpacked"] = "binary" - VLLM_USE_V2_MODEL_RUNNER: bool = False + VLLM_USE_V2_MODEL_RUNNER: bool | None = None VLLM_LOG_MODEL_INSPECTION: bool = False VLLM_DEBUG_MFU_METRICS: bool = False VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY: bool = False @@ -1710,9 +1710,9 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_COMPILE_CACHE_SAVE_FORMAT": env_with_choices( "VLLM_COMPILE_CACHE_SAVE_FORMAT", "binary", ["binary", "unpacked"] ), - # Flag to enable v2 model runner. - "VLLM_USE_V2_MODEL_RUNNER": lambda: bool( - int(os.getenv("VLLM_USE_V2_MODEL_RUNNER", "0")) + # Flag to control the v2 model runner. If unset, use config defaults. + "VLLM_USE_V2_MODEL_RUNNER": lambda: maybe_convert_bool( + os.getenv("VLLM_USE_V2_MODEL_RUNNER", None) ), # Log model inspection after loading. # If enabled, logs a transformers-style hierarchical view of the model diff --git a/vllm/v1/attention/backends/flashinfer.py b/vllm/v1/attention/backends/flashinfer.py index dc4b1cccac..2b206cbcf2 100755 --- a/vllm/v1/attention/backends/flashinfer.py +++ b/vllm/v1/attention/backends/flashinfer.py @@ -681,7 +681,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): # reused CPU buffers to avoid a race condition between step N async copies to # GPU and step N+1 buffer updates. self.pin_memory = ( - not envs.VLLM_USE_V2_MODEL_RUNNER and is_pin_memory_available() + not vllm_config.use_v2_model_runner and is_pin_memory_available() ) self.paged_kv_indptr = self._make_buffer(max_num_reqs + 1) self.paged_kv_indptr_cpu_buffer = torch.zeros_like( diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index e6edbb232b..1b46657889 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -7,7 +7,6 @@ from collections.abc import Iterable from dataclasses import replace from typing import Any -from vllm import envs from vllm.compilation.cuda_graph import CUDAGraphStat from vllm.config import VllmConfig from vllm.distributed.ec_transfer.ec_connector.base import ( @@ -242,7 +241,7 @@ class Scheduler(SchedulerInterface): self.connector.bind_gpu_block_pool(self.kv_cache_manager.block_pool) self.use_pp = self.parallel_config.pipeline_parallel_size > 1 - self.use_v2_model_runner = envs.VLLM_USE_V2_MODEL_RUNNER + self.use_v2_model_runner = vllm_config.use_v2_model_runner self.scheduler_reserve_full_isl = ( self.scheduler_config.scheduler_reserve_full_isl ) diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 18c7941a0e..53e62c9256 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -153,7 +153,7 @@ class Worker(WorkerBase): if self.profiler_config.profiler not in ("torch", "cuda", None): raise ValueError(f"Unknown profiler type: {self.profiler_config.profiler}") - self.use_v2_model_runner = envs.VLLM_USE_V2_MODEL_RUNNER + self.use_v2_model_runner = vllm_config.use_v2_model_runner # pending non-blocking PP send work from the previous iteration self._pp_send_work: list[Handle] = []