From 946452961265514622b95ea170839b214f39c2a0 Mon Sep 17 00:00:00 2001 From: Taneem Ibrahim Date: Fri, 7 Aug 2026 00:21:52 +0200 Subject: [PATCH] [ModelRunner v2] Enable decoder token-wise pooling (#50931) Signed-off-by: Taneem Ibrahim --- .../layers/test_pooler_methods.py | 36 +++++++++++++++---- .../test_all_pooling_plus_chunked_prefill.py | 8 +++-- .../language/pooling/test_jina_reranker_v3.py | 4 ++- tests/models/language/pooling/test_reward.py | 10 ++++++ .../pooling/test_splade_sparse_pooler.py | 23 +++++------- .../layers/pooler/tokwise/methods.py | 25 ++++++++++--- vllm/v1/worker/gpu/model_runner.py | 4 +-- vllm/v1/worker/gpu/pool/pooling_runner.py | 34 +++++------------- 8 files changed, 86 insertions(+), 58 deletions(-) diff --git a/tests/model_executor/layers/test_pooler_methods.py b/tests/model_executor/layers/test_pooler_methods.py index 28b2fc7a78..8460d7dfe8 100644 --- a/tests/model_executor/layers/test_pooler_methods.py +++ b/tests/model_executor/layers/test_pooler_methods.py @@ -256,6 +256,7 @@ class TestGetSeqPoolingMethod: @dataclass class _FakeSchedulerConfig: enable_chunked_prefill: bool = False + async_scheduling: bool = False @dataclass @@ -265,10 +266,13 @@ class _FakeVllmConfig: class TestAllPool: @staticmethod - def _make_all_pool(*, chunked: bool = False) -> AllPool: + def _make_all_pool( + *, chunked: bool = False, async_scheduling: bool = False + ) -> AllPool: fake_config = _FakeVllmConfig( scheduler_config=_FakeSchedulerConfig( enable_chunked_prefill=chunked, + async_scheduling=async_scheduling, ), ) with patch( @@ -319,6 +323,8 @@ class TestAllPool: ) out1 = pooler(chunk1, metadata1) assert out1[0] is None + expected_chunk1 = chunk1.clone() + chunk1.zero_() chunk2 = torch.tensor([[5.0, 6.0], [7.0, 8.0]]) metadata2 = _make_metadata( @@ -329,19 +335,37 @@ class TestAllPool: metadata2.pooling_states = metadata1.pooling_states out2 = pooler(chunk2, metadata2) assert out2[0] is not None - expected = torch.cat([chunk1, chunk2], dim=0) + expected = torch.cat([expected_chunk1, chunk2], dim=0) assert torch.equal(out2[0], expected) - def test_chunked_prefill_single_shot_matches_non_chunked(self): - pooler = self._make_all_pool(chunked=True) + @pytest.mark.parametrize("async_scheduling", [False, True]) + def test_chunked_prefill_single_shot_matches_non_chunked( + self, async_scheduling: bool + ): + pooler = self._make_all_pool(chunked=True, async_scheduling=async_scheduling) hidden = torch.tensor( [[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0], [9.0, 10.0]] ) + expected = [hidden[:2].clone(), hidden[2:].clone()] metadata = _make_metadata([2, 3]) out = pooler(hidden, metadata) + if async_scheduling: + # Finished outputs must own their storage under async scheduling. + hidden.zero_() assert len(out) == 2 - assert torch.equal(out[0], hidden[:2]) - assert torch.equal(out[1], hidden[2:]) + assert torch.equal(out[0], expected[0]) + assert torch.equal(out[1], expected[1]) + + def test_non_chunked_owns_output_under_async_scheduling(self): + pooler = self._make_all_pool(async_scheduling=True) + hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]]) + expected = [hidden[:2].clone(), hidden[2:].clone()] + metadata = _make_metadata([2, 1]) + out = pooler(hidden, metadata) + hidden.zero_() + assert len(out) == 2 + assert torch.equal(out[0], expected[0]) + assert torch.equal(out[1], expected[1]) def test_chunked_prefill_mixed_finished_unfinished(self): pooler = self._make_all_pool(chunked=True) diff --git a/tests/models/language/pooling/test_all_pooling_plus_chunked_prefill.py b/tests/models/language/pooling/test_all_pooling_plus_chunked_prefill.py index c987e92cac..ece33d6c0d 100644 --- a/tests/models/language/pooling/test_all_pooling_plus_chunked_prefill.py +++ b/tests/models/language/pooling/test_all_pooling_plus_chunked_prefill.py @@ -15,9 +15,10 @@ from vllm.config import PoolerConfig ["Qwen/Qwen3-Embedding-0.6B"], ) @torch.inference_mode -def test_embed_models(hf_runner, vllm_runner, monkeypatch, model: str): - # Keep token_embed on MRV1 when sequence pooling becomes MRV2 by default. - monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0") +def test_decoder_token_embed_model_runner_v2( + hf_runner, vllm_runner, monkeypatch, model: str +): + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1") chunk_size = 10 n_prompt_tokens = [55, 56, 57] token_prompts = [[1024 + i for i in range(n)] for n in n_prompt_tokens] @@ -33,6 +34,7 @@ def test_embed_models(hf_runner, vllm_runner, monkeypatch, model: str): enable_chunked_prefill=True, enable_prefix_caching=True, ) as vllm_model: + assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner vllm_outputs = vllm_model.token_embed( [TokensPrompt(prompt_token_ids=t) for t in token_prompts], ) diff --git a/tests/models/language/pooling/test_jina_reranker_v3.py b/tests/models/language/pooling/test_jina_reranker_v3.py index e76a3745c5..3fc5c48c5a 100644 --- a/tests/models/language/pooling/test_jina_reranker_v3.py +++ b/tests/models/language/pooling/test_jina_reranker_v3.py @@ -45,8 +45,10 @@ INSTRUCTION = ( ) -def test_offline(vllm_runner): +def test_offline(vllm_runner, monkeypatch): + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1") with vllm_runner(model_name, runner="pooling") as llm_runner: + assert llm_runner.llm.llm_engine.vllm_config.use_v2_model_runner llm = llm_runner.get_llm() _test_offline_1_v_1(llm) _test_offline_1_v_n(llm) diff --git a/tests/models/language/pooling/test_reward.py b/tests/models/language/pooling/test_reward.py index f981266d0c..111770863c 100644 --- a/tests/models/language/pooling/test_reward.py +++ b/tests/models/language/pooling/test_reward.py @@ -98,6 +98,7 @@ def test_prm_models( hf_runner, vllm_runner, math_step_prompts, + monkeypatch, model: str, dtype: str, ) -> None: @@ -107,12 +108,14 @@ def test_prm_models( if current_platform.is_cpu(): pytest.skip("CPU only supports V1") + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1") with vllm_runner( model, max_model_len=1024, dtype=dtype, kv_cache_memory_bytes=64 * MiB_bytes, ) as vllm_model: + assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner vllm_outputs = vllm_model.token_classify(math_step_prompts) with hf_runner(model, dtype=dtype, auto_cls=AutoModel) as hf_model: @@ -145,18 +148,25 @@ def test_prm_models( def test_prm_models_with_golden_outputs( vllm_runner, math_step_prompts, + monkeypatch, model: str, dtype: str, ) -> None: if not FIXTURE_REWARD_RESULT.get(model): pytest.skip(f"No available golden outputs for {model}.") + use_v2_model_runner = not current_platform.is_cpu() + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if use_v2_model_runner else "0") with vllm_runner( model, max_model_len=1024, dtype=dtype, kv_cache_memory_bytes=64 * MiB_bytes, ) as vllm_model: + assert ( + vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner + is use_v2_model_runner + ) vllm_outputs = vllm_model.token_classify(math_step_prompts) golden_outputs = load_reward_outputs(FIXTURE_REWARD_RESULT[model]) diff --git a/tests/models/language/pooling/test_splade_sparse_pooler.py b/tests/models/language/pooling/test_splade_sparse_pooler.py index 2fbeadacb9..0218d3694f 100644 --- a/tests/models/language/pooling/test_splade_sparse_pooler.py +++ b/tests/models/language/pooling/test_splade_sparse_pooler.py @@ -231,7 +231,7 @@ def test_pooling_runner_supports_encoder_token_embedding() -> None: assert runner.supported_tasks == {"token_embed"} -def test_pooling_runner_rejects_decoder_token_classification() -> None: +def test_pooling_runner_supports_decoder_token_classification() -> None: model = MagicMock() model.pooler.get_supported_tasks.return_value = {"token_classify"} vllm_config = MagicMock() @@ -239,13 +239,12 @@ def test_pooling_runner_rejects_decoder_token_classification() -> None: vllm_config.model_config.attn_type = "decoder" vllm_config.model_config.get_pooling_task.return_value = "token_classify" - with pytest.raises(ValueError, match="selects 'token_classify'") as exc_info: - PoolingRunner(model, vllm_config) + runner = PoolingRunner(model, vllm_config) - assert "Set an explicitly supported task" not in str(exc_info.value) + assert runner.supported_tasks == {"token_classify"} -def test_pooling_runner_filters_decoder_token_classification() -> None: +def test_pooling_runner_keeps_decoder_token_classification() -> None: model = MagicMock() model.pooler.get_supported_tasks.return_value = {"embed", "token_classify"} vllm_config = MagicMock() @@ -255,10 +254,10 @@ def test_pooling_runner_filters_decoder_token_classification() -> None: runner = PoolingRunner(model, vllm_config) - assert runner.supported_tasks == {"embed"} + assert runner.supported_tasks == {"embed", "token_classify"} -def test_pooling_runner_filters_decoder_token_embedding() -> None: +def test_pooling_runner_keeps_decoder_token_embedding() -> None: model = MagicMock() model.pooler.get_supported_tasks.return_value = {"embed", "token_embed"} vllm_config = MagicMock() @@ -268,12 +267,10 @@ def test_pooling_runner_filters_decoder_token_embedding() -> None: runner = PoolingRunner(model, vllm_config) - assert runner.supported_tasks == {"embed"} + assert runner.supported_tasks == {"embed", "token_embed"} -def test_pooling_runner_gates_embed_token_classification_to_encoder() -> None: - # BGE-M3's combined task emits per-token weights alongside the embedding, - # so it is enabled only where prefill is unchunked. +def test_pooling_runner_supports_embed_token_classification() -> None: model = MagicMock() model.pooler.get_supported_tasks.return_value = {"embed", "embed&token_classify"} vllm_config = MagicMock() @@ -285,7 +282,3 @@ def test_pooling_runner_gates_embed_token_classification_to_encoder() -> None: "embed", "embed&token_classify", } - - vllm_config.model_config.attn_type = "decoder" - vllm_config.model_config.get_pooling_task.return_value = "embed" - assert PoolingRunner(model, vllm_config).supported_tasks == {"embed"} diff --git a/vllm/model_executor/layers/pooler/tokwise/methods.py b/vllm/model_executor/layers/pooler/tokwise/methods.py index 259bfbfa46..98b58c286c 100644 --- a/vllm/model_executor/layers/pooler/tokwise/methods.py +++ b/vllm/model_executor/layers/pooler/tokwise/methods.py @@ -40,9 +40,15 @@ class AllPool(TokenPoolingMethod): scheduler_config = vllm_config.scheduler_config self.enable_chunked_prefill = scheduler_config.enable_chunked_prefill + # With async scheduling, the next step may overwrite the hidden states + # buffer before outputs are copied to CPU, so they must own their storage. + self.clone_finished = bool(scheduler_config.async_scheduling) def extra_repr(self) -> str: - return f"enable_chunked_prefill={self.enable_chunked_prefill}" + return ( + f"enable_chunked_prefill={self.enable_chunked_prefill}, " + f"clone_finished={self.clone_finished}" + ) def forward( self, @@ -58,18 +64,29 @@ class AllPool(TokenPoolingMethod): ) if not self.enable_chunked_prefill: + if self.clone_finished: + return [hs.clone() for hs in hidden_states_lst] return hidden_states_lst pooling_states = pooling_metadata.pooling_states + finished_mask = pooling_cursor.is_finished().tolist() # If chunked_prefill is enabled # 1. first store the chunked hidden_states in pooling_states.hidden_states_cache - for p, hs_chunk in zip(pooling_states, hidden_states_lst): - p.hidden_states_cache.append(hs_chunk) + for p, hs_chunk, finished in zip( + pooling_states, hidden_states_lst, finished_mask + ): + # Own data retained across steps or returned without concatenation. + needs_owned_storage = not finished or ( + self.clone_finished and not p.hidden_states_cache + ) + p.hidden_states_cache.append( + hs_chunk.clone() if needs_owned_storage else hs_chunk + ) # 2. Once prefill is finished, send hidden_states_cache to PoolerHead output_list = list[TokenPoolingMethodOutputItem]() - for p, finished in zip(pooling_states, pooling_cursor.is_finished()): + for p, finished in zip(pooling_states, finished_mask): if finished: hidden_states_cache = p.hidden_states_cache if len(hidden_states_cache) == 1: diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 640f3ea202..5fa39436b1 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -313,9 +313,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): # Do not rely on pooling_runner here, since this information is needed # on the first PP rank, while pooling_runner is only initialized # on the last PP rank. - tasks.extend( - PoolingRunner.get_supported_tasks(self.model, self.model_config) - ) + tasks.extend(PoolingRunner.get_supported_tasks(self.model)) return tuple(tasks) def load_model(self, load_dummy_weights: bool = False, *args, **kwargs) -> None: diff --git a/vllm/v1/worker/gpu/pool/pooling_runner.py b/vllm/v1/worker/gpu/pool/pooling_runner.py index 9aa3be93c2..29fae75273 100644 --- a/vllm/v1/worker/gpu/pool/pooling_runner.py +++ b/vllm/v1/worker/gpu/pool/pooling_runner.py @@ -6,7 +6,7 @@ import numpy as np import torch import torch.nn as nn -from vllm.config import ModelConfig, VllmConfig +from vllm.config import VllmConfig from vllm.model_executor.models import VllmModelForPooling, is_pooling_model from vllm.pooling_params import PoolingParams from vllm.tasks import PoolingTask @@ -20,11 +20,6 @@ from vllm.v1.worker.gpu.states import RequestState _SUPPORTED_TASKS: frozenset[PoolingTask] = frozenset( {"embed", "classify", "token_embed", "token_classify", "embed&token_classify"} ) -# `embed&token_classify` concatenates a fixed-size embedding with per-token -# weights, so its output length scales with the prompt like the others here. -_TOKEN_TASKS: frozenset[PoolingTask] = frozenset( - {"token_embed", "token_classify", "embed&token_classify"} -) class PoolingRunner: @@ -33,26 +28,23 @@ class PoolingRunner: self.model_config = vllm_config.model_config self.max_num_reqs = vllm_config.scheduler_config.max_num_seqs model_tasks = tuple(sorted(self.model.pooler.get_supported_tasks())) - enabled_tasks = self._get_enabled_tasks(self.model_config) selected_task = self.model_config.get_pooling_task(model_tasks) - if selected_task not in enabled_tasks: + if selected_task not in _SUPPORTED_TASKS: hint = ( "Set an explicitly supported task or VLLM_USE_V2_MODEL_RUNNER=0." - if enabled_tasks.intersection(model_tasks) + if _SUPPORTED_TASKS.intersection(model_tasks) else "Set VLLM_USE_V2_MODEL_RUNNER=0 to use this model." ) raise ValueError( "Model Runner V2 supports pooling tasks " - f"{sorted(enabled_tasks)}, but this model selects " + f"{sorted(_SUPPORTED_TASKS)}, but this model selects " f"{selected_task!r} from {list(model_tasks)}. {hint}" ) - self.supported_tasks = frozenset( - self.get_supported_tasks(model, self.model_config) - ) + self.supported_tasks = frozenset(self.get_supported_tasks(model)) if not self.supported_tasks: raise ValueError( "Model Runner V2 supports pooling tasks " - f"{sorted(enabled_tasks)}, but this model supports " + f"{sorted(_SUPPORTED_TASKS)}, but this model supports " f"{list(model_tasks)}. " "Set VLLM_USE_V2_MODEL_RUNNER=0 to use this model." ) @@ -62,20 +54,10 @@ class PoolingRunner: self.late_interaction_runner = LateInteractionRunner() @staticmethod - def _get_enabled_tasks(model_config: ModelConfig) -> frozenset[PoolingTask]: - # Token-wise pooling is validated only for unchunked encoder-only prefill. - if model_config.attn_type == "encoder_only": - return _SUPPORTED_TASKS - return _SUPPORTED_TASKS - _TOKEN_TASKS - - @staticmethod - def get_supported_tasks( - model: nn.Module, model_config: ModelConfig - ) -> list[PoolingTask]: + def get_supported_tasks(model: nn.Module) -> list[PoolingTask]: if not is_pooling_model(model): return [] - enabled_tasks = PoolingRunner._get_enabled_tasks(model_config) - return sorted(model.pooler.get_supported_tasks() & enabled_tasks) + return sorted(model.pooler.get_supported_tasks() & _SUPPORTED_TASKS) def add_request( self,