[ModelRunner v2] Enable decoder token-wise pooling (#50931)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
Taneem Ibrahim
2026-08-06 22:21:52 +00:00
committed by GitHub
parent 4d341ca829
commit 9464529612
8 changed files with 86 additions and 58 deletions
@@ -256,6 +256,7 @@ class TestGetSeqPoolingMethod:
@dataclass
class _FakeSchedulerConfig:
enable_chunked_prefill: bool = False
async_scheduling: bool = False
@dataclass
@@ -265,10 +266,13 @@ class _FakeVllmConfig:
class TestAllPool:
@staticmethod
def _make_all_pool(*, chunked: bool = False) -> AllPool:
def _make_all_pool(
*, chunked: bool = False, async_scheduling: bool = False
) -> AllPool:
fake_config = _FakeVllmConfig(
scheduler_config=_FakeSchedulerConfig(
enable_chunked_prefill=chunked,
async_scheduling=async_scheduling,
),
)
with patch(
@@ -319,6 +323,8 @@ class TestAllPool:
)
out1 = pooler(chunk1, metadata1)
assert out1[0] is None
expected_chunk1 = chunk1.clone()
chunk1.zero_()
chunk2 = torch.tensor([[5.0, 6.0], [7.0, 8.0]])
metadata2 = _make_metadata(
@@ -329,19 +335,37 @@ class TestAllPool:
metadata2.pooling_states = metadata1.pooling_states
out2 = pooler(chunk2, metadata2)
assert out2[0] is not None
expected = torch.cat([chunk1, chunk2], dim=0)
expected = torch.cat([expected_chunk1, chunk2], dim=0)
assert torch.equal(out2[0], expected)
def test_chunked_prefill_single_shot_matches_non_chunked(self):
pooler = self._make_all_pool(chunked=True)
@pytest.mark.parametrize("async_scheduling", [False, True])
def test_chunked_prefill_single_shot_matches_non_chunked(
self, async_scheduling: bool
):
pooler = self._make_all_pool(chunked=True, async_scheduling=async_scheduling)
hidden = torch.tensor(
[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0], [9.0, 10.0]]
)
expected = [hidden[:2].clone(), hidden[2:].clone()]
metadata = _make_metadata([2, 3])
out = pooler(hidden, metadata)
if async_scheduling:
# Finished outputs must own their storage under async scheduling.
hidden.zero_()
assert len(out) == 2
assert torch.equal(out[0], hidden[:2])
assert torch.equal(out[1], hidden[2:])
assert torch.equal(out[0], expected[0])
assert torch.equal(out[1], expected[1])
def test_non_chunked_owns_output_under_async_scheduling(self):
pooler = self._make_all_pool(async_scheduling=True)
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
expected = [hidden[:2].clone(), hidden[2:].clone()]
metadata = _make_metadata([2, 1])
out = pooler(hidden, metadata)
hidden.zero_()
assert len(out) == 2
assert torch.equal(out[0], expected[0])
assert torch.equal(out[1], expected[1])
def test_chunked_prefill_mixed_finished_unfinished(self):
pooler = self._make_all_pool(chunked=True)
@@ -15,9 +15,10 @@ from vllm.config import PoolerConfig
["Qwen/Qwen3-Embedding-0.6B"],
)
@torch.inference_mode
def test_embed_models(hf_runner, vllm_runner, monkeypatch, model: str):
# Keep token_embed on MRV1 when sequence pooling becomes MRV2 by default.
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0")
def test_decoder_token_embed_model_runner_v2(
hf_runner, vllm_runner, monkeypatch, model: str
):
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
chunk_size = 10
n_prompt_tokens = [55, 56, 57]
token_prompts = [[1024 + i for i in range(n)] for n in n_prompt_tokens]
@@ -33,6 +34,7 @@ def test_embed_models(hf_runner, vllm_runner, monkeypatch, model: str):
enable_chunked_prefill=True,
enable_prefix_caching=True,
) as vllm_model:
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
vllm_outputs = vllm_model.token_embed(
[TokensPrompt(prompt_token_ids=t) for t in token_prompts],
)
@@ -45,8 +45,10 @@ INSTRUCTION = (
)
def test_offline(vllm_runner):
def test_offline(vllm_runner, monkeypatch):
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
with vllm_runner(model_name, runner="pooling") as llm_runner:
assert llm_runner.llm.llm_engine.vllm_config.use_v2_model_runner
llm = llm_runner.get_llm()
_test_offline_1_v_1(llm)
_test_offline_1_v_n(llm)
@@ -98,6 +98,7 @@ def test_prm_models(
hf_runner,
vllm_runner,
math_step_prompts,
monkeypatch,
model: str,
dtype: str,
) -> None:
@@ -107,12 +108,14 @@ def test_prm_models(
if current_platform.is_cpu():
pytest.skip("CPU only supports V1")
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
with vllm_runner(
model,
max_model_len=1024,
dtype=dtype,
kv_cache_memory_bytes=64 * MiB_bytes,
) as vllm_model:
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
vllm_outputs = vllm_model.token_classify(math_step_prompts)
with hf_runner(model, dtype=dtype, auto_cls=AutoModel) as hf_model:
@@ -145,18 +148,25 @@ def test_prm_models(
def test_prm_models_with_golden_outputs(
vllm_runner,
math_step_prompts,
monkeypatch,
model: str,
dtype: str,
) -> None:
if not FIXTURE_REWARD_RESULT.get(model):
pytest.skip(f"No available golden outputs for {model}.")
use_v2_model_runner = not current_platform.is_cpu()
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if use_v2_model_runner else "0")
with vllm_runner(
model,
max_model_len=1024,
dtype=dtype,
kv_cache_memory_bytes=64 * MiB_bytes,
) as vllm_model:
assert (
vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
is use_v2_model_runner
)
vllm_outputs = vllm_model.token_classify(math_step_prompts)
golden_outputs = load_reward_outputs(FIXTURE_REWARD_RESULT[model])
@@ -231,7 +231,7 @@ def test_pooling_runner_supports_encoder_token_embedding() -> None:
assert runner.supported_tasks == {"token_embed"}
def test_pooling_runner_rejects_decoder_token_classification() -> None:
def test_pooling_runner_supports_decoder_token_classification() -> None:
model = MagicMock()
model.pooler.get_supported_tasks.return_value = {"token_classify"}
vllm_config = MagicMock()
@@ -239,13 +239,12 @@ def test_pooling_runner_rejects_decoder_token_classification() -> None:
vllm_config.model_config.attn_type = "decoder"
vllm_config.model_config.get_pooling_task.return_value = "token_classify"
with pytest.raises(ValueError, match="selects 'token_classify'") as exc_info:
PoolingRunner(model, vllm_config)
runner = PoolingRunner(model, vllm_config)
assert "Set an explicitly supported task" not in str(exc_info.value)
assert runner.supported_tasks == {"token_classify"}
def test_pooling_runner_filters_decoder_token_classification() -> None:
def test_pooling_runner_keeps_decoder_token_classification() -> None:
model = MagicMock()
model.pooler.get_supported_tasks.return_value = {"embed", "token_classify"}
vllm_config = MagicMock()
@@ -255,10 +254,10 @@ def test_pooling_runner_filters_decoder_token_classification() -> None:
runner = PoolingRunner(model, vllm_config)
assert runner.supported_tasks == {"embed"}
assert runner.supported_tasks == {"embed", "token_classify"}
def test_pooling_runner_filters_decoder_token_embedding() -> None:
def test_pooling_runner_keeps_decoder_token_embedding() -> None:
model = MagicMock()
model.pooler.get_supported_tasks.return_value = {"embed", "token_embed"}
vllm_config = MagicMock()
@@ -268,12 +267,10 @@ def test_pooling_runner_filters_decoder_token_embedding() -> None:
runner = PoolingRunner(model, vllm_config)
assert runner.supported_tasks == {"embed"}
assert runner.supported_tasks == {"embed", "token_embed"}
def test_pooling_runner_gates_embed_token_classification_to_encoder() -> None:
# BGE-M3's combined task emits per-token weights alongside the embedding,
# so it is enabled only where prefill is unchunked.
def test_pooling_runner_supports_embed_token_classification() -> None:
model = MagicMock()
model.pooler.get_supported_tasks.return_value = {"embed", "embed&token_classify"}
vllm_config = MagicMock()
@@ -285,7 +282,3 @@ def test_pooling_runner_gates_embed_token_classification_to_encoder() -> None:
"embed",
"embed&token_classify",
}
vllm_config.model_config.attn_type = "decoder"
vllm_config.model_config.get_pooling_task.return_value = "embed"
assert PoolingRunner(model, vllm_config).supported_tasks == {"embed"}