[ModelRunner v2] Enable decoder token-wise pooling (#50931)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
@@ -256,6 +256,7 @@ class TestGetSeqPoolingMethod:
|
||||
@dataclass
|
||||
class _FakeSchedulerConfig:
|
||||
enable_chunked_prefill: bool = False
|
||||
async_scheduling: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -265,10 +266,13 @@ class _FakeVllmConfig:
|
||||
|
||||
class TestAllPool:
|
||||
@staticmethod
|
||||
def _make_all_pool(*, chunked: bool = False) -> AllPool:
|
||||
def _make_all_pool(
|
||||
*, chunked: bool = False, async_scheduling: bool = False
|
||||
) -> AllPool:
|
||||
fake_config = _FakeVllmConfig(
|
||||
scheduler_config=_FakeSchedulerConfig(
|
||||
enable_chunked_prefill=chunked,
|
||||
async_scheduling=async_scheduling,
|
||||
),
|
||||
)
|
||||
with patch(
|
||||
@@ -319,6 +323,8 @@ class TestAllPool:
|
||||
)
|
||||
out1 = pooler(chunk1, metadata1)
|
||||
assert out1[0] is None
|
||||
expected_chunk1 = chunk1.clone()
|
||||
chunk1.zero_()
|
||||
|
||||
chunk2 = torch.tensor([[5.0, 6.0], [7.0, 8.0]])
|
||||
metadata2 = _make_metadata(
|
||||
@@ -329,19 +335,37 @@ class TestAllPool:
|
||||
metadata2.pooling_states = metadata1.pooling_states
|
||||
out2 = pooler(chunk2, metadata2)
|
||||
assert out2[0] is not None
|
||||
expected = torch.cat([chunk1, chunk2], dim=0)
|
||||
expected = torch.cat([expected_chunk1, chunk2], dim=0)
|
||||
assert torch.equal(out2[0], expected)
|
||||
|
||||
def test_chunked_prefill_single_shot_matches_non_chunked(self):
|
||||
pooler = self._make_all_pool(chunked=True)
|
||||
@pytest.mark.parametrize("async_scheduling", [False, True])
|
||||
def test_chunked_prefill_single_shot_matches_non_chunked(
|
||||
self, async_scheduling: bool
|
||||
):
|
||||
pooler = self._make_all_pool(chunked=True, async_scheduling=async_scheduling)
|
||||
hidden = torch.tensor(
|
||||
[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0], [9.0, 10.0]]
|
||||
)
|
||||
expected = [hidden[:2].clone(), hidden[2:].clone()]
|
||||
metadata = _make_metadata([2, 3])
|
||||
out = pooler(hidden, metadata)
|
||||
if async_scheduling:
|
||||
# Finished outputs must own their storage under async scheduling.
|
||||
hidden.zero_()
|
||||
assert len(out) == 2
|
||||
assert torch.equal(out[0], hidden[:2])
|
||||
assert torch.equal(out[1], hidden[2:])
|
||||
assert torch.equal(out[0], expected[0])
|
||||
assert torch.equal(out[1], expected[1])
|
||||
|
||||
def test_non_chunked_owns_output_under_async_scheduling(self):
|
||||
pooler = self._make_all_pool(async_scheduling=True)
|
||||
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
|
||||
expected = [hidden[:2].clone(), hidden[2:].clone()]
|
||||
metadata = _make_metadata([2, 1])
|
||||
out = pooler(hidden, metadata)
|
||||
hidden.zero_()
|
||||
assert len(out) == 2
|
||||
assert torch.equal(out[0], expected[0])
|
||||
assert torch.equal(out[1], expected[1])
|
||||
|
||||
def test_chunked_prefill_mixed_finished_unfinished(self):
|
||||
pooler = self._make_all_pool(chunked=True)
|
||||
|
||||
@@ -15,9 +15,10 @@ from vllm.config import PoolerConfig
|
||||
["Qwen/Qwen3-Embedding-0.6B"],
|
||||
)
|
||||
@torch.inference_mode
|
||||
def test_embed_models(hf_runner, vllm_runner, monkeypatch, model: str):
|
||||
# Keep token_embed on MRV1 when sequence pooling becomes MRV2 by default.
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0")
|
||||
def test_decoder_token_embed_model_runner_v2(
|
||||
hf_runner, vllm_runner, monkeypatch, model: str
|
||||
):
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
chunk_size = 10
|
||||
n_prompt_tokens = [55, 56, 57]
|
||||
token_prompts = [[1024 + i for i in range(n)] for n in n_prompt_tokens]
|
||||
@@ -33,6 +34,7 @@ def test_embed_models(hf_runner, vllm_runner, monkeypatch, model: str):
|
||||
enable_chunked_prefill=True,
|
||||
enable_prefix_caching=True,
|
||||
) as vllm_model:
|
||||
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
vllm_outputs = vllm_model.token_embed(
|
||||
[TokensPrompt(prompt_token_ids=t) for t in token_prompts],
|
||||
)
|
||||
|
||||
@@ -45,8 +45,10 @@ INSTRUCTION = (
|
||||
)
|
||||
|
||||
|
||||
def test_offline(vllm_runner):
|
||||
def test_offline(vllm_runner, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
with vllm_runner(model_name, runner="pooling") as llm_runner:
|
||||
assert llm_runner.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
llm = llm_runner.get_llm()
|
||||
_test_offline_1_v_1(llm)
|
||||
_test_offline_1_v_n(llm)
|
||||
|
||||
@@ -98,6 +98,7 @@ def test_prm_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
math_step_prompts,
|
||||
monkeypatch,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
@@ -107,12 +108,14 @@ def test_prm_models(
|
||||
if current_platform.is_cpu():
|
||||
pytest.skip("CPU only supports V1")
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=1024,
|
||||
dtype=dtype,
|
||||
kv_cache_memory_bytes=64 * MiB_bytes,
|
||||
) as vllm_model:
|
||||
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
vllm_outputs = vllm_model.token_classify(math_step_prompts)
|
||||
|
||||
with hf_runner(model, dtype=dtype, auto_cls=AutoModel) as hf_model:
|
||||
@@ -145,18 +148,25 @@ def test_prm_models(
|
||||
def test_prm_models_with_golden_outputs(
|
||||
vllm_runner,
|
||||
math_step_prompts,
|
||||
monkeypatch,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
if not FIXTURE_REWARD_RESULT.get(model):
|
||||
pytest.skip(f"No available golden outputs for {model}.")
|
||||
|
||||
use_v2_model_runner = not current_platform.is_cpu()
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if use_v2_model_runner else "0")
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=1024,
|
||||
dtype=dtype,
|
||||
kv_cache_memory_bytes=64 * MiB_bytes,
|
||||
) as vllm_model:
|
||||
assert (
|
||||
vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
is use_v2_model_runner
|
||||
)
|
||||
vllm_outputs = vllm_model.token_classify(math_step_prompts)
|
||||
|
||||
golden_outputs = load_reward_outputs(FIXTURE_REWARD_RESULT[model])
|
||||
|
||||
@@ -231,7 +231,7 @@ def test_pooling_runner_supports_encoder_token_embedding() -> None:
|
||||
assert runner.supported_tasks == {"token_embed"}
|
||||
|
||||
|
||||
def test_pooling_runner_rejects_decoder_token_classification() -> None:
|
||||
def test_pooling_runner_supports_decoder_token_classification() -> None:
|
||||
model = MagicMock()
|
||||
model.pooler.get_supported_tasks.return_value = {"token_classify"}
|
||||
vllm_config = MagicMock()
|
||||
@@ -239,13 +239,12 @@ def test_pooling_runner_rejects_decoder_token_classification() -> None:
|
||||
vllm_config.model_config.attn_type = "decoder"
|
||||
vllm_config.model_config.get_pooling_task.return_value = "token_classify"
|
||||
|
||||
with pytest.raises(ValueError, match="selects 'token_classify'") as exc_info:
|
||||
PoolingRunner(model, vllm_config)
|
||||
runner = PoolingRunner(model, vllm_config)
|
||||
|
||||
assert "Set an explicitly supported task" not in str(exc_info.value)
|
||||
assert runner.supported_tasks == {"token_classify"}
|
||||
|
||||
|
||||
def test_pooling_runner_filters_decoder_token_classification() -> None:
|
||||
def test_pooling_runner_keeps_decoder_token_classification() -> None:
|
||||
model = MagicMock()
|
||||
model.pooler.get_supported_tasks.return_value = {"embed", "token_classify"}
|
||||
vllm_config = MagicMock()
|
||||
@@ -255,10 +254,10 @@ def test_pooling_runner_filters_decoder_token_classification() -> None:
|
||||
|
||||
runner = PoolingRunner(model, vllm_config)
|
||||
|
||||
assert runner.supported_tasks == {"embed"}
|
||||
assert runner.supported_tasks == {"embed", "token_classify"}
|
||||
|
||||
|
||||
def test_pooling_runner_filters_decoder_token_embedding() -> None:
|
||||
def test_pooling_runner_keeps_decoder_token_embedding() -> None:
|
||||
model = MagicMock()
|
||||
model.pooler.get_supported_tasks.return_value = {"embed", "token_embed"}
|
||||
vllm_config = MagicMock()
|
||||
@@ -268,12 +267,10 @@ def test_pooling_runner_filters_decoder_token_embedding() -> None:
|
||||
|
||||
runner = PoolingRunner(model, vllm_config)
|
||||
|
||||
assert runner.supported_tasks == {"embed"}
|
||||
assert runner.supported_tasks == {"embed", "token_embed"}
|
||||
|
||||
|
||||
def test_pooling_runner_gates_embed_token_classification_to_encoder() -> None:
|
||||
# BGE-M3's combined task emits per-token weights alongside the embedding,
|
||||
# so it is enabled only where prefill is unchunked.
|
||||
def test_pooling_runner_supports_embed_token_classification() -> None:
|
||||
model = MagicMock()
|
||||
model.pooler.get_supported_tasks.return_value = {"embed", "embed&token_classify"}
|
||||
vllm_config = MagicMock()
|
||||
@@ -285,7 +282,3 @@ def test_pooling_runner_gates_embed_token_classification_to_encoder() -> None:
|
||||
"embed",
|
||||
"embed&token_classify",
|
||||
}
|
||||
|
||||
vllm_config.model_config.attn_type = "decoder"
|
||||
vllm_config.model_config.get_pooling_task.return_value = "embed"
|
||||
assert PoolingRunner(model, vllm_config).supported_tasks == {"embed"}
|
||||
|
||||
@@ -40,9 +40,15 @@ class AllPool(TokenPoolingMethod):
|
||||
scheduler_config = vllm_config.scheduler_config
|
||||
|
||||
self.enable_chunked_prefill = scheduler_config.enable_chunked_prefill
|
||||
# With async scheduling, the next step may overwrite the hidden states
|
||||
# buffer before outputs are copied to CPU, so they must own their storage.
|
||||
self.clone_finished = bool(scheduler_config.async_scheduling)
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
return f"enable_chunked_prefill={self.enable_chunked_prefill}"
|
||||
return (
|
||||
f"enable_chunked_prefill={self.enable_chunked_prefill}, "
|
||||
f"clone_finished={self.clone_finished}"
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -58,18 +64,29 @@ class AllPool(TokenPoolingMethod):
|
||||
)
|
||||
|
||||
if not self.enable_chunked_prefill:
|
||||
if self.clone_finished:
|
||||
return [hs.clone() for hs in hidden_states_lst]
|
||||
return hidden_states_lst
|
||||
|
||||
pooling_states = pooling_metadata.pooling_states
|
||||
finished_mask = pooling_cursor.is_finished().tolist()
|
||||
|
||||
# If chunked_prefill is enabled
|
||||
# 1. first store the chunked hidden_states in pooling_states.hidden_states_cache
|
||||
for p, hs_chunk in zip(pooling_states, hidden_states_lst):
|
||||
p.hidden_states_cache.append(hs_chunk)
|
||||
for p, hs_chunk, finished in zip(
|
||||
pooling_states, hidden_states_lst, finished_mask
|
||||
):
|
||||
# Own data retained across steps or returned without concatenation.
|
||||
needs_owned_storage = not finished or (
|
||||
self.clone_finished and not p.hidden_states_cache
|
||||
)
|
||||
p.hidden_states_cache.append(
|
||||
hs_chunk.clone() if needs_owned_storage else hs_chunk
|
||||
)
|
||||
|
||||
# 2. Once prefill is finished, send hidden_states_cache to PoolerHead
|
||||
output_list = list[TokenPoolingMethodOutputItem]()
|
||||
for p, finished in zip(pooling_states, pooling_cursor.is_finished()):
|
||||
for p, finished in zip(pooling_states, finished_mask):
|
||||
if finished:
|
||||
hidden_states_cache = p.hidden_states_cache
|
||||
if len(hidden_states_cache) == 1:
|
||||
|
||||
@@ -313,9 +313,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# Do not rely on pooling_runner here, since this information is needed
|
||||
# on the first PP rank, while pooling_runner is only initialized
|
||||
# on the last PP rank.
|
||||
tasks.extend(
|
||||
PoolingRunner.get_supported_tasks(self.model, self.model_config)
|
||||
)
|
||||
tasks.extend(PoolingRunner.get_supported_tasks(self.model))
|
||||
return tuple(tasks)
|
||||
|
||||
def load_model(self, load_dummy_weights: bool = False, *args, **kwargs) -> None:
|
||||
|
||||
@@ -6,7 +6,7 @@ import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.models import VllmModelForPooling, is_pooling_model
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.tasks import PoolingTask
|
||||
@@ -20,11 +20,6 @@ from vllm.v1.worker.gpu.states import RequestState
|
||||
_SUPPORTED_TASKS: frozenset[PoolingTask] = frozenset(
|
||||
{"embed", "classify", "token_embed", "token_classify", "embed&token_classify"}
|
||||
)
|
||||
# `embed&token_classify` concatenates a fixed-size embedding with per-token
|
||||
# weights, so its output length scales with the prompt like the others here.
|
||||
_TOKEN_TASKS: frozenset[PoolingTask] = frozenset(
|
||||
{"token_embed", "token_classify", "embed&token_classify"}
|
||||
)
|
||||
|
||||
|
||||
class PoolingRunner:
|
||||
@@ -33,26 +28,23 @@ class PoolingRunner:
|
||||
self.model_config = vllm_config.model_config
|
||||
self.max_num_reqs = vllm_config.scheduler_config.max_num_seqs
|
||||
model_tasks = tuple(sorted(self.model.pooler.get_supported_tasks()))
|
||||
enabled_tasks = self._get_enabled_tasks(self.model_config)
|
||||
selected_task = self.model_config.get_pooling_task(model_tasks)
|
||||
if selected_task not in enabled_tasks:
|
||||
if selected_task not in _SUPPORTED_TASKS:
|
||||
hint = (
|
||||
"Set an explicitly supported task or VLLM_USE_V2_MODEL_RUNNER=0."
|
||||
if enabled_tasks.intersection(model_tasks)
|
||||
if _SUPPORTED_TASKS.intersection(model_tasks)
|
||||
else "Set VLLM_USE_V2_MODEL_RUNNER=0 to use this model."
|
||||
)
|
||||
raise ValueError(
|
||||
"Model Runner V2 supports pooling tasks "
|
||||
f"{sorted(enabled_tasks)}, but this model selects "
|
||||
f"{sorted(_SUPPORTED_TASKS)}, but this model selects "
|
||||
f"{selected_task!r} from {list(model_tasks)}. {hint}"
|
||||
)
|
||||
self.supported_tasks = frozenset(
|
||||
self.get_supported_tasks(model, self.model_config)
|
||||
)
|
||||
self.supported_tasks = frozenset(self.get_supported_tasks(model))
|
||||
if not self.supported_tasks:
|
||||
raise ValueError(
|
||||
"Model Runner V2 supports pooling tasks "
|
||||
f"{sorted(enabled_tasks)}, but this model supports "
|
||||
f"{sorted(_SUPPORTED_TASKS)}, but this model supports "
|
||||
f"{list(model_tasks)}. "
|
||||
"Set VLLM_USE_V2_MODEL_RUNNER=0 to use this model."
|
||||
)
|
||||
@@ -62,20 +54,10 @@ class PoolingRunner:
|
||||
self.late_interaction_runner = LateInteractionRunner()
|
||||
|
||||
@staticmethod
|
||||
def _get_enabled_tasks(model_config: ModelConfig) -> frozenset[PoolingTask]:
|
||||
# Token-wise pooling is validated only for unchunked encoder-only prefill.
|
||||
if model_config.attn_type == "encoder_only":
|
||||
return _SUPPORTED_TASKS
|
||||
return _SUPPORTED_TASKS - _TOKEN_TASKS
|
||||
|
||||
@staticmethod
|
||||
def get_supported_tasks(
|
||||
model: nn.Module, model_config: ModelConfig
|
||||
) -> list[PoolingTask]:
|
||||
def get_supported_tasks(model: nn.Module) -> list[PoolingTask]:
|
||||
if not is_pooling_model(model):
|
||||
return []
|
||||
enabled_tasks = PoolingRunner._get_enabled_tasks(model_config)
|
||||
return sorted(model.pooler.get_supported_tasks() & enabled_tasks)
|
||||
return sorted(model.pooler.get_supported_tasks() & _SUPPORTED_TASKS)
|
||||
|
||||
def add_request(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user