[ModelRunner v2] Enable decoder token-wise pooling (#50931)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
@@ -15,9 +15,10 @@ from vllm.config import PoolerConfig
|
||||
["Qwen/Qwen3-Embedding-0.6B"],
|
||||
)
|
||||
@torch.inference_mode
|
||||
def test_embed_models(hf_runner, vllm_runner, monkeypatch, model: str):
|
||||
# Keep token_embed on MRV1 when sequence pooling becomes MRV2 by default.
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0")
|
||||
def test_decoder_token_embed_model_runner_v2(
|
||||
hf_runner, vllm_runner, monkeypatch, model: str
|
||||
):
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
chunk_size = 10
|
||||
n_prompt_tokens = [55, 56, 57]
|
||||
token_prompts = [[1024 + i for i in range(n)] for n in n_prompt_tokens]
|
||||
@@ -33,6 +34,7 @@ def test_embed_models(hf_runner, vllm_runner, monkeypatch, model: str):
|
||||
enable_chunked_prefill=True,
|
||||
enable_prefix_caching=True,
|
||||
) as vllm_model:
|
||||
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
vllm_outputs = vllm_model.token_embed(
|
||||
[TokensPrompt(prompt_token_ids=t) for t in token_prompts],
|
||||
)
|
||||
|
||||
@@ -45,8 +45,10 @@ INSTRUCTION = (
|
||||
)
|
||||
|
||||
|
||||
def test_offline(vllm_runner):
|
||||
def test_offline(vllm_runner, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
with vllm_runner(model_name, runner="pooling") as llm_runner:
|
||||
assert llm_runner.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
llm = llm_runner.get_llm()
|
||||
_test_offline_1_v_1(llm)
|
||||
_test_offline_1_v_n(llm)
|
||||
|
||||
@@ -98,6 +98,7 @@ def test_prm_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
math_step_prompts,
|
||||
monkeypatch,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
@@ -107,12 +108,14 @@ def test_prm_models(
|
||||
if current_platform.is_cpu():
|
||||
pytest.skip("CPU only supports V1")
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=1024,
|
||||
dtype=dtype,
|
||||
kv_cache_memory_bytes=64 * MiB_bytes,
|
||||
) as vllm_model:
|
||||
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
vllm_outputs = vllm_model.token_classify(math_step_prompts)
|
||||
|
||||
with hf_runner(model, dtype=dtype, auto_cls=AutoModel) as hf_model:
|
||||
@@ -145,18 +148,25 @@ def test_prm_models(
|
||||
def test_prm_models_with_golden_outputs(
|
||||
vllm_runner,
|
||||
math_step_prompts,
|
||||
monkeypatch,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
if not FIXTURE_REWARD_RESULT.get(model):
|
||||
pytest.skip(f"No available golden outputs for {model}.")
|
||||
|
||||
use_v2_model_runner = not current_platform.is_cpu()
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if use_v2_model_runner else "0")
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=1024,
|
||||
dtype=dtype,
|
||||
kv_cache_memory_bytes=64 * MiB_bytes,
|
||||
) as vllm_model:
|
||||
assert (
|
||||
vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
is use_v2_model_runner
|
||||
)
|
||||
vllm_outputs = vllm_model.token_classify(math_step_prompts)
|
||||
|
||||
golden_outputs = load_reward_outputs(FIXTURE_REWARD_RESULT[model])
|
||||
|
||||
@@ -231,7 +231,7 @@ def test_pooling_runner_supports_encoder_token_embedding() -> None:
|
||||
assert runner.supported_tasks == {"token_embed"}
|
||||
|
||||
|
||||
def test_pooling_runner_rejects_decoder_token_classification() -> None:
|
||||
def test_pooling_runner_supports_decoder_token_classification() -> None:
|
||||
model = MagicMock()
|
||||
model.pooler.get_supported_tasks.return_value = {"token_classify"}
|
||||
vllm_config = MagicMock()
|
||||
@@ -239,13 +239,12 @@ def test_pooling_runner_rejects_decoder_token_classification() -> None:
|
||||
vllm_config.model_config.attn_type = "decoder"
|
||||
vllm_config.model_config.get_pooling_task.return_value = "token_classify"
|
||||
|
||||
with pytest.raises(ValueError, match="selects 'token_classify'") as exc_info:
|
||||
PoolingRunner(model, vllm_config)
|
||||
runner = PoolingRunner(model, vllm_config)
|
||||
|
||||
assert "Set an explicitly supported task" not in str(exc_info.value)
|
||||
assert runner.supported_tasks == {"token_classify"}
|
||||
|
||||
|
||||
def test_pooling_runner_filters_decoder_token_classification() -> None:
|
||||
def test_pooling_runner_keeps_decoder_token_classification() -> None:
|
||||
model = MagicMock()
|
||||
model.pooler.get_supported_tasks.return_value = {"embed", "token_classify"}
|
||||
vllm_config = MagicMock()
|
||||
@@ -255,10 +254,10 @@ def test_pooling_runner_filters_decoder_token_classification() -> None:
|
||||
|
||||
runner = PoolingRunner(model, vllm_config)
|
||||
|
||||
assert runner.supported_tasks == {"embed"}
|
||||
assert runner.supported_tasks == {"embed", "token_classify"}
|
||||
|
||||
|
||||
def test_pooling_runner_filters_decoder_token_embedding() -> None:
|
||||
def test_pooling_runner_keeps_decoder_token_embedding() -> None:
|
||||
model = MagicMock()
|
||||
model.pooler.get_supported_tasks.return_value = {"embed", "token_embed"}
|
||||
vllm_config = MagicMock()
|
||||
@@ -268,12 +267,10 @@ def test_pooling_runner_filters_decoder_token_embedding() -> None:
|
||||
|
||||
runner = PoolingRunner(model, vllm_config)
|
||||
|
||||
assert runner.supported_tasks == {"embed"}
|
||||
assert runner.supported_tasks == {"embed", "token_embed"}
|
||||
|
||||
|
||||
def test_pooling_runner_gates_embed_token_classification_to_encoder() -> None:
|
||||
# BGE-M3's combined task emits per-token weights alongside the embedding,
|
||||
# so it is enabled only where prefill is unchunked.
|
||||
def test_pooling_runner_supports_embed_token_classification() -> None:
|
||||
model = MagicMock()
|
||||
model.pooler.get_supported_tasks.return_value = {"embed", "embed&token_classify"}
|
||||
vllm_config = MagicMock()
|
||||
@@ -285,7 +282,3 @@ def test_pooling_runner_gates_embed_token_classification_to_encoder() -> None:
|
||||
"embed",
|
||||
"embed&token_classify",
|
||||
}
|
||||
|
||||
vllm_config.model_config.attn_type = "decoder"
|
||||
vllm_config.model_config.get_pooling_task.return_value = "embed"
|
||||
assert PoolingRunner(model, vllm_config).supported_tasks == {"embed"}
|
||||
|
||||
Reference in New Issue
Block a user