[Model Runner V2] Enable encoder token embedding (#50574)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
@@ -15,8 +15,10 @@ from vllm.model_executor.models.bert import (
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
from vllm.v1.pool.late_interaction_runner import LateInteractionRunner
|
||||
from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
|
||||
from vllm.v1.worker.gpu.pool.pooling_runner import PoolingRunner
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
|
||||
@@ -138,9 +140,11 @@ def test_pooling_runner_stores_only_required_token_ids() -> None:
|
||||
runner.pooling_params = {}
|
||||
runner.pooling_states = {}
|
||||
runner.prompt_token_ids = {}
|
||||
runner.late_interaction_runner = MagicMock()
|
||||
|
||||
runner.add_request(1, PoolingParams(task="embed"), [101, 102])
|
||||
runner.add_request("req-1", 1, PoolingParams(task="embed"), [101, 102])
|
||||
runner.add_request(
|
||||
"req-2",
|
||||
2,
|
||||
PoolingParams(task="embed", requires_token_ids=True),
|
||||
[101, 11, 102],
|
||||
@@ -150,6 +154,42 @@ def test_pooling_runner_stores_only_required_token_ids() -> None:
|
||||
torch.testing.assert_close(runner.prompt_token_ids[2], torch.tensor([101, 11, 102]))
|
||||
|
||||
|
||||
def test_pooling_runner_releases_aborted_late_interaction_doc() -> None:
|
||||
runner = PoolingRunner.__new__(PoolingRunner)
|
||||
runner.late_interaction_runner = LateInteractionRunner()
|
||||
|
||||
query_key = "query-abort"
|
||||
late_interaction_runner = runner.late_interaction_runner
|
||||
late_interaction_runner._query_cache[query_key] = torch.ones(2, 4)
|
||||
late_interaction_runner._query_uses[query_key] = 1
|
||||
late_interaction_runner._doc_query_keys["doc-req"] = query_key
|
||||
|
||||
runner.on_requests_finished({"doc-req"})
|
||||
|
||||
assert not late_interaction_runner._query_cache
|
||||
assert not late_interaction_runner._query_uses
|
||||
assert not late_interaction_runner._doc_query_keys
|
||||
|
||||
|
||||
def test_encoder_cache_reset_clears_late_interaction_state() -> None:
|
||||
# Cached query embeddings are only invalidated by weight reloads, which
|
||||
# reach the runner via reset_encoder_cache. Resetting the multi-modal
|
||||
# cache is unrelated and must leave them intact.
|
||||
runner = GPUModelRunner.__new__(GPUModelRunner)
|
||||
runner.encoder_cache = MagicMock()
|
||||
runner.pooling_runner = MagicMock()
|
||||
|
||||
runner.reset_mm_cache()
|
||||
|
||||
runner.encoder_cache.reset_mm_cache.assert_called_once_with()
|
||||
runner.pooling_runner.clear.assert_not_called()
|
||||
|
||||
runner.reset_encoder_cache()
|
||||
|
||||
runner.encoder_cache.reset_encoder_cache.assert_called_once_with()
|
||||
runner.pooling_runner.clear.assert_called_once_with()
|
||||
|
||||
|
||||
def test_pooling_runner_rejects_unsupported_selected_task() -> None:
|
||||
model = MagicMock()
|
||||
model.pooler.get_supported_tasks.return_value = {
|
||||
@@ -182,6 +222,19 @@ def test_pooling_runner_supports_encoder_token_classification() -> None:
|
||||
assert runner.supported_tasks == {"token_classify"}
|
||||
|
||||
|
||||
def test_pooling_runner_supports_encoder_token_embedding() -> None:
|
||||
model = MagicMock()
|
||||
model.pooler.get_supported_tasks.return_value = {"token_embed"}
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.scheduler_config.max_num_seqs = 2
|
||||
vllm_config.model_config.attn_type = "encoder_only"
|
||||
vllm_config.model_config.get_pooling_task.return_value = "token_embed"
|
||||
|
||||
runner = PoolingRunner(model, vllm_config)
|
||||
|
||||
assert runner.supported_tasks == {"token_embed"}
|
||||
|
||||
|
||||
def test_pooling_runner_rejects_decoder_token_classification() -> None:
|
||||
model = MagicMock()
|
||||
model.pooler.get_supported_tasks.return_value = {"token_classify"}
|
||||
@@ -207,3 +260,16 @@ def test_pooling_runner_filters_decoder_token_classification() -> None:
|
||||
runner = PoolingRunner(model, vllm_config)
|
||||
|
||||
assert runner.supported_tasks == {"embed"}
|
||||
|
||||
|
||||
def test_pooling_runner_filters_decoder_token_embedding() -> None:
|
||||
model = MagicMock()
|
||||
model.pooler.get_supported_tasks.return_value = {"embed", "token_embed"}
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.scheduler_config.max_num_seqs = 2
|
||||
vllm_config.model_config.attn_type = "decoder"
|
||||
vllm_config.model_config.get_pooling_task.return_value = "embed"
|
||||
|
||||
runner = PoolingRunner(model, vllm_config)
|
||||
|
||||
assert runner.supported_tasks == {"embed"}
|
||||
|
||||
Reference in New Issue
Block a user