[CI] Shard (H200 MIG 35GB / MI355 DPX) Entrypoints Integration (Pooling) into named jobs (#58652)
Signed-off-by: Thang Nguyen <thangnguyenvn647@gmail.com> Signed-off-by: khluu <khluu000@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: khluu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
khluu
parent
004e37ece2
commit
413ae4b167
@@ -0,0 +1,111 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for ScoringIOProcessor post-tokenization helpers."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from tokenizers import Tokenizer, models, pre_tokenizers, processors
|
||||
from transformers import TokenizersBackend
|
||||
|
||||
from vllm import TokensPrompt
|
||||
from vllm.entrypoints.pooling.scoring.io_processor import (
|
||||
CrossEncoderIOProcessor,
|
||||
_apply_post_tokenization_to_token_type_ids,
|
||||
)
|
||||
from vllm.entrypoints.pooling.scoring.utils import compress_token_type_ids
|
||||
from vllm.renderers import TokenizeParams
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
|
||||
@dataclass
|
||||
class _DummyTokenizer:
|
||||
truncation_side: str = "left"
|
||||
# Outside the range of the prompt ids below, so a test can tell a pad
|
||||
# token apart from a real one.
|
||||
pad_token_id: int = 99999
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def llm_reranker_processor() -> CrossEncoderIOProcessor:
|
||||
vocab = {
|
||||
"[UNK]": 0,
|
||||
"[CLS]": 1,
|
||||
"[SEP]": 2,
|
||||
"a": 3,
|
||||
"bc": 4,
|
||||
"##b": 5,
|
||||
"##c": 6,
|
||||
}
|
||||
backend = Tokenizer(models.WordPiece(vocab, unk_token="[UNK]"))
|
||||
backend.pre_tokenizer = pre_tokenizers.WhitespaceSplit()
|
||||
backend.post_processor = processors.TemplateProcessing(
|
||||
single="[CLS] $A [SEP]",
|
||||
special_tokens=[("[CLS]", 1), ("[SEP]", 2)],
|
||||
)
|
||||
|
||||
processor = CrossEncoderIOProcessor.__new__(CrossEncoderIOProcessor)
|
||||
processor.model_config = SimpleNamespace(enable_prompt_embeds=False)
|
||||
processor.tokenizer = TokenizersBackend(
|
||||
tokenizer_object=backend,
|
||||
unk_token="[UNK]",
|
||||
cls_token="[CLS]",
|
||||
sep_token="[SEP]",
|
||||
)
|
||||
processor.supports_score_template = False
|
||||
processor.use_sep_token = False
|
||||
processor.model = None
|
||||
return processor
|
||||
|
||||
|
||||
def test_llm_reranker_tokenization_is_independent_of_nonbinding_doc_limit(
|
||||
llm_reranker_processor: CrossEncoderIOProcessor,
|
||||
):
|
||||
encode_kwargs = {"add_special_tokens": True}
|
||||
|
||||
_, uncapped = llm_reranker_processor.get_score_prompt("a", "bc", encode_kwargs)
|
||||
_, nonbinding = llm_reranker_processor.get_score_prompt(
|
||||
"a", "bc", encode_kwargs, max_tokens_per_doc=10
|
||||
)
|
||||
|
||||
expected = [1, 3, 5, 6, 2]
|
||||
assert uncapped["prompt_token_ids"] == expected
|
||||
assert nonbinding["prompt_token_ids"] == expected
|
||||
|
||||
|
||||
def test_token_type_ids_stay_aligned_with_a_truncated_padded_prompt():
|
||||
"""The cross-encoder segment boundary must survive truncate + pad.
|
||||
|
||||
`token_type_ids` are parallel to `prompt_token_ids` and are reduced to a
|
||||
single boundary index by `compress_token_type_ids`. If the two arrays are
|
||||
truncated and padded in different orders they no longer describe the same
|
||||
positions, and the model is told the query segment is empty.
|
||||
"""
|
||||
tokenizer = _DummyTokenizer()
|
||||
num_query, num_doc = 20, 30
|
||||
prompt = TokensPrompt(prompt_token_ids=list(range(num_query + num_doc)))
|
||||
token_type_ids = [0] * num_query + [1] * num_doc
|
||||
|
||||
tok_params = TokenizeParams(
|
||||
max_total_tokens=100,
|
||||
pad_prompt_tokens=-1,
|
||||
truncate_prompt_tokens=40,
|
||||
truncation_side="left",
|
||||
)
|
||||
|
||||
prompt = tok_params.apply_post_tokenization(tokenizer, prompt)
|
||||
token_type_ids = _apply_post_tokenization_to_token_type_ids(
|
||||
tokenizer, tok_params, token_type_ids
|
||||
)
|
||||
|
||||
prompt_token_ids = prompt["prompt_token_ids"]
|
||||
assert len(token_type_ids) == len(prompt_token_ids)
|
||||
|
||||
# Keeping the last 40 tokens drops the first 10 query tokens, so 10 query
|
||||
# tokens survive and the document starts at index 10.
|
||||
first_doc = compress_token_type_ids(token_type_ids)
|
||||
assert first_doc == 10
|
||||
assert prompt_token_ids[:first_doc] == list(range(10, num_query))
|
||||
assert prompt_token_ids[first_doc:40] == list(range(num_query, 50))
|
||||
@@ -0,0 +1,144 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for JinaRankingIOProcessor request building."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from tokenizers import Tokenizer, models, pre_tokenizers
|
||||
from transformers import TokenizersBackend
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.entrypoints.pooling.base.io_processor import PoolingIOProcessor
|
||||
from vllm.entrypoints.pooling.scoring.io_processor import JinaRankingIOProcessor
|
||||
from vllm.entrypoints.pooling.scoring.protocol import RerankRequest
|
||||
from vllm.entrypoints.pooling.scoring.typing import ScoringData
|
||||
from vllm.entrypoints.pooling.typing import OfflineScoringInputsContext
|
||||
from vllm.renderers import TokenizeParams
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
|
||||
def test_online_forwards_truncate_prompt_tokens_to_proxy(monkeypatch):
|
||||
"""The proxy request handed to the base factory must carry
|
||||
truncate_prompt_tokens/truncation_side from the real request.
|
||||
|
||||
JinaRankingIOProcessor swaps ctx.request for a proxy
|
||||
PoolingCompletionRequest before delegating to the base factory, which
|
||||
reads truncation off ctx.request. Dropping the fields on the proxy
|
||||
silently disables truncate_prompt_tokens for Jina rerank/score.
|
||||
"""
|
||||
proc = JinaRankingIOProcessor.__new__(JinaRankingIOProcessor)
|
||||
proc.valid_inputs_online = MagicMock(
|
||||
return_value=ScoringData(data_1=["query"], data_2=["doc"])
|
||||
)
|
||||
proc._get_token_limits = MagicMock(return_value=(0, 0))
|
||||
proc.ensure_str = MagicMock(side_effect=lambda data: list(data))
|
||||
proc.format_docs_prompts_func = MagicMock(return_value="formatted prompt")
|
||||
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def _spy_base(self, ctx):
|
||||
captured["truncate_prompt_tokens"] = ctx.request.truncate_prompt_tokens
|
||||
captured["truncation_side"] = ctx.request.truncation_side
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(PoolingIOProcessor, "get_request_factory_online", _spy_base)
|
||||
|
||||
request = RerankRequest(
|
||||
model="m",
|
||||
query="query",
|
||||
documents=["doc"],
|
||||
truncate_prompt_tokens=512,
|
||||
truncation_side="left",
|
||||
)
|
||||
ctx = MagicMock()
|
||||
ctx.request = request
|
||||
ctx.prompt_extras = None
|
||||
|
||||
proc.get_request_factory_online(ctx)
|
||||
|
||||
assert captured["truncate_prompt_tokens"] == 512
|
||||
assert captured["truncation_side"] == "left"
|
||||
# The real request is restored after delegating.
|
||||
assert ctx.request is request
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def offline_processor_and_context():
|
||||
backend = Tokenizer(models.WordLevel({"[UNK]": 0}, unk_token="[UNK]"))
|
||||
backend.pre_tokenizer = pre_tokenizers.WhitespaceSplit()
|
||||
proc = JinaRankingIOProcessor.__new__(JinaRankingIOProcessor)
|
||||
proc.tokenizer = TokenizersBackend(tokenizer_object=backend)
|
||||
proc.model_config = SimpleNamespace(max_model_len=1024, is_encoder_decoder=False)
|
||||
proc.renderer = SimpleNamespace(
|
||||
default_cmpl_tok_params=TokenizeParams(max_total_tokens=1024)
|
||||
)
|
||||
ctx = OfflineScoringInputsContext(
|
||||
pooling_task="token_embed",
|
||||
scoring_data=ScoringData(
|
||||
data_1=["query alpha beta", "query gamma delta"],
|
||||
data_2=["document one two", "document three four"],
|
||||
),
|
||||
pooling_params=PoolingParams(
|
||||
extra_kwargs={
|
||||
"chat_template_kwargs": {"instruction": "Keep this instruction"}
|
||||
}
|
||||
),
|
||||
tokenization_kwargs=None,
|
||||
chat_template=None,
|
||||
lora_request=None,
|
||||
priorities=None,
|
||||
)
|
||||
return proc, ctx
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("query_limit", "doc_limit", "n_queries"),
|
||||
[(None, None, 1), (0, 0, 2), (1, None, 1), (None, 2, 1), (1, 2, 1), (1, 2, 2)],
|
||||
ids=["unset", "zero", "query", "doc", "both-1-to-n", "both-n-to-n"],
|
||||
)
|
||||
def test_offline_truncates_text_before_formatting(
|
||||
offline_processor_and_context, query_limit, doc_limit, n_queries
|
||||
):
|
||||
proc, ctx = offline_processor_and_context
|
||||
queries = ctx.scoring_data.data_1[:n_queries]
|
||||
docs = ctx.scoring_data.data_2
|
||||
ctx.scoring_data = ScoringData(data_1=queries, data_2=docs)
|
||||
ctx.pooling_params.extra_kwargs.update(
|
||||
{
|
||||
name: limit
|
||||
for name, limit in (
|
||||
("max_tokens_per_query", query_limit),
|
||||
("max_tokens_per_doc", doc_limit),
|
||||
)
|
||||
if limit is not None
|
||||
}
|
||||
)
|
||||
|
||||
factory, num_requests = proc.get_request_factory_offline(ctx)
|
||||
requests = list(factory())
|
||||
assert len(requests) == num_requests == n_queries
|
||||
for i, request in enumerate(requests):
|
||||
prompt = request["prompts"]["prompt"]
|
||||
query = " ".join(queries[i].split()[: query_limit or None])
|
||||
assert f"<query>\n{query}<|rerank_token|>\n</query>" in prompt
|
||||
prompt_docs = docs if n_queries == 1 else [docs[i]]
|
||||
for j, doc in enumerate(prompt_docs):
|
||||
doc = " ".join(doc.split()[: doc_limit or None])
|
||||
assert f'<passage id="{j}">\n{doc}<|embed_token|>\n</passage>' in prompt
|
||||
assert prompt.count("<|embed_token|>") == len(prompt_docs)
|
||||
assert prompt.count("<|rerank_token|>") == 1
|
||||
assert "<instruct>\nKeep this instruction\n</instruct>" in prompt
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", ["max_tokens_per_query", "max_tokens_per_doc"])
|
||||
@pytest.mark.parametrize("limit", [-1, 1024])
|
||||
def test_offline_rejects_invalid_token_limits(
|
||||
offline_processor_and_context, name, limit
|
||||
):
|
||||
proc, ctx = offline_processor_and_context
|
||||
ctx.pooling_params.extra_kwargs[name] = limit
|
||||
with pytest.raises(ValueError, match=name):
|
||||
proc.get_request_factory_offline(ctx)
|
||||
@@ -0,0 +1,84 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
from tokenizers import Tokenizer, models, pre_tokenizers
|
||||
from transformers import TokenizersBackend
|
||||
|
||||
from vllm.entrypoints.pooling.scoring.typing import ScoreInput
|
||||
from vllm.entrypoints.pooling.scoring.utils import (
|
||||
truncate_text_to_tokens,
|
||||
validate_score_input,
|
||||
)
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("data_1", "data_2", "is_multimodal_model", "architecture", "message"),
|
||||
[
|
||||
pytest.param(
|
||||
{"content": []},
|
||||
"document",
|
||||
False,
|
||||
"TestModel",
|
||||
"MultiModalParam is not supported for TestModel",
|
||||
id="unsupported-multimodal-input",
|
||||
),
|
||||
pytest.param(
|
||||
["query 1", "query 2"],
|
||||
["document"],
|
||||
False,
|
||||
"TestModel",
|
||||
"Input lengths must be either 1:1, 1:N or N:N",
|
||||
id="incompatible-input-lengths",
|
||||
),
|
||||
pytest.param(
|
||||
[],
|
||||
["document"],
|
||||
False,
|
||||
"TestModel",
|
||||
"At least one text element must be given",
|
||||
id="empty-first-input",
|
||||
),
|
||||
pytest.param(
|
||||
["query"],
|
||||
[],
|
||||
False,
|
||||
"TestModel",
|
||||
"At least one text_pair element must be given",
|
||||
id="empty-second-input",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_validate_score_input_rejects_invalid_inputs(
|
||||
data_1: ScoreInput | list[ScoreInput],
|
||||
data_2: ScoreInput | list[ScoreInput],
|
||||
is_multimodal_model: bool,
|
||||
architecture: str,
|
||||
message: str,
|
||||
):
|
||||
with pytest.raises(VLLMValidationError) as exc_info:
|
||||
validate_score_input(
|
||||
data_1,
|
||||
data_2,
|
||||
is_multimodal_model=is_multimodal_model,
|
||||
architecture=architecture,
|
||||
)
|
||||
|
||||
assert str(exc_info.value) == message
|
||||
assert exc_info.value.parameter is None
|
||||
assert exc_info.value.value is None
|
||||
|
||||
|
||||
def test_truncate_text_to_tokens_handles_shared_character_offsets():
|
||||
vocab = {
|
||||
token: i for i, token in enumerate(sorted(pre_tokenizers.ByteLevel.alphabet()))
|
||||
}
|
||||
backend = Tokenizer(models.BPE(vocab=vocab, merges=[]))
|
||||
backend.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)
|
||||
tokenizer = TokenizersBackend(tokenizer_object=backend)
|
||||
|
||||
truncated = truncate_text_to_tokens("aéx", tokenizer, max_tokens=2)
|
||||
|
||||
assert truncated == "a"
|
||||
assert len(tokenizer(truncated, add_special_tokens=False)["input_ids"]) == 1
|
||||
Reference in New Issue
Block a user