diff --git a/tests/entrypoints/pooling/scoring/test_io_processor_unit.py b/tests/entrypoints/pooling/scoring/test_io_processor_unit.py index b8511ea2c9..cd4d58fea6 100644 --- a/tests/entrypoints/pooling/scoring/test_io_processor_unit.py +++ b/tests/entrypoints/pooling/scoring/test_io_processor_unit.py @@ -31,6 +31,7 @@ class _DummyTokenizer: # Outside the range of the prompt ids below, so a test can tell a pad # token apart from a real one. pad_token_id: int = 99999 + pad_token_type_id: int = 0 @pytest.fixture @@ -83,10 +84,11 @@ def test_llm_reranker_tokenization_is_independent_of_nonbinding_doc_limit( def test_token_type_ids_stay_aligned_with_a_truncated_padded_prompt(): """The cross-encoder segment boundary must survive truncate + pad. - `token_type_ids` are parallel to `prompt_token_ids` and are reduced to a - single boundary index by `compress_token_type_ids`. If the two arrays are - truncated and padded in different orders they no longer describe the same - positions, and the model is told the query segment is empty. + `token_type_ids` are parallel to `prompt_token_ids` and are reduced to the + document segment's start and end by `compress_token_type_ids`. If the two + arrays are truncated and padded in different orders they no longer + describe the same positions, and the model is told the query segment is + empty. """ tokenizer = _DummyTokenizer() num_query, num_doc = 20, 30 @@ -114,10 +116,34 @@ def test_token_type_ids_stay_aligned_with_a_truncated_padded_prompt(): # Keeping the last 40 tokens drops the first 10 query tokens, so 10 query # tokens survive and the document starts at index 10. - first_doc = compress_token_type_ids(token_type_ids) - assert first_doc == 10 - assert prompt_token_ids[:first_doc] == list(range(10, num_query)) - assert prompt_token_ids[first_doc:40] == list(range(num_query, 50)) + doc_start, doc_end = compress_token_type_ids(token_type_ids) + assert (doc_start, doc_end) == (10, 40) + assert prompt_token_ids[:doc_start] == list(range(10, num_query)) + assert prompt_token_ids[doc_start:doc_end] == list(range(num_query, 50)) + assert token_type_ids[doc_end:] == [tokenizer.pad_token_type_id] * 60 + + +def test_token_type_padding_uses_the_tokenizer_pad_type(): + tokenizer = _DummyTokenizer() + tok_params = TokenizeParams( + max_total_tokens=8, + pad_prompt_tokens=8, + truncate_prompt_tokens=None, + truncation_side="right", + ) + + token_type_ids = _apply_post_tokenization_to_token_type_ids( + tokenizer, tok_params, [0, 0, 0, 1, 1] + ) + + assert token_type_ids == [0, 0, 0, 1, 1, 0, 0, 0] + assert compress_token_type_ids(token_type_ids) == (3, 5) + + +@pytest.mark.parametrize("token_type_ids", [[0, 1, 0, 1], [0, 2]]) +def test_compress_token_type_ids_rejects_invalid_sequences(token_type_ids): + with pytest.raises(ValueError, match="optional trailing zeros"): + compress_token_type_ids(token_type_ids) def _pooling_output( diff --git a/vllm/entrypoints/pooling/scoring/io_processor.py b/vllm/entrypoints/pooling/scoring/io_processor.py index 9ab36a1395..f6678a7bbd 100644 --- a/vllm/entrypoints/pooling/scoring/io_processor.py +++ b/vllm/entrypoints/pooling/scoring/io_processor.py @@ -84,7 +84,7 @@ def _apply_post_tokenization_to_token_type_ids( pad_length = tok_params.max_input_tokens if pad_length is not None and pad_length > len(token_type_ids): - pad_token_type_id = token_type_ids[-1] if token_type_ids else 0 + pad_token_type_id = getattr(tokenizer, "pad_token_type_id", 0) or 0 token_type_ids = token_type_ids + [pad_token_type_id] * ( pad_length - len(token_type_ids) ) diff --git a/vllm/entrypoints/pooling/scoring/utils.py b/vllm/entrypoints/pooling/scoring/utils.py index b3617ae9c6..dd879778ae 100644 --- a/vllm/entrypoints/pooling/scoring/utils.py +++ b/vllm/entrypoints/pooling/scoring/utils.py @@ -262,21 +262,30 @@ def parse_score_data( return prompt_1, prompt_2, mm_items, mm_uuids -def compress_token_type_ids(token_type_ids: list[int]) -> int: - """Return position of the first 1 or the length of the list - if not found. +def compress_token_type_ids(token_type_ids: list[int]) -> tuple[int, int]: + """Return the half-open range containing token type 1. + + Right padding uses the tokenizer's padding token type, which is commonly + zero. Therefore, a padded query-document pair can contain zeros followed + by ones followed by trailing zeros. """ first_one = len(token_type_ids) + first_zero_after_one = len(token_type_ids) err_msg = ( "Token type ids are expected to be a sequence" - " of zeros followed by a sequence of ones" + " of zeros followed by ones and optional trailing zeros" ) for i, type_id in enumerate(token_type_ids): - if type_id == 0 and first_one < i: - raise ValueError(err_msg) - elif type_id == 1 and first_one > i: - first_one = i - elif type_id > 1: + if type_id not in (0, 1): raise ValueError(err_msg) + if type_id == 1: + if first_zero_after_one < len(token_type_ids): + raise ValueError(err_msg) + if first_one == len(token_type_ids): + first_one = i + elif first_one < len(token_type_ids) and first_zero_after_one == len( + token_type_ids + ): + first_zero_after_one = i - return first_one + return first_one, first_zero_after_one diff --git a/vllm/v1/worker/gpu/model_states/encoder_only.py b/vllm/v1/worker/gpu/model_states/encoder_only.py index 1b23723fdc..82d9e63ed8 100644 --- a/vllm/v1/worker/gpu/model_states/encoder_only.py +++ b/vllm/v1/worker/gpu/model_states/encoder_only.py @@ -105,12 +105,15 @@ class EncoderOnlyModelState(DefaultModelState): if pooling_params is None or pooling_params.extra_kwargs is None: return - token_type_start = pooling_params.extra_kwargs.get("compressed_token_type_ids") - if token_type_start is not None: + token_type_range = pooling_params.extra_kwargs.get("compressed_token_type_ids") + if token_type_range is not None: assert new_req_data.prompt_token_ids is not None + token_type_start, token_type_end = token_type_range + positions = torch.arange( + len(new_req_data.prompt_token_ids), dtype=torch.int32 + ) self.token_type_ids[new_req_data.req_id] = ( - torch.arange(len(new_req_data.prompt_token_ids), dtype=torch.int32) - >= token_type_start + (positions >= token_type_start) & (positions < token_type_end) ).to(torch.int32) def remove_request(self, req_id: str) -> None: diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 4bb39c80cb..229acdfcd2 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -1079,8 +1079,9 @@ class GPUModelRunner( for i in range(num_reqs): seq_len_i = seq_lens_cpu[i] - pos = token_type_id_requests.get(i, seq_len_i) - ids = (torch.arange(seq_len_i) >= pos).int() + start, end = token_type_id_requests.get(i, (seq_len_i, seq_len_i)) + positions = torch.arange(seq_len_i) + ids = ((positions >= start) & (positions < end)).int() token_type_ids.append(ids) token_type_ids_cpu = torch.empty(