[Feature][Pooling] Add offline fixed-length late chunking (#60164)

Signed-off-by: Linze-Shi <linzeshi0@gmail.com>
This commit is contained in:
Linze Shi
2026-10-13 11:16:21 +00:00
committed by GitHub
parent 426f3bd36b
commit e6e84f350c
15 changed files with 1147 additions and 7 deletions
+92
View File
@@ -1,8 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from types import SimpleNamespace
from typing import Any
import msgspec
import pytest
from pydantic import TypeAdapter, ValidationError
@@ -13,6 +15,7 @@ from vllm.entrypoints.pooling.classify.protocol import ClassificationRequest
from vllm.entrypoints.pooling.embed.protocol import EmbeddingRequest
from vllm.entrypoints.pooling.pooling.protocol import PoolingRequest
from vllm.exceptions import VLLMValidationError
from vllm.pooling_params import LateChunkingParams
EMBEDDING_MODELS = [
EmbedModelInfo("intfloat/multilingual-e5-small", is_matryoshka=False),
@@ -211,3 +214,92 @@ def test_token_classify(pooling_type: str):
with pytest.raises(VLLMValidationError):
pooling_params = PoolingParams(task=task, **{p: True})
pooling_params.verify(model_config)
@pytest.mark.parametrize("value", [0, -1, True, False, 1.5, "2"])
def test_late_chunk_size_rejects_non_positive_integers(value):
with pytest.raises(VLLMValidationError, match="positive integer"):
PoolingParams(late_chunking_params=LateChunkingParams(chunk_size=value))
def _late_chunking_model_config():
return SimpleNamespace(
architecture="NomicBertModel",
model_impl="auto",
is_matryoshka=False,
hf_config=SimpleNamespace(),
pooler_config=PoolerConfig(seq_pooling_type="MEAN", tok_pooling_type="ALL"),
)
def test_late_chunking_params_preserve_defaults_clone_and_wire_format():
model_config = _late_chunking_model_config()
params = PoolingParams(
task="token_embed", late_chunking_params=LateChunkingParams(chunk_size=3)
)
params.verify(model_config)
assert params.skip_reading_prefix_cache is True
assert params.use_activation is True
# Chunk means must include cached prefixes even when the user opts into reads.
explicit_cache_flag = PoolingParams(
task="token_embed",
late_chunking_params=LateChunkingParams(chunk_size=3),
skip_reading_prefix_cache=False,
)
explicit_cache_flag.verify(model_config)
assert explicit_cache_flag.skip_reading_prefix_cache is True
clone = params.clone()
assert clone.late_chunking_params is not None
clone.late_chunking_params.chunk_size = 7
assert params.late_chunking_params is not None
assert params.late_chunking_params.chunk_size == 3
# Workers receive only the fixed chunk size, never source ranges.
assert msgspec.msgpack.decode(
msgspec.msgpack.encode(params.late_chunking_params)
) == [3]
assert (
msgspec.msgpack.decode(msgspec.msgpack.encode(params), type=PoolingParams)
== params
)
# A message written before the appended field still uses the old default.
wire = msgspec.msgpack.decode(msgspec.msgpack.encode(PoolingParams()))
assert msgspec.convert(wire[:-1], type=PoolingParams).late_chunking_params is None
@pytest.mark.parametrize(
"task", ["embed", "classify", "token_classify", "plugin", None]
)
def test_late_chunking_rejects_other_tasks(task):
with pytest.raises(VLLMValidationError, match="requires token_embed"):
PoolingParams(
task=task, late_chunking_params=LateChunkingParams(chunk_size=2)
).verify(_late_chunking_model_config())
def test_late_chunking_uses_existing_dimension_validation():
config = MockMatryoshkaModelConfig(
pooler_config=PoolerConfig(tok_pooling_type="ALL", use_activation=False),
)
params = PoolingParams(
task="token_embed", late_chunking_params=LateChunkingParams(2), dimensions=4
)
params.verify(config)
assert params.dimensions == 4
assert params.use_activation is False
params.dimensions = config.embedding_size + 1
with pytest.raises(VLLMValidationError, match="dimensions in range"):
params.verify(config)
@pytest.mark.parametrize("from_config", [False, True])
def test_late_chunking_rejects_token_filtering_after_resolving_defaults(from_config):
config = MockModelConfig(pooler_config=PoolerConfig(tok_pooling_type="STEP"))
params = PoolingParams(
task="token_embed", late_chunking_params=LateChunkingParams(2)
)
if from_config:
config.pooler_config.step_tag_id = 42
else:
params.step_tag_id = 42
with pytest.raises(VLLMValidationError, match="all token rows"):
params.verify(config)