[Feature][Pooling] Add offline fixed-length late chunking (#60164)
Signed-off-by: Linze-Shi <linzeshi0@gmail.com>
This commit is contained in:
@@ -0,0 +1,314 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.config import PoolerConfig
|
||||
from vllm.entrypoints.pooling.embed.io_processor import (
|
||||
JinaRankingTokenEmbedIOProcessor,
|
||||
TokenEmbedIOProcessor,
|
||||
)
|
||||
from vllm.entrypoints.pooling.late_chunking import build_late_chunking_metadata
|
||||
from vllm.entrypoints.pooling.offline import PoolingOfflineMixin
|
||||
from vllm.entrypoints.pooling.typing import (
|
||||
OfflineEncodeInputsContext,
|
||||
OfflineOutputsContext,
|
||||
)
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.outputs import (
|
||||
LateChunk,
|
||||
PoolingOutput,
|
||||
PoolingRequestOutput,
|
||||
RequestError,
|
||||
)
|
||||
from vllm.pooling_params import LateChunkingParams
|
||||
from vllm.renderers import TokenizeParams
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def late_chunk_processor():
|
||||
processor = object.__new__(TokenEmbedIOProcessor)
|
||||
processor.model_config = SimpleNamespace(
|
||||
is_encoder_decoder=False,
|
||||
architecture="NomicBertModel",
|
||||
model_impl="auto",
|
||||
is_matryoshka=False,
|
||||
hf_config=SimpleNamespace(),
|
||||
pooler_config=PoolerConfig(seq_pooling_type="MEAN", tok_pooling_type="ALL"),
|
||||
)
|
||||
processor.vllm_config = SimpleNamespace(
|
||||
model_config=processor.model_config,
|
||||
cache_config=SimpleNamespace(enable_prefix_caching=False),
|
||||
scheduler_config=SimpleNamespace(enable_chunked_prefill=False),
|
||||
lora_config=None,
|
||||
)
|
||||
processor.renderer = Mock()
|
||||
processor.renderer.default_cmpl_tok_params = TokenizeParams(max_total_tokens=32)
|
||||
processor.renderer.render_cmpl.side_effect = lambda **_: [
|
||||
{
|
||||
"type": "token",
|
||||
"prompt_token_ids": [101, 1, 2, 102],
|
||||
"prompt_token_offsets": [(0, 0), (0, 1), (2, 3), (0, 0)],
|
||||
}
|
||||
]
|
||||
return processor
|
||||
|
||||
|
||||
def _late_chunk_context(prompts="a b", **kwargs):
|
||||
return OfflineEncodeInputsContext(
|
||||
pooling_task="token_embed",
|
||||
tokenization_kwargs=kwargs or None,
|
||||
lora_request=None,
|
||||
priorities=None,
|
||||
prompts=prompts,
|
||||
pooling_params=PoolingParams(
|
||||
late_chunking_params=LateChunkingParams(chunk_size=2)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_late_chunk_render_requests_offsets_once(late_chunk_processor):
|
||||
processor = late_chunk_processor
|
||||
ctx = _late_chunk_context()
|
||||
factory, count = processor.get_request_factory_offline(ctx)
|
||||
render_params = next(factory())
|
||||
result = processor.render(render_params)
|
||||
assert count == 1
|
||||
processor.renderer.render_cmpl.assert_called_once()
|
||||
assert processor.renderer.render_cmpl.call_args.kwargs[
|
||||
"tok_params"
|
||||
].return_token_offsets
|
||||
assert not render_params["tok_params"].return_token_offsets
|
||||
assert "prompt_token_offsets" not in result["prompts"]
|
||||
assert "late_chunking" not in result
|
||||
assert [c.char_range for c in ctx.late_chunking[0].metadata.chunks] == [
|
||||
(0, 1),
|
||||
(2, 3),
|
||||
]
|
||||
assert ctx.late_chunking[0].metadata.input_tokens == 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize("as_sequence", [False, True])
|
||||
@pytest.mark.parametrize("chunk_size", [None, 2])
|
||||
def test_jina_ranking_rejects_chunking_before_tokenization(
|
||||
late_chunk_processor, as_sequence, chunk_size
|
||||
):
|
||||
processor = object.__new__(JinaRankingTokenEmbedIOProcessor)
|
||||
processor.__dict__.update(late_chunk_processor.__dict__)
|
||||
ctx = _late_chunk_context(prompts=["document", "query"])
|
||||
params = PoolingParams(
|
||||
late_chunking_params=LateChunkingParams(chunk_size) if chunk_size else None
|
||||
)
|
||||
ctx.pooling_params = [params] if as_sequence else params
|
||||
factory, count = processor.get_request_factory_offline(ctx)
|
||||
assert count == 1
|
||||
request = next(factory())
|
||||
if chunk_size is not None:
|
||||
with pytest.raises(VLLMValidationError, match="JinaForRanking.*late chunking"):
|
||||
processor.render(request)
|
||||
late_chunk_processor.renderer.render_cmpl.assert_not_called()
|
||||
else:
|
||||
result = processor.render(request)
|
||||
assert result["prompts"]["type"] == "token"
|
||||
assert result["prompts"]["prompt_token_ids"] == [101, 1, 2, 102]
|
||||
late_chunk_processor.renderer.render_cmpl.assert_called_once()
|
||||
assert not request["tok_params"].return_token_offsets
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"prompts, kwargs, error",
|
||||
[
|
||||
pytest.param("", {}, "plain-text", id="empty"),
|
||||
pytest.param([1, 2], {}, "plain-text", id="token-ids"),
|
||||
pytest.param(
|
||||
{"prompt": "abc", "multi_modal_data": {}},
|
||||
{},
|
||||
"plain-text",
|
||||
id="multimodal",
|
||||
),
|
||||
]
|
||||
+ [
|
||||
pytest.param("abc", {key: value}, "does not support", id=key)
|
||||
for key, value in (
|
||||
("truncate_prompt_tokens", 2),
|
||||
("truncation", True),
|
||||
("pad_prompt_tokens", 8),
|
||||
("padding", True),
|
||||
("do_lower_case", True),
|
||||
)
|
||||
],
|
||||
)
|
||||
def test_late_chunking_rejects_unsupported_input_before_render(
|
||||
late_chunk_processor, prompts, kwargs, error
|
||||
):
|
||||
processor = late_chunk_processor
|
||||
with pytest.raises(VLLMValidationError, match=error):
|
||||
factory, _ = processor.get_request_factory_offline(
|
||||
_late_chunk_context(prompts, **kwargs)
|
||||
)
|
||||
processor.render(next(factory()))
|
||||
processor.renderer.render_cmpl.assert_not_called()
|
||||
|
||||
|
||||
def test_late_chunk_ranges_keep_unicode_overlaps_and_special_only_chunks():
|
||||
text = "中 😀 e\u0301"
|
||||
offsets = [(0, 0), (0, 1), (2, 3), (2, 3), (4, 6), (0, 0)]
|
||||
metadata = build_late_chunking_metadata(text, len(offsets), offsets, 1)
|
||||
assert [c.char_range for c in metadata.chunks] == [
|
||||
None,
|
||||
(0, 1),
|
||||
(2, 3),
|
||||
(2, 3),
|
||||
(4, 6),
|
||||
None,
|
||||
]
|
||||
assert [
|
||||
text[slice(*c.char_range)] if c.char_range else None for c in metadata.chunks
|
||||
] == [None, "中", "😀", "😀", "e\u0301", None]
|
||||
chunks = build_late_chunking_metadata(text, len(offsets), offsets, 4).chunks
|
||||
assert [(c.token_range, c.char_range) for c in chunks] == [
|
||||
((0, 4), (0, 3)),
|
||||
((4, 6), (4, 6)),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"offsets", [None, [(0, 1)], [(-1, 1), (0, 0)], [(0, 4), (0, 0)], [(2, 3), (0, 1)]]
|
||||
)
|
||||
def test_late_chunk_ranges_reject_missing_or_invalid_offsets(offsets):
|
||||
with pytest.raises(VLLMValidationError, match="offsets"):
|
||||
build_late_chunking_metadata("abc", 2, offsets, 2)
|
||||
|
||||
|
||||
def _mock_chunk_llm(processor, outputs):
|
||||
llm = mock.Mock(spec=PoolingOfflineMixin)
|
||||
llm.encode = PoolingOfflineMixin.encode.__get__(llm)
|
||||
llm._run_tiling_engine = PoolingOfflineMixin._run_tiling_engine.__get__(llm)
|
||||
llm.pooling_io_processors = {"token_embed": processor}
|
||||
llm._executor = SimpleNamespace(map=map)
|
||||
llm.llm_engine = mock.Mock()
|
||||
llm.llm_engine.vllm_config.scheduler_config.max_num_seqs = 2
|
||||
llm.llm_engine.has_unfinished_requests.return_value = False
|
||||
llm.llm_engine.step.side_effect = outputs
|
||||
llm._render_and_add_requests = mock.Mock(
|
||||
side_effect=lambda **kw: [
|
||||
str(i) + "-internal" for i in range(len(kw["params"]))
|
||||
]
|
||||
)
|
||||
return llm
|
||||
|
||||
|
||||
def _chunk_output(request_id, rows=2, **kwargs):
|
||||
return PoolingRequestOutput(
|
||||
str(request_id),
|
||||
PoolingOutput(torch.ones(rows, 4)),
|
||||
[101, 1, 2, 102],
|
||||
0,
|
||||
True,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def test_late_chunk_mapping_preserves_order_mixed_requests_and_errors(
|
||||
late_chunk_processor,
|
||||
):
|
||||
error = RequestError("test_error", "original error")
|
||||
failed = _chunk_output(3, rows=0, error=error)
|
||||
llm = _mock_chunk_llm(late_chunk_processor, RuntimeError("step failed"))
|
||||
params = [
|
||||
PoolingParams(late_chunking_params=LateChunkingParams(2)),
|
||||
PoolingParams(),
|
||||
PoolingParams(late_chunking_params=LateChunkingParams(1)),
|
||||
PoolingParams(late_chunking_params=LateChunkingParams(2)),
|
||||
]
|
||||
# A failed encode call must not leave metadata for the following batch.
|
||||
with pytest.raises(RuntimeError, match="step failed"):
|
||||
llm.encode(
|
||||
["a b"] * 2,
|
||||
pooling_task="token_embed",
|
||||
pooling_params=params[:2],
|
||||
use_tqdm=False,
|
||||
)
|
||||
llm.llm_engine.step.side_effect = [
|
||||
[
|
||||
_chunk_output(2, rows=4),
|
||||
failed,
|
||||
_chunk_output(1, rows=4),
|
||||
_chunk_output(0),
|
||||
]
|
||||
]
|
||||
outputs = llm.encode(
|
||||
["a b"] * 4, pooling_task="token_embed", pooling_params=params, use_tqdm=False
|
||||
)
|
||||
assert outputs[0].late_chunking.chunks == [
|
||||
LateChunk((0, 2), (0, 1)),
|
||||
LateChunk((2, 4), (2, 3)),
|
||||
]
|
||||
assert outputs[1].late_chunking is None
|
||||
assert len(outputs[2].late_chunking.chunks) == 4
|
||||
assert outputs[3].late_chunking is None
|
||||
assert outputs[3].error is error
|
||||
|
||||
|
||||
def test_late_chunk_mapping_rejects_successful_output_with_wrong_row_count(
|
||||
late_chunk_processor,
|
||||
):
|
||||
llm = _mock_chunk_llm(late_chunk_processor, [[_chunk_output(0, rows=0)]])
|
||||
with pytest.raises(ValueError, match="does not match"):
|
||||
llm.encode(
|
||||
"a b",
|
||||
pooling_task="token_embed",
|
||||
pooling_params=PoolingParams(late_chunking_params=LateChunkingParams(2)),
|
||||
use_tqdm=False,
|
||||
)
|
||||
# All engine requests have finished before post-processing validates the output.
|
||||
llm.llm_engine.abort_request.assert_not_called()
|
||||
|
||||
|
||||
def test_late_chunk_contexts_remain_isolated_when_rendering_interleaves(
|
||||
late_chunk_processor,
|
||||
):
|
||||
processor = late_chunk_processor
|
||||
first, second = _late_chunk_context("xy z"), _late_chunk_context()
|
||||
shared = PoolingParams(late_chunking_params=LateChunkingParams(2))
|
||||
first.pooling_params = second.pooling_params = shared
|
||||
first_factory, _ = processor.get_request_factory_offline(first)
|
||||
second_factory, _ = processor.get_request_factory_offline(second)
|
||||
first_request, second_request = next(first_factory()), next(second_factory())
|
||||
processor.render(second_request)
|
||||
assert first.late_chunking[0].metadata is None
|
||||
processor.renderer.render_cmpl.side_effect = lambda **_: [
|
||||
{
|
||||
"type": "token",
|
||||
"prompt_token_ids": [101, 1, 2, 102],
|
||||
"prompt_token_offsets": [(0, 0), (0, 2), (3, 4), (0, 0)],
|
||||
}
|
||||
]
|
||||
processor.render(first_request)
|
||||
first_output = processor.post_process_offline(
|
||||
OfflineOutputsContext([_chunk_output(0)], late_chunking=first.late_chunking)
|
||||
)[0]
|
||||
second_output = processor.post_process_offline(
|
||||
OfflineOutputsContext([_chunk_output(0)], late_chunking=second.late_chunking)
|
||||
)[0]
|
||||
assert first_output.late_chunking.chunks[0].char_range == (0, 2)
|
||||
assert second_output.late_chunking.chunks[0].char_range == (0, 1)
|
||||
|
||||
|
||||
def test_plain_token_embed_skips_chunk_metadata(late_chunk_processor):
|
||||
ctx = _late_chunk_context()
|
||||
ctx.pooling_params = PoolingParams()
|
||||
factory, _ = late_chunk_processor.get_request_factory_offline(ctx)
|
||||
request = next(factory())
|
||||
result = late_chunk_processor.render(request)
|
||||
assert not ctx.late_chunking
|
||||
assert result["params"].late_chunking_params is None
|
||||
assert not late_chunk_processor.renderer.render_cmpl.call_args.kwargs[
|
||||
"tok_params"
|
||||
].return_token_offsets
|
||||
@@ -3,9 +3,13 @@
|
||||
import weakref
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import LLM, PoolingRequestOutput
|
||||
from tests.models.utils import check_embeddings_close
|
||||
from vllm import LLM, PoolingParams, PoolingRequestOutput
|
||||
from vllm.config import PoolerConfig
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.pooling_params import LateChunkingParams
|
||||
from vllm.tasks import PoolingTask
|
||||
|
||||
MODEL_NAME = "intfloat/multilingual-e5-small"
|
||||
@@ -61,3 +65,218 @@ def test_unsupported_tasks(llm: LLM, task: PoolingTask, caplog_vllm):
|
||||
|
||||
with pytest.raises(ValueError, match=err_msg):
|
||||
llm.encode(prompt, pooling_task=task, use_tqdm=False)
|
||||
|
||||
|
||||
def test_bge_m3_late_chunking_keeps_special_tokens(vllm_runner):
|
||||
with vllm_runner(
|
||||
"BAAI/bge-m3",
|
||||
hf_overrides={"architectures": ["BgeM3EmbeddingModel"]},
|
||||
pooler_config=PoolerConfig(task="token_embed"),
|
||||
dtype="float32",
|
||||
max_model_len=128,
|
||||
# Keep encoder batch size fixed for the numerical reference.
|
||||
max_num_seqs=1,
|
||||
gpu_memory_utilization=0.5,
|
||||
enforce_eager=True,
|
||||
) as runner:
|
||||
llm = runner.llm
|
||||
text = "A document about Berlin and its museums."
|
||||
sizes = (1, 3, 128, None)
|
||||
tokens, *outputs = llm.encode(
|
||||
[text] * len(sizes),
|
||||
pooling_task="token_embed",
|
||||
pooling_params=[
|
||||
PoolingParams(
|
||||
use_activation=size not in (1, None),
|
||||
late_chunking_params=LateChunkingParams(size) if size else None,
|
||||
)
|
||||
for size in sizes
|
||||
],
|
||||
use_tqdm=False,
|
||||
)
|
||||
assert tokens.outputs.data.shape[0] == len(tokens.prompt_token_ids)
|
||||
assert tokens.late_chunking.chunks[0].char_range is None
|
||||
for size, output in zip(sizes[1:], outputs):
|
||||
if size is None:
|
||||
# Ordinary BGE-M3 token embeddings still drop BOS only.
|
||||
expected = tokens.outputs.data[1:]
|
||||
assert output.late_chunking is None
|
||||
else:
|
||||
# BGE-M3's projector is linear, so its unnormalized token
|
||||
# outputs also provide a reference for projected chunk means.
|
||||
expected = torch.nn.functional.normalize(
|
||||
torch.stack([p.mean(0) for p in tokens.outputs.data.split(size)]),
|
||||
dim=-1,
|
||||
)
|
||||
assert len(output.late_chunking.chunks) == len(expected)
|
||||
torch.testing.assert_close(
|
||||
output.outputs.data, expected, atol=2e-5, rtol=2e-4
|
||||
)
|
||||
|
||||
|
||||
def test_nomic_late_chunking_without_special_tokens(vllm_runner):
|
||||
"""Keep source ranges without special tokens and reject empty tokenization."""
|
||||
with vllm_runner(
|
||||
"nomic-ai/nomic-embed-text-v1",
|
||||
revision="720244025c1a7e15661a174c63cce63c8218e52b",
|
||||
trust_remote_code=True,
|
||||
runner="pooling",
|
||||
pooler_config=PoolerConfig(task="token_embed"),
|
||||
dtype="float32",
|
||||
max_model_len=128,
|
||||
max_num_seqs=2,
|
||||
gpu_memory_utilization=0.3,
|
||||
enforce_eager=True,
|
||||
enable_prefix_caching=False,
|
||||
enable_chunked_prefill=False,
|
||||
) as runner:
|
||||
llm = runner.llm
|
||||
text = "search_document: 中文 😀 café e\u0301"
|
||||
params = PoolingParams(
|
||||
late_chunking_params=LateChunkingParams(3), use_activation=False
|
||||
)
|
||||
kwargs = {"add_special_tokens": False}
|
||||
raw = llm.encode(
|
||||
text,
|
||||
pooling_task="token_embed",
|
||||
use_tqdm=False,
|
||||
pooling_params=PoolingParams(use_activation=False),
|
||||
tokenization_kwargs=kwargs,
|
||||
)[0]
|
||||
actual = llm.encode(
|
||||
text,
|
||||
pooling_task="token_embed",
|
||||
use_tqdm=False,
|
||||
pooling_params=params,
|
||||
tokenization_kwargs=kwargs,
|
||||
)[0]
|
||||
expected = torch.stack([part.mean(0) for part in raw.outputs.data.split(3)])
|
||||
torch.testing.assert_close(actual.outputs.data, expected, atol=1e-5, rtol=1e-4)
|
||||
encoded = llm.get_tokenizer()(text, return_offsets_mapping=True, **kwargs)
|
||||
assert actual.prompt_token_ids == encoded["input_ids"]
|
||||
for chunk in actual.late_chunking.chunks:
|
||||
start, end = chunk.token_range
|
||||
offsets = [(a, b) for a, b in encoded["offset_mapping"][start:end] if a < b]
|
||||
assert chunk.char_range == (offsets[0][0], offsets[-1][1])
|
||||
for rejected in ("long " * 256, " "):
|
||||
with pytest.raises(VLLMValidationError):
|
||||
llm.encode(
|
||||
rejected,
|
||||
pooling_task="token_embed",
|
||||
use_tqdm=False,
|
||||
pooling_params=params,
|
||||
tokenization_kwargs=kwargs,
|
||||
)
|
||||
assert (
|
||||
llm.encode(text, pooling_task="token_embed", use_tqdm=False)[
|
||||
0
|
||||
].late_chunking
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
@pytest.mark.parametrize("normalize", [False, True])
|
||||
def test_e5_late_chunking_mixed_requests(llm: LLM, normalize: bool):
|
||||
texts = [prompt, "Short text.", "中文 😀 café"]
|
||||
raw = llm.encode(
|
||||
texts,
|
||||
pooling_task="token_embed",
|
||||
use_tqdm=False,
|
||||
pooling_params=PoolingParams(use_activation=False),
|
||||
)
|
||||
sizes = [3, None, 100]
|
||||
params = [
|
||||
PoolingParams(
|
||||
late_chunking_params=LateChunkingParams(size) if size is not None else None,
|
||||
use_activation=normalize,
|
||||
)
|
||||
for size in sizes
|
||||
]
|
||||
actual = llm.encode(
|
||||
texts, pooling_task="token_embed", pooling_params=params, use_tqdm=False
|
||||
)
|
||||
for original, result, size in zip(raw, actual, sizes):
|
||||
expected = original.outputs.data
|
||||
if size is not None:
|
||||
expected = torch.stack(
|
||||
[part.float().mean(0) for part in expected.split(size)]
|
||||
)
|
||||
assert result.late_chunking is not None
|
||||
assert len(result.late_chunking.chunks) == len(expected)
|
||||
assert result.late_chunking.input_tokens == len(result.prompt_token_ids)
|
||||
else:
|
||||
assert result.late_chunking is None
|
||||
if normalize:
|
||||
expected = torch.nn.functional.normalize(expected, dim=-1)
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=result.outputs.data.tolist(),
|
||||
embeddings_1_lst=expected.tolist(),
|
||||
name_0="late_chunking",
|
||||
name_1="token_reference",
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
result.outputs.data.norm(dim=-1),
|
||||
expected.norm(dim=-1),
|
||||
atol=1e-3,
|
||||
rtol=1e-3,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_v2_runner", [False, True])
|
||||
def test_late_chunking_recomputes_cached_prefix_with_chunked_prefill(
|
||||
vllm_runner, monkeypatch, use_v2_runner
|
||||
):
|
||||
# Dummy weights suffice for testing scheduling and reduction of the same states.
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if use_v2_runner else "0")
|
||||
with vllm_runner(
|
||||
"openai-community/gpt2",
|
||||
runner="pooling",
|
||||
convert="embed",
|
||||
load_format="dummy",
|
||||
pooler_config=PoolerConfig(
|
||||
task="token_embed", seq_pooling_type="LAST", tok_pooling_type="ALL"
|
||||
),
|
||||
dtype="float32",
|
||||
max_model_len=128,
|
||||
max_num_batched_tokens=16,
|
||||
max_num_seqs=2,
|
||||
enable_prefix_caching=True,
|
||||
enable_chunked_prefill=True,
|
||||
enforce_eager=True,
|
||||
gpu_memory_utilization=0.3,
|
||||
seed=0,
|
||||
) as runner:
|
||||
text = "The document has several important words. " * 6
|
||||
llm = runner.llm
|
||||
raw = llm.encode(
|
||||
text,
|
||||
pooling_task="token_embed",
|
||||
use_tqdm=False,
|
||||
pooling_params=PoolingParams(use_activation=False),
|
||||
)[0]
|
||||
assert len(raw.prompt_token_ids) > 16
|
||||
cached = llm.encode(
|
||||
text,
|
||||
pooling_task="token_embed",
|
||||
use_tqdm=False,
|
||||
pooling_params=PoolingParams(
|
||||
use_activation=False, skip_reading_prefix_cache=False
|
||||
),
|
||||
)[0]
|
||||
assert cached.num_cached_tokens > 0
|
||||
params = PoolingParams(
|
||||
late_chunking_params=LateChunkingParams(7),
|
||||
use_activation=False,
|
||||
skip_reading_prefix_cache=False,
|
||||
)
|
||||
actual = llm.encode(
|
||||
text, pooling_task="token_embed", pooling_params=params, use_tqdm=False
|
||||
)[0]
|
||||
assert actual.num_cached_tokens == 0
|
||||
assert actual.prompt_token_ids == raw.prompt_token_ids
|
||||
expected = torch.stack([part.mean(0) for part in raw.outputs.data.split(7)])
|
||||
torch.testing.assert_close(actual.outputs.data, expected, atol=1e-5, rtol=1e-4)
|
||||
assert actual.late_chunking.input_tokens == len(raw.prompt_token_ids)
|
||||
assert params.skip_reading_prefix_cache is False
|
||||
assert params.late_chunking_params is not None
|
||||
|
||||
Reference in New Issue
Block a user