[Model] Add num_cached_tokens for PoolingRequestOutput (#27378)
Signed-off-by: wang.yuqi <noooop@126.com>
This commit is contained in:
@@ -19,14 +19,25 @@ def test_classify_models(
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
example_prompts = example_prompts * 2
|
||||
# example_prompts is too short for testing prefix_caching
|
||||
example_prompts = [s * 10 for s in example_prompts]
|
||||
|
||||
with vllm_runner(
|
||||
model, max_model_len=512, dtype=dtype, enable_prefix_caching=True
|
||||
) as vllm_model:
|
||||
cache_config = vllm_model.llm.llm_engine.cache_config
|
||||
assert cache_config.enable_prefix_caching
|
||||
vllm_outputs = vllm_model.classify(example_prompts)
|
||||
|
||||
# First Run
|
||||
vllm_model.classify(example_prompts)
|
||||
|
||||
# assert prefix_caching works
|
||||
pooling_outputs = vllm_model.llm.encode(
|
||||
example_prompts, pooling_task="classify"
|
||||
)
|
||||
for output in pooling_outputs:
|
||||
assert output.num_cached_tokens > 0
|
||||
vllm_outputs = [req_output.outputs.data for req_output in pooling_outputs]
|
||||
|
||||
with hf_runner(
|
||||
model, dtype=dtype, auto_cls=AutoModelForSequenceClassification
|
||||
@@ -54,7 +65,8 @@ def test_embed_models(
|
||||
model: str,
|
||||
dtype: str,
|
||||
):
|
||||
example_prompts = [str(s).strip() for s in example_prompts] * 2
|
||||
# example_prompts is too short for testing prefix_caching
|
||||
example_prompts = [str(s).strip() * 10 for s in example_prompts]
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
@@ -64,7 +76,15 @@ def test_embed_models(
|
||||
) as vllm_model:
|
||||
cache_config = vllm_model.llm.llm_engine.cache_config
|
||||
assert cache_config.enable_prefix_caching
|
||||
vllm_outputs = vllm_model.embed(example_prompts)
|
||||
|
||||
# First Run
|
||||
vllm_model.embed(example_prompts)
|
||||
|
||||
# assert prefix_caching works
|
||||
pooling_outputs = vllm_model.llm.encode(example_prompts, pooling_task="embed")
|
||||
for output in pooling_outputs:
|
||||
assert output.num_cached_tokens > 0
|
||||
vllm_outputs = [req_output.outputs.data for req_output in pooling_outputs]
|
||||
|
||||
with hf_runner(
|
||||
model,
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import TokensPrompt
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
["Qwen/Qwen3-0.6B"],
|
||||
)
|
||||
@torch.inference_mode
|
||||
def test_embed_models(hf_runner, vllm_runner, model: str):
|
||||
n_prompt_tokens = [55, 56, 57]
|
||||
token_prompts = [[1024 + i for i in range(n)] for n in n_prompt_tokens]
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=128,
|
||||
enforce_eager=True,
|
||||
runner="pooling",
|
||||
enable_chunked_prefill=False,
|
||||
enable_prefix_caching=False,
|
||||
) as vllm_model:
|
||||
pooling_outputs = vllm_model.llm.encode(
|
||||
[TokensPrompt(prompt_token_ids=t) for t in token_prompts],
|
||||
pooling_task="token_embed",
|
||||
)
|
||||
|
||||
for n, output in zip(n_prompt_tokens, pooling_outputs):
|
||||
assert len(output.prompt_token_ids) == n
|
||||
assert output.num_cached_tokens == 0
|
||||
Reference in New Issue
Block a user