[Bugfix] Treat max_tokens as output upper bound, not input-space reservation (#57035)
Signed-off-by: YuvrajSinghBhadoria2 <245782664+YuvrajSinghBhadoria2@users.noreply.github.com> Signed-off-by: YuvrajSinghBhadoria2 <yuvrajsinghbhado2030@gmail.com> Signed-off-by: Yuvraj Singh Bhadoria <yuvrajsinghbhado2030@gmail.com> Co-authored-by: YuvrajSinghBhadoria2 <245782664+YuvrajSinghBhadoria2@users.noreply.github.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
YuvrajSinghBhadoria2
Cyrus Leung
Claude Opus 4.8
parent
89ac8d971b
commit
1a3a7a8f01
@@ -4,12 +4,15 @@
|
||||
underlying packing helpers, plus Cohere-specific serving helpers."""
|
||||
|
||||
import struct
|
||||
from unittest.mock import Mock
|
||||
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
|
||||
from vllm.config import ModelConfig, PoolerConfig
|
||||
from vllm.entrypoints.pooling.embed.protocol import (
|
||||
EmbeddingCompletionRequest,
|
||||
build_typed_embeddings,
|
||||
)
|
||||
|
||||
@@ -162,3 +165,59 @@ class TestBuildTypedEmbeddingsMultiple:
|
||||
def test_unknown_type_ignored(self, sample_embeddings: list[list[float]]):
|
||||
result = build_typed_embeddings(sample_embeddings, ["float", "unknown_type"])
|
||||
assert result.float is not None
|
||||
|
||||
|
||||
def _embedding_model_config(
|
||||
*,
|
||||
max_model_len: int = 128,
|
||||
max_embed_len: int | None = None,
|
||||
enable_chunked_processing: bool = False,
|
||||
) -> Mock:
|
||||
model_config = Mock(spec=ModelConfig)
|
||||
model_config.max_model_len = max_model_len
|
||||
model_config.encoder_config = None
|
||||
model_config.pooler_config = PoolerConfig(
|
||||
seq_pooling_type="CLS",
|
||||
max_embed_len=max_embed_len,
|
||||
enable_chunked_processing=enable_chunked_processing,
|
||||
)
|
||||
return model_config
|
||||
|
||||
|
||||
class TestMaxEmbedLenCapsInput:
|
||||
"""``build_tok_params`` used to enforce ``max_embed_len`` indirectly, by
|
||||
setting ``max_output_tokens = max_model_len - max_embed_len`` and relying on
|
||||
``max_input_tokens == max_total_tokens - max_output_tokens``. The cap is now
|
||||
expressed directly as ``max_total_tokens``."""
|
||||
|
||||
def test_non_chunked_max_embed_len_caps_input(self):
|
||||
cfg = _embedding_model_config(max_model_len=128, max_embed_len=64)
|
||||
|
||||
tok_params = EmbeddingCompletionRequest(model="m", input="hi").build_tok_params(
|
||||
cfg
|
||||
)
|
||||
|
||||
assert tok_params.max_input_tokens == 64
|
||||
assert tok_params.max_output_tokens == 0
|
||||
|
||||
def test_chunked_max_embed_len_caps_input(self):
|
||||
cfg = _embedding_model_config(
|
||||
max_model_len=128, max_embed_len=64, enable_chunked_processing=True
|
||||
)
|
||||
|
||||
tok_params = EmbeddingCompletionRequest(model="m", input="hi").build_tok_params(
|
||||
cfg
|
||||
)
|
||||
|
||||
assert tok_params.max_input_tokens == 64
|
||||
assert tok_params.max_output_tokens == 0
|
||||
|
||||
def test_no_max_embed_len_falls_back_to_model_len(self):
|
||||
cfg = _embedding_model_config(max_model_len=128)
|
||||
|
||||
tok_params = EmbeddingCompletionRequest(model="m", input="hi").build_tok_params(
|
||||
cfg
|
||||
)
|
||||
|
||||
assert tok_params.max_input_tokens == 128
|
||||
assert tok_params.max_output_tokens == 0
|
||||
|
||||
Reference in New Issue
Block a user