[Frontend] Add truncation side to OpenAI endpoints (#43260)
Signed-off-by: Rui Zhang <rza21.bc@gmail.com> Signed-off-by: Rui Zhang <rui.zhang@globalrelay.net> Co-authored-by: Rui Zhang <rui.zhang@globalrelay.net>
This commit is contained in:
@@ -600,6 +600,7 @@ class MockEngine:
|
||||
model_config: MockModelConfig = field(default_factory=MockModelConfig)
|
||||
input_processor: MagicMock = field(default_factory=MagicMock)
|
||||
renderer: MagicMock = field(default_factory=MagicMock)
|
||||
errored: bool = False
|
||||
|
||||
|
||||
async def _async_serving_chat_init():
|
||||
@@ -858,6 +859,50 @@ async def test_serving_chat_truncate_prompt_tokens_max_token_accounting():
|
||||
assert mock_engine.generate.call_args.args[1].max_tokens == 93
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_serving_chat_truncation_side_controls_prompt_truncation():
|
||||
model_config = MockModelConfig()
|
||||
model_config.model = MODEL_NAME_SHORT
|
||||
model_config.tokenizer = MODEL_NAME_SHORT
|
||||
mock_engine = MockEngine(
|
||||
model_config=model_config,
|
||||
renderer=_build_renderer(model_config),
|
||||
)
|
||||
|
||||
serving_chat = _build_serving_chat(mock_engine)
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Summarize how prompt truncation works in one sentence.",
|
||||
}
|
||||
]
|
||||
|
||||
full_token_ids = await _render_chat_prompt_token_ids(
|
||||
serving_chat,
|
||||
messages,
|
||||
model_name=MODEL_NAME_SHORT,
|
||||
)
|
||||
assert len(full_token_ids) > 4
|
||||
|
||||
right_token_ids = await _render_chat_prompt_token_ids(
|
||||
serving_chat,
|
||||
messages,
|
||||
model_name=MODEL_NAME_SHORT,
|
||||
truncate_prompt_tokens=4,
|
||||
truncation_side="right",
|
||||
)
|
||||
assert right_token_ids == full_token_ids[:4]
|
||||
|
||||
left_token_ids = await _render_chat_prompt_token_ids(
|
||||
serving_chat,
|
||||
messages,
|
||||
model_name=MODEL_NAME_SHORT,
|
||||
truncate_prompt_tokens=4,
|
||||
truncation_side="left",
|
||||
)
|
||||
assert left_token_ids == full_token_ids[-4:]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_serving_chat_mistral_token_ids_prompt_is_validated():
|
||||
"""Regression test: when the Mistral tokenizer path returns token IDs
|
||||
@@ -1111,6 +1156,37 @@ async def test_serving_chat_data_parallel_rank_extraction():
|
||||
assert mock_engine.generate.call_args.kwargs["data_parallel_rank"] is None
|
||||
|
||||
|
||||
async def _render_chat_prompt_token_ids(
|
||||
serving_chat: OpenAIServingChat,
|
||||
messages: list[dict[str, str]],
|
||||
*,
|
||||
model_name: str = MODEL_NAME,
|
||||
truncate_prompt_tokens: int | None = None,
|
||||
truncation_side: str | None = None,
|
||||
) -> list[int]:
|
||||
request = ChatCompletionRequest(
|
||||
model=model_name,
|
||||
messages=messages,
|
||||
max_tokens=1,
|
||||
temperature=0.0,
|
||||
return_token_ids=True,
|
||||
truncate_prompt_tokens=truncate_prompt_tokens,
|
||||
truncation_side=truncation_side,
|
||||
)
|
||||
|
||||
result = await serving_chat.render_chat_request(request)
|
||||
assert not isinstance(result, ErrorResponse)
|
||||
|
||||
_, engine_inputs = result
|
||||
assert len(engine_inputs) == 1
|
||||
|
||||
prompt_token_ids = serving_chat._extract_prompt_components(
|
||||
engine_inputs[0]
|
||||
).token_ids
|
||||
assert prompt_token_ids is not None
|
||||
return prompt_token_ids
|
||||
|
||||
|
||||
class TestServingChatWithHarmony:
|
||||
"""
|
||||
These tests ensure Chat Completion requests are being properly converted into
|
||||
|
||||
@@ -73,6 +73,43 @@ async def test_single_completion(client: openai.AsyncOpenAI, model_name: str) ->
|
||||
assert completion.choices[0].prompt_logprobs is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_completion_truncation_side_controls_prompt_truncation(
|
||||
client: openai.AsyncOpenAI, model_name: str
|
||||
) -> None:
|
||||
prompt_token_ids = list(range(8))
|
||||
|
||||
right_completion = await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=prompt_token_ids,
|
||||
max_tokens=1,
|
||||
temperature=0.0,
|
||||
extra_body={
|
||||
"return_token_ids": True,
|
||||
"truncate_prompt_tokens": 4,
|
||||
"truncation_side": "right",
|
||||
},
|
||||
)
|
||||
assert right_completion.choices[0].prompt_token_ids == prompt_token_ids[:4]
|
||||
|
||||
left_completion = await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=prompt_token_ids,
|
||||
max_tokens=1,
|
||||
temperature=0.0,
|
||||
extra_body={
|
||||
"return_token_ids": True,
|
||||
"truncate_prompt_tokens": 4,
|
||||
"truncation_side": "left",
|
||||
},
|
||||
)
|
||||
assert left_completion.choices[0].prompt_token_ids == prompt_token_ids[-4:]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
|
||||
Reference in New Issue
Block a user