[Frontend] Add truncation side to OpenAI endpoints (#43260)

Signed-off-by: Rui Zhang <rza21.bc@gmail.com>
Signed-off-by: Rui Zhang <rui.zhang@globalrelay.net>
Co-authored-by: Rui Zhang <rui.zhang@globalrelay.net>
This commit is contained in:
ruizhang
2026-05-21 20:32:31 -07:00
committed by GitHub
co-authored by Rui Zhang
parent 35d0141a0b
commit 60af5c16ee
5 changed files with 147 additions and 6 deletions
@@ -600,6 +600,7 @@ class MockEngine:
model_config: MockModelConfig = field(default_factory=MockModelConfig)
input_processor: MagicMock = field(default_factory=MagicMock)
renderer: MagicMock = field(default_factory=MagicMock)
errored: bool = False
async def _async_serving_chat_init():
@@ -858,6 +859,50 @@ async def test_serving_chat_truncate_prompt_tokens_max_token_accounting():
assert mock_engine.generate.call_args.args[1].max_tokens == 93
@pytest.mark.asyncio
async def test_serving_chat_truncation_side_controls_prompt_truncation():
model_config = MockModelConfig()
model_config.model = MODEL_NAME_SHORT
model_config.tokenizer = MODEL_NAME_SHORT
mock_engine = MockEngine(
model_config=model_config,
renderer=_build_renderer(model_config),
)
serving_chat = _build_serving_chat(mock_engine)
messages = [
{
"role": "user",
"content": "Summarize how prompt truncation works in one sentence.",
}
]
full_token_ids = await _render_chat_prompt_token_ids(
serving_chat,
messages,
model_name=MODEL_NAME_SHORT,
)
assert len(full_token_ids) > 4
right_token_ids = await _render_chat_prompt_token_ids(
serving_chat,
messages,
model_name=MODEL_NAME_SHORT,
truncate_prompt_tokens=4,
truncation_side="right",
)
assert right_token_ids == full_token_ids[:4]
left_token_ids = await _render_chat_prompt_token_ids(
serving_chat,
messages,
model_name=MODEL_NAME_SHORT,
truncate_prompt_tokens=4,
truncation_side="left",
)
assert left_token_ids == full_token_ids[-4:]
@pytest.mark.asyncio
async def test_serving_chat_mistral_token_ids_prompt_is_validated():
"""Regression test: when the Mistral tokenizer path returns token IDs
@@ -1111,6 +1156,37 @@ async def test_serving_chat_data_parallel_rank_extraction():
assert mock_engine.generate.call_args.kwargs["data_parallel_rank"] is None
async def _render_chat_prompt_token_ids(
serving_chat: OpenAIServingChat,
messages: list[dict[str, str]],
*,
model_name: str = MODEL_NAME,
truncate_prompt_tokens: int | None = None,
truncation_side: str | None = None,
) -> list[int]:
request = ChatCompletionRequest(
model=model_name,
messages=messages,
max_tokens=1,
temperature=0.0,
return_token_ids=True,
truncate_prompt_tokens=truncate_prompt_tokens,
truncation_side=truncation_side,
)
result = await serving_chat.render_chat_request(request)
assert not isinstance(result, ErrorResponse)
_, engine_inputs = result
assert len(engine_inputs) == 1
prompt_token_ids = serving_chat._extract_prompt_components(
engine_inputs[0]
).token_ids
assert prompt_token_ids is not None
return prompt_token_ids
class TestServingChatWithHarmony:
"""
These tests ensure Chat Completion requests are being properly converted into
@@ -73,6 +73,43 @@ async def test_single_completion(client: openai.AsyncOpenAI, model_name: str) ->
assert completion.choices[0].prompt_logprobs is None
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME],
)
async def test_completion_truncation_side_controls_prompt_truncation(
client: openai.AsyncOpenAI, model_name: str
) -> None:
prompt_token_ids = list(range(8))
right_completion = await client.completions.create(
model=model_name,
prompt=prompt_token_ids,
max_tokens=1,
temperature=0.0,
extra_body={
"return_token_ids": True,
"truncate_prompt_tokens": 4,
"truncation_side": "right",
},
)
assert right_completion.choices[0].prompt_token_ids == prompt_token_ids[:4]
left_completion = await client.completions.create(
model=model_name,
prompt=prompt_token_ids,
max_tokens=1,
temperature=0.0,
extra_body={
"return_token_ids": True,
"truncate_prompt_tokens": 4,
"truncation_side": "left",
},
)
assert left_completion.choices[0].prompt_token_ids == prompt_token_ids[-4:]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",