[Frontend] Add truncation side to OpenAI endpoints (#43260)
Signed-off-by: Rui Zhang <rza21.bc@gmail.com> Signed-off-by: Rui Zhang <rui.zhang@globalrelay.net> Co-authored-by: Rui Zhang <rui.zhang@globalrelay.net>
This commit is contained in:
@@ -73,6 +73,43 @@ async def test_single_completion(client: openai.AsyncOpenAI, model_name: str) ->
|
||||
assert completion.choices[0].prompt_logprobs is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_completion_truncation_side_controls_prompt_truncation(
|
||||
client: openai.AsyncOpenAI, model_name: str
|
||||
) -> None:
|
||||
prompt_token_ids = list(range(8))
|
||||
|
||||
right_completion = await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=prompt_token_ids,
|
||||
max_tokens=1,
|
||||
temperature=0.0,
|
||||
extra_body={
|
||||
"return_token_ids": True,
|
||||
"truncate_prompt_tokens": 4,
|
||||
"truncation_side": "right",
|
||||
},
|
||||
)
|
||||
assert right_completion.choices[0].prompt_token_ids == prompt_token_ids[:4]
|
||||
|
||||
left_completion = await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=prompt_token_ids,
|
||||
max_tokens=1,
|
||||
temperature=0.0,
|
||||
extra_body={
|
||||
"return_token_ids": True,
|
||||
"truncate_prompt_tokens": 4,
|
||||
"truncation_side": "left",
|
||||
},
|
||||
)
|
||||
assert left_completion.choices[0].prompt_token_ids == prompt_token_ids[-4:]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
|
||||
Reference in New Issue
Block a user