[Feature] Add fixed-token prefill scoring (#54335)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
aoshen02
Claude Opus 5
parent
a7ff44355c
commit
b721a4c709
@@ -1,9 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import io
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from transformers import AutoTokenizer
|
||||
@@ -311,6 +314,52 @@ async def test_generate_logprobs(client, logprobs_value):
|
||||
assert len(entry["top_logprobs"]) == max(logprobs_value, 1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_prompt_token_id_logprobs(client):
|
||||
token_ids = [11, 22, 33, 44, 55]
|
||||
candidates = [22, 33, 44, 55]
|
||||
sampling_params = {
|
||||
"max_tokens": 1,
|
||||
"prompt_logprobs": 0,
|
||||
"prompt_logprob_token_ids": candidates,
|
||||
"prompt_logprob_start": 1,
|
||||
}
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": token_ids,
|
||||
"sampling_params": sampling_params,
|
||||
}
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
scores = np.load(io.BytesIO(base64.b64decode(data["prompt_token_id_logprobs"])))
|
||||
assert scores.shape == (len(token_ids) - 2, len(candidates))
|
||||
for row, target in enumerate(token_ids[2:]):
|
||||
expected = data["prompt_logprobs"][row + 2][str(target)]["logprob"]
|
||||
assert scores[row, candidates.index(target)] == pytest.approx(
|
||||
expected, abs=1e-3
|
||||
)
|
||||
|
||||
stream_only_ids = {"prompt_logprob_token_ids": candidates}
|
||||
resp = await client.post(
|
||||
GEN_ENDPOINT,
|
||||
json={**payload, "sampling_params": stream_only_ids, "stream": True},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
assert "prompt_logprob_token_ids" in resp.text
|
||||
|
||||
past_end = {**sampling_params, "prompt_logprob_start": len(token_ids)}
|
||||
resp = await client.post(
|
||||
GEN_ENDPOINT, json={**payload, "sampling_params": past_end}
|
||||
)
|
||||
resp.raise_for_status()
|
||||
empty = np.load(
|
||||
io.BytesIO(base64.b64decode(resp.json()["prompt_token_id_logprobs"]))
|
||||
)
|
||||
assert empty.shape == (0, len(candidates))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_same_response_as_chat_completions(client, tokenizer, messages):
|
||||
token_ids = tokenizer.apply_chat_template(
|
||||
|
||||
Reference in New Issue
Block a user