[Frontend] Apply each endpoint's top_logprobs cut in derender (#59518)
Signed-off-by: Yin Li <kevinli.ai.work@gmail.com> Co-authored-by: Yin Li <kevinli.ai.work@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Yin Li
Cyrus Leung
parent
011fa68ca9
commit
3dcfaeb4e8
@@ -139,7 +139,7 @@ def test_sentencepiece_leading_space_matches_engine():
|
||||
assert [t.token for t in entry.top_logprobs] == convert_ids_list_to_tokens(
|
||||
tokenizer, [spaced, bare]
|
||||
)
|
||||
completion = _convert_chat_logprobs_to_completion_logprobs(resolved)
|
||||
completion = _convert_chat_logprobs_to_completion_logprobs(resolved, logprobs)
|
||||
assert completion.top_logprobs == [{" true": -0.5, "true": -1.5}]
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,212 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""generate + derender must return the same `top_logprobs` as the coupled
|
||||
`/v1/chat/completions` and `/v1/completions` endpoints (#59513).
|
||||
|
||||
Both sides start from the same per-position engine dicts, built with the
|
||||
engine's own `append_logprobs_for_next_position` and detokenized the way the
|
||||
engine does, and go through the real coupled serving objects on one side and
|
||||
`/inference/v1/generate` + derender on the other. The cases include a sampled
|
||||
token outside the top k, where chat keeps k entries and completions k + 1; the
|
||||
greedy end-to-end parity tests never produce that case.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.entrypoints.openai.chat_completion.test_serving_chat import (
|
||||
_build_mock_engine,
|
||||
_build_serving_chat,
|
||||
)
|
||||
from tests.entrypoints.openai.completion.test_completion_error import (
|
||||
_build_serving_completion,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateLogProbs
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
|
||||
from vllm.logprobs import append_logprobs_for_next_position
|
||||
from vllm.renderers.online_derenderer import (
|
||||
_chat_top_logprobs_limit,
|
||||
_completion_top_logprobs_limit,
|
||||
_convert_chat_logprobs_to_completion_logprobs,
|
||||
_resolve_logprobs,
|
||||
)
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
from vllm.tokenizers.detokenizer_utils import convert_ids_list_to_tokens
|
||||
|
||||
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def tokenizer():
|
||||
return get_tokenizer(MODEL_NAME)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def serving_chat():
|
||||
return _build_serving_chat(_build_mock_engine())
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def serving_completion():
|
||||
return _build_serving_completion(_build_mock_engine())
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def vocab_ids(tokenizer):
|
||||
"""Ordinary word pieces (no byte-fallback tokens, which need the U+FFFD
|
||||
context repair that is compared elsewhere)."""
|
||||
ids = tokenizer.encode(
|
||||
"the quick brown fox jumps over a lazy dog while seven cats watch "
|
||||
"from an old wooden fence near the river bank",
|
||||
add_special_tokens=False,
|
||||
)
|
||||
pieces = tokenizer.convert_ids_to_tokens(ids)
|
||||
return list(dict.fromkeys(i for i, p in zip(ids, pieces) if "<0x" not in p))
|
||||
|
||||
|
||||
def _positions(tokenizer, vocab_ids, k, sampled_ranks):
|
||||
"""Engine output for one completion: per position the sampled id and its
|
||||
dict (sampled first, then ranks 1..k), with decoded tokens as the engine
|
||||
fills them. k = -1 returns every candidate, as for the whole vocabulary."""
|
||||
token_ids, dicts = [], []
|
||||
for pos, sampled_rank in enumerate(sampled_ranks):
|
||||
n_top = max(k, sampled_rank) if k != -1 else len(vocab_ids) - 1
|
||||
ranked = [vocab_ids[(pos + j) % len(vocab_ids)] for j in range(n_top + 1)]
|
||||
sampled = ranked[sampled_rank - 1]
|
||||
top = ranked[: len(ranked) - 1] if k == -1 else ranked[:k]
|
||||
ids = [sampled, *top]
|
||||
logprobs = [-0.25 * sampled_rank] + [-0.25 * (r + 1) for r in range(len(top))]
|
||||
per_position: list = []
|
||||
append_logprobs_for_next_position(
|
||||
per_position,
|
||||
ids,
|
||||
logprobs,
|
||||
convert_ids_list_to_tokens(tokenizer, ids),
|
||||
sampled_rank,
|
||||
len(top),
|
||||
)
|
||||
token_ids.append(sampled)
|
||||
dicts.append(per_position[0])
|
||||
return token_ids, dicts
|
||||
|
||||
|
||||
def _generate(token_ids, dicts) -> GenerateLogProbs:
|
||||
out = ServingTokens._create_tokens_logprobs(
|
||||
None, token_ids=token_ids, top_logprobs=dicts
|
||||
)
|
||||
# Over the wire, as a derender client would send it back.
|
||||
return GenerateLogProbs.model_validate_json(out.model_dump_json())
|
||||
|
||||
|
||||
# Sampled token rank per position: top-1, inside the top k, outside it.
|
||||
RANKS = [1, 2, 1, 7, 3, 9, 1]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("k", [0, 1, 2, 5, -1])
|
||||
def test_chat_derender_matches_coupled_chat(tokenizer, vocab_ids, serving_chat, k):
|
||||
token_ids, dicts = _positions(tokenizer, vocab_ids, k, RANKS)
|
||||
coupled = serving_chat._create_chat_logprobs(
|
||||
token_ids=token_ids,
|
||||
top_logprobs=dicts,
|
||||
tokenizer=tokenizer,
|
||||
num_output_top_logprobs=k,
|
||||
)
|
||||
request = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
logprobs=True,
|
||||
top_logprobs=k,
|
||||
)
|
||||
derendered = _resolve_logprobs(
|
||||
_generate(token_ids, dicts),
|
||||
tokenizer,
|
||||
top_limit=_chat_top_logprobs_limit(request),
|
||||
)
|
||||
assert derendered.model_dump() == coupled.model_dump()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("k", [0, 1, 2, 5, -1])
|
||||
def test_completion_derender_matches_coupled_completion(
|
||||
tokenizer, vocab_ids, serving_completion, k
|
||||
):
|
||||
token_ids, dicts = _positions(tokenizer, vocab_ids, k, RANKS)
|
||||
coupled = serving_completion._create_completion_logprobs(
|
||||
token_ids=token_ids,
|
||||
top_logprobs=dicts,
|
||||
num_output_top_logprobs=k,
|
||||
tokenizer=tokenizer,
|
||||
)
|
||||
request = CompletionRequest(model=MODEL_NAME, prompt="hi", logprobs=k)
|
||||
generated = _generate(token_ids, dicts)
|
||||
derendered = _convert_chat_logprobs_to_completion_logprobs(
|
||||
_resolve_logprobs(
|
||||
generated,
|
||||
tokenizer,
|
||||
top_limit=_completion_top_logprobs_limit(request),
|
||||
),
|
||||
generated,
|
||||
)
|
||||
assert derendered.model_dump() == coupled.model_dump()
|
||||
if k > 0:
|
||||
# The case #59513 is about: completions keeps k + 1 entries when the
|
||||
# sampled token is outside the top k.
|
||||
outside = [i for i, r in enumerate(RANKS) if r > k]
|
||||
assert all(len(derendered.top_logprobs[i]) == k + 1 for i in outside)
|
||||
if k == -1:
|
||||
# /v1/completions returns every candidate for logprobs=-1 (#55717).
|
||||
assert [len(top) for top in derendered.top_logprobs] == [len(d) for d in dicts]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"top_limit",
|
||||
[
|
||||
_chat_top_logprobs_limit(None),
|
||||
# A request that does not ask for logprobs says nothing about the cut.
|
||||
_chat_top_logprobs_limit(
|
||||
ChatCompletionRequest(
|
||||
model=MODEL_NAME, messages=[{"role": "user", "content": "hi"}]
|
||||
)
|
||||
),
|
||||
_completion_top_logprobs_limit(None),
|
||||
_completion_top_logprobs_limit(
|
||||
CompletionRequest(model=MODEL_NAME, prompt="hi")
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_without_a_logprobs_request_derender_keeps_every_candidate(
|
||||
tokenizer, vocab_ids, top_limit
|
||||
):
|
||||
token_ids, dicts = _positions(tokenizer, vocab_ids, 2, RANKS)
|
||||
derendered = _resolve_logprobs(
|
||||
_generate(token_ids, dicts), tokenizer, top_limit=top_limit
|
||||
)
|
||||
assert [len(e.top_logprobs) for e in derendered.content] == [len(d) for d in dicts]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("k", [-1, 0, 2])
|
||||
def test_completion_position_without_logprobs_stays_none(
|
||||
tokenizer, vocab_ids, serving_completion, k
|
||||
):
|
||||
"""A position the engine returned no logprobs for is `None` in
|
||||
`/v1/completions`, not `{}`; derender keeps it that way."""
|
||||
token_ids, dicts = _positions(tokenizer, vocab_ids, k, RANKS)
|
||||
dicts[3] = None
|
||||
coupled = serving_completion._create_completion_logprobs(
|
||||
token_ids=token_ids,
|
||||
top_logprobs=dicts,
|
||||
num_output_top_logprobs=k,
|
||||
tokenizer=tokenizer,
|
||||
)
|
||||
request = CompletionRequest(model=MODEL_NAME, prompt="hi", logprobs=k)
|
||||
generated = _generate(token_ids, dicts)
|
||||
derendered = _convert_chat_logprobs_to_completion_logprobs(
|
||||
_resolve_logprobs(
|
||||
generated,
|
||||
tokenizer,
|
||||
top_limit=_completion_top_logprobs_limit(request),
|
||||
),
|
||||
generated,
|
||||
)
|
||||
assert derendered.top_logprobs == coupled.top_logprobs
|
||||
assert derendered.top_logprobs[3] is None
|
||||
@@ -14,11 +14,11 @@ from vllm.logprobs import Logprob
|
||||
|
||||
def test_top_logprobs_alternatives_have_own_token_ids():
|
||||
"""Each top_logprobs alternative must carry its own integer token id."""
|
||||
# Engine dict at logprobs=2 with the sampled token 262 also top-1.
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
None,
|
||||
token_ids=[262],
|
||||
top_logprobs=[{262: Logprob(-0.1), 257: Logprob(-1.2), 428: Logprob(-2.3)}],
|
||||
num_output_top_logprobs=2,
|
||||
top_logprobs=[{262: Logprob(-0.1), 257: Logprob(-1.2)}],
|
||||
)
|
||||
token_ids = {e.token_id for e in result.content[0].top_logprobs}
|
||||
assert token_ids == {262, 257}, f"got {token_ids}"
|
||||
@@ -32,7 +32,6 @@ def test_sampled_entry_carries_token_id_and_rank():
|
||||
top_logprobs=[
|
||||
{262: Logprob(-0.1, rank=1), 257: Logprob(-1.2, rank=2)},
|
||||
],
|
||||
num_output_top_logprobs=2,
|
||||
)
|
||||
entry = result.content[0]
|
||||
assert entry.token_id == 262
|
||||
@@ -50,7 +49,6 @@ def test_sampled_token_absent_from_topk_uses_sentinel():
|
||||
None,
|
||||
token_ids=[5],
|
||||
top_logprobs=[{7: Logprob(-0.9)}],
|
||||
num_output_top_logprobs=1,
|
||||
)
|
||||
entry = result.content[0]
|
||||
assert entry.token_id == 5
|
||||
@@ -60,14 +58,13 @@ def test_sampled_token_absent_from_topk_uses_sentinel():
|
||||
|
||||
|
||||
def test_logprobs_zero_emits_sampled_token():
|
||||
"""logprobs=0 must still emit 1 entry (the sampled token)."""
|
||||
"""logprobs=0: the engine returns only the sampled token, and it is kept."""
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
None,
|
||||
token_ids=[7],
|
||||
top_logprobs=[{7: Logprob(-0.9), 8: Logprob(-1.1)}],
|
||||
num_output_top_logprobs=0,
|
||||
top_logprobs=[{7: Logprob(-0.9)}],
|
||||
)
|
||||
assert len(result.content[0].top_logprobs) == 1
|
||||
assert [t.token_id for t in result.content[0].top_logprobs] == [7]
|
||||
|
||||
|
||||
def test_logprobs_minus_one_emits_all_tokens():
|
||||
@@ -75,7 +72,6 @@ def test_logprobs_minus_one_emits_all_tokens():
|
||||
None,
|
||||
token_ids=[7],
|
||||
top_logprobs=[{7: Logprob(-0.9), 8: Logprob(-1.1)}],
|
||||
num_output_top_logprobs=-1,
|
||||
)
|
||||
assert len(result.content[0].top_logprobs) == 2
|
||||
|
||||
@@ -86,7 +82,6 @@ def test_text_logprobs_without_tokenizer_are_placeholders_without_bytes():
|
||||
None,
|
||||
token_ids=[7],
|
||||
top_logprobs=[{7: Logprob(-0.9), 8: Logprob(-1.1)}],
|
||||
num_output_top_logprobs=2,
|
||||
)
|
||||
entry = result.content[0]
|
||||
assert entry.token == "token_id:7"
|
||||
@@ -102,7 +97,6 @@ def test_resolved_logprobs_use_engine_decoded_tokens_and_bytes():
|
||||
top_logprobs=[
|
||||
{7: Logprob(-0.9, decoded_token="é"), 8: Logprob(-1.1, decoded_token="e")}
|
||||
],
|
||||
num_output_top_logprobs=2,
|
||||
tokenizer=tokenizer,
|
||||
)
|
||||
entry = result.content[0]
|
||||
@@ -121,7 +115,6 @@ def test_resolved_logprobs_decode_tokens_the_engine_did_not():
|
||||
None,
|
||||
token_ids=[7, 9],
|
||||
top_logprobs=[{7: Logprob(-0.9)}, None],
|
||||
num_output_top_logprobs=1,
|
||||
tokenizer=tokenizer,
|
||||
)
|
||||
assert [e.token for e in result.content] == ["<7>", "<9>"]
|
||||
@@ -130,9 +123,9 @@ def test_resolved_logprobs_decode_tokens_the_engine_did_not():
|
||||
|
||||
|
||||
def test_sampled_token_outside_topk_comes_first():
|
||||
"""The engine puts the sampled token first, then ranks 1..k. When the
|
||||
sampled token is outside the top k it takes one of the k slots, so rank k
|
||||
is left out (same as the OpenAI endpoints)."""
|
||||
"""The engine puts the sampled token first, then ranks 1..k. Generate keeps
|
||||
all k + 1 candidates when the sampled token is outside the top k; derender
|
||||
applies each endpoint's cut (#59513)."""
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
None,
|
||||
token_ids=[50],
|
||||
@@ -143,9 +136,8 @@ def test_sampled_token_outside_topk_comes_first():
|
||||
20: Logprob(-1.0, rank=2),
|
||||
}
|
||||
],
|
||||
num_output_top_logprobs=2,
|
||||
)
|
||||
assert [t.rank for t in result.content[0].top_logprobs] == [5, 1]
|
||||
assert [t.rank for t in result.content[0].top_logprobs] == [5, 1, 2]
|
||||
|
||||
|
||||
def test_logprob_is_required_on_the_wire():
|
||||
@@ -160,7 +152,6 @@ def test_rank_zero_from_a_nan_logprob_is_none():
|
||||
None,
|
||||
token_ids=[7],
|
||||
top_logprobs=[{7: Logprob(float("nan"), rank=0), 8: Logprob(-0.2, rank=1)}],
|
||||
num_output_top_logprobs=1,
|
||||
)
|
||||
entry = result.content[0]
|
||||
assert entry.rank is None
|
||||
@@ -176,7 +167,6 @@ def test_text_logprobs_clamp_a_nan_logprob():
|
||||
None,
|
||||
token_ids=[7],
|
||||
top_logprobs=[{7: Logprob(float("nan"), rank=0, decoded_token="a")}],
|
||||
num_output_top_logprobs=1,
|
||||
tokenizer=MagicMock(),
|
||||
)
|
||||
entry = result.content[0]
|
||||
|
||||
Reference in New Issue
Block a user