[Frontend] Resolve logprobs on the parsed streaming derender path (#60437)
Signed-off-by: Shimi Bandiel <shimib@google.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Claude
wang.yuqi
Kevin H. Luu
parent
9a16d43ef0
commit
024c5188bf
@@ -1133,6 +1133,50 @@ async def test_e2e_parsed_reasoning(parser_client, parser_tokenizer):
|
||||
assert "<think>" not in msg["content"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("include_reasoning", [True, False])
|
||||
async def test_e2e_parsed_logprobs_follow_include_reasoning(
|
||||
parser_client, parser_tokenizer, include_reasoning
|
||||
):
|
||||
"""With a parser configured, batch derender returns logprobs only when
|
||||
reasoning is shown. Hidden reasoning drops them, as the coupled chat
|
||||
endpoint does, because decoded logprob tokens would reveal it."""
|
||||
messages = [{"role": "user", "content": "What is 2+3?"}]
|
||||
gen_req = await _e2e_render_chat(parser_client, PARSER_MODEL, messages)
|
||||
output_ids = _require_markers_survive(
|
||||
parser_tokenizer, "<think>Two plus three.</think>Five.", "</think>"
|
||||
)
|
||||
generate_response = _e2e_generate_response(output_ids)
|
||||
generate_response["choices"][0]["logprobs"] = {
|
||||
"content": [
|
||||
{"token_id": tid, "logprob": -0.5, "rank": 1, "top_logprobs": []}
|
||||
for tid in output_ids
|
||||
]
|
||||
}
|
||||
resp = await parser_client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": PARSER_MODEL,
|
||||
"generate_response": generate_response,
|
||||
"prompt_tokens": len(gen_req["token_ids"]),
|
||||
"chat_request": {
|
||||
"model": PARSER_MODEL,
|
||||
"messages": messages,
|
||||
"include_reasoning": include_reasoning,
|
||||
},
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
choice = resp.json()["choices"][0]
|
||||
assert choice["message"]["content"] == "Five."
|
||||
if include_reasoning:
|
||||
assert choice["message"]["reasoning"]
|
||||
assert len(choice["logprobs"]["content"]) == len(output_ids)
|
||||
else:
|
||||
assert choice["message"]["reasoning"] is None
|
||||
assert choice["logprobs"] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_e2e_parsed_tool_call(parser_client, parser_tokenizer):
|
||||
"""<tool_call> extracted into tool_calls field."""
|
||||
|
||||
@@ -316,13 +316,18 @@ async def test_parity_logprobs(client):
|
||||
|
||||
|
||||
async def _coupled_stream(
|
||||
client: httpx.AsyncClient, messages: list[dict], **extra
|
||||
) -> tuple[list[dict], list[int]]:
|
||||
client: httpx.AsyncClient,
|
||||
messages: list[dict],
|
||||
require_token_ids: bool = True,
|
||||
**extra,
|
||||
) -> tuple[list[dict], list[int] | None]:
|
||||
"""Stream `/v1/chat/completions` and return the choices that carry
|
||||
`token_ids` along with the prompt token IDs.
|
||||
|
||||
The leading role only chunk has no `token_ids` and no counterpart on
|
||||
`/inference/v1/generate`, so it is dropped.
|
||||
`/inference/v1/generate`, so it is dropped. With `require_token_ids`
|
||||
off every choice is kept and `prompt_token_ids` may be `None`, for
|
||||
requests where the coupled server hides token ids (hidden reasoning).
|
||||
"""
|
||||
choices: list[dict] = []
|
||||
prompt_token_ids: list[int] | None = None
|
||||
@@ -351,9 +356,12 @@ async def _coupled_stream(
|
||||
if chunk.get("prompt_token_ids") is not None:
|
||||
prompt_token_ids = chunk["prompt_token_ids"]
|
||||
choices.extend(
|
||||
ch for ch in chunk["choices"] if ch.get("token_ids") is not None
|
||||
ch
|
||||
for ch in chunk["choices"]
|
||||
if not require_token_ids or ch.get("token_ids") is not None
|
||||
)
|
||||
assert prompt_token_ids is not None
|
||||
if require_token_ids:
|
||||
assert prompt_token_ids is not None
|
||||
return choices, prompt_token_ids
|
||||
|
||||
|
||||
@@ -473,6 +481,151 @@ async def test_stream_parity_reasoning_and_tool_call(client, chunking):
|
||||
pytest.skip("Model did not emit both a <think> block and a tool call")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Streaming logprobs on the parser path
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _logprob_tokens(choice: dict) -> list[tuple[str, list[int] | None]] | None:
|
||||
lp = choice.get("logprobs")
|
||||
if lp is None or lp.get("content") is None:
|
||||
return None
|
||||
return [(e["token"], e.get("bytes")) for e in lp["content"]]
|
||||
|
||||
|
||||
def _sampled_generate_logprobs(choice: dict) -> dict | None:
|
||||
"""`GenerateLogProbs` for one coupled stream chunk from its `token_ids`
|
||||
and per-token logprob values, without `top_logprobs`.
|
||||
|
||||
The coupled stream's `token_ids` are the integer ids a GPU-less worker
|
||||
would have sent, so one stream is both the derender input and the string
|
||||
reference. A second greedy run is not needed, and greedy decoding is not
|
||||
bit-reproducible across calls on every platform anyway.
|
||||
"""
|
||||
lp = choice.get("logprobs")
|
||||
if lp is None or lp.get("content") is None:
|
||||
return None
|
||||
content = lp["content"]
|
||||
assert len(content) == len(choice["token_ids"]), choice
|
||||
return {
|
||||
"content": [
|
||||
{"token_id": tid, "logprob": entry["logprob"], "top_logprobs": []}
|
||||
for tid, entry in zip(choice["token_ids"], content)
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
async def _run_stream_logprob_parity_case(
|
||||
client: httpx.AsyncClient, messages: list[dict], **extra
|
||||
) -> tuple[dict, list[dict], list[dict]]:
|
||||
"""Stream with logprobs through the coupled server, replay the same
|
||||
chunks through the streaming derender endpoint and compare per chunk.
|
||||
Returns the assembled coupled message, the coupled choices and the
|
||||
derender choices.
|
||||
"""
|
||||
extra = {"logprobs": True, **extra}
|
||||
coupled_choices, prompt_token_ids = await _coupled_stream(client, messages, **extra)
|
||||
finish_reason = coupled_choices[-1]["finish_reason"]
|
||||
derender_choices: list[dict] = []
|
||||
disagg = await stream_chat_derender(
|
||||
client,
|
||||
[tid for ch in coupled_choices for tid in ch["token_ids"]],
|
||||
[len(ch["token_ids"]) for ch in coupled_choices],
|
||||
{"model": MODEL, "messages": messages, **extra},
|
||||
len(prompt_token_ids),
|
||||
prompt_token_ids,
|
||||
finish_reason="stop" if finish_reason == "tool_calls" else finish_reason,
|
||||
logprobs=[_sampled_generate_logprobs(ch) for ch in coupled_choices],
|
||||
raw_choices=derender_choices,
|
||||
)
|
||||
coupled = assemble_stream(coupled_choices)
|
||||
for message in (coupled, disagg):
|
||||
for tc in message["tool_calls"]:
|
||||
tc["id"] = _tool_call_id_shape(tc["id"])
|
||||
assert disagg == coupled
|
||||
assert len(derender_choices) == len(coupled_choices)
|
||||
for ref, der in zip(coupled_choices, derender_choices):
|
||||
assert _logprob_tokens(der) == _logprob_tokens(ref), (ref, der)
|
||||
return coupled, coupled_choices, derender_choices
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_parity_logprobs_reasoning(client):
|
||||
"""Per-chunk logprobs on the parser path match the coupled stream across
|
||||
the reasoning -> content transition."""
|
||||
messages = [{"role": "user", "content": "What is 17 times 23? Think it through."}]
|
||||
coupled, _, derender_choices = await _run_stream_logprob_parity_case(
|
||||
client, messages, include_reasoning=True, max_tokens=256
|
||||
)
|
||||
if not coupled["reasoning"]:
|
||||
pytest.skip("Model did not emit a <think> block")
|
||||
assert any(
|
||||
ch["delta"].get("reasoning") and ch.get("logprobs") for ch in derender_choices
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_parity_logprobs_tool_call(client):
|
||||
"""Chunks the parser swallows (tool call markup) still carry their
|
||||
logprobs, on both sides, so no token's logprobs are lost."""
|
||||
messages = [{"role": "user", "content": "What's the weather in Paris?"}]
|
||||
coupled, _, derender_choices = await _run_stream_logprob_parity_case(
|
||||
client,
|
||||
messages,
|
||||
tools=TOOLS,
|
||||
tool_choice=FORCE_WEATHER_TOOL,
|
||||
include_reasoning=True,
|
||||
max_tokens=512,
|
||||
)
|
||||
if not coupled["tool_calls"]:
|
||||
pytest.skip("Model did not emit a tool call")
|
||||
swallowed = [
|
||||
ch
|
||||
for ch in derender_choices
|
||||
if ch.get("logprobs")
|
||||
and not ch["delta"].get("content")
|
||||
and not ch["delta"].get("reasoning")
|
||||
and not ch["delta"].get("tool_calls")
|
||||
]
|
||||
assert swallowed, "expected at least one buffered-markup chunk with logprobs"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_parity_logprobs_hidden_reasoning(client):
|
||||
"""With `include_reasoning=False` and a parser configured, neither side
|
||||
returns logprobs on any chunk; derender still emits the content."""
|
||||
messages = [{"role": "user", "content": "What is 17 times 23? Think it through."}]
|
||||
extra = {"logprobs": True, "max_tokens": 256}
|
||||
# Token ids come from a shown-reasoning run: the coupled server hides
|
||||
# them along with the logprobs when reasoning is hidden.
|
||||
shown_choices, prompt_token_ids = await _coupled_stream(
|
||||
client, messages, include_reasoning=True, **extra
|
||||
)
|
||||
hidden_choices, _ = await _coupled_stream(
|
||||
client, messages, require_token_ids=False, include_reasoning=False, **extra
|
||||
)
|
||||
assert all(ch.get("logprobs") is None for ch in hidden_choices)
|
||||
assert assemble_stream(hidden_choices)["reasoning"] is None
|
||||
|
||||
finish_reason = shown_choices[-1]["finish_reason"]
|
||||
derender_choices: list[dict] = []
|
||||
disagg = await stream_chat_derender(
|
||||
client,
|
||||
[tid for ch in shown_choices for tid in ch["token_ids"]],
|
||||
[len(ch["token_ids"]) for ch in shown_choices],
|
||||
{"model": MODEL, "messages": messages, "include_reasoning": False, **extra},
|
||||
len(prompt_token_ids),
|
||||
prompt_token_ids,
|
||||
finish_reason="stop" if finish_reason == "tool_calls" else finish_reason,
|
||||
logprobs=[_sampled_generate_logprobs(ch) for ch in shown_choices],
|
||||
raw_choices=derender_choices,
|
||||
)
|
||||
assert all(ch.get("logprobs") is None for ch in derender_choices)
|
||||
shown = assemble_stream(shown_choices)
|
||||
assert disagg["reasoning"] is None
|
||||
assert disagg["content"] == shown["content"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Inline text level: generate(output_mode="text") == derender(generate(...))
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -1427,6 +1427,159 @@ class TestDerenderChatStreamParsed:
|
||||
assert delta.reasoning is None
|
||||
assert delta.content == "c"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_logprobs_attached_to_each_parsed_delta(
|
||||
self, parsed_derenderer, tokenizer
|
||||
):
|
||||
"""A chunk's logprobs ride on whatever delta the parser emits for it
|
||||
(reasoning, content or tool call), with token ids decoded, as on
|
||||
`/v1/chat/completions`."""
|
||||
chunks = [
|
||||
[_FakeParser.REASON],
|
||||
[_FakeParser.CONTENT],
|
||||
[_FakeParser.TOOL_START, _FakeParser.TOOL_ARG],
|
||||
]
|
||||
state = None
|
||||
seen = []
|
||||
for ids in chunks:
|
||||
chunk, state = await parsed_derenderer.derender_chat_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk(
|
||||
ids, logprobs=_generate_logprobs(ids)
|
||||
),
|
||||
state=state,
|
||||
chat_request=_chat_request(),
|
||||
prompt_token_ids=[1, 2],
|
||||
)
|
||||
choice = chunk.choices[0]
|
||||
assert choice.logprobs is not None and choice.logprobs.content is not None
|
||||
assert [e.token for e in choice.logprobs.content] == (
|
||||
convert_ids_list_to_tokens(tokenizer, ids)
|
||||
)
|
||||
seen.append(choice.delta)
|
||||
assert seen[0].reasoning == "r"
|
||||
assert seen[1].content == "c"
|
||||
assert (
|
||||
seen[2].tool_calls and seen[2].tool_calls[0].function.name == "get_weather"
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("request_kwargs", "expected_candidates"),
|
||||
[({"logprobs": True, "top_logprobs": 2}, 2), ({}, 4)],
|
||||
)
|
||||
async def test_top_logprobs_follow_the_chat_cut(
|
||||
self, parsed_derenderer, request_kwargs, expected_candidates
|
||||
):
|
||||
"""The parsed path trims generate's candidate list to the request's
|
||||
`top_logprobs` like the plain path does, and keeps every candidate
|
||||
when the request does not ask for logprobs."""
|
||||
ids = [_FakeParser.CONTENT]
|
||||
logprobs = _generate_logprobs(ids)
|
||||
logprobs["content"][0]["top_logprobs"] = [
|
||||
{"token_id": tid, "logprob": -float(rank)}
|
||||
for rank, tid in enumerate([ids[0], 11, 12, 13])
|
||||
]
|
||||
chunk, _ = await parsed_derenderer.derender_chat_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk(ids, logprobs=logprobs),
|
||||
chat_request=_chat_request(**request_kwargs),
|
||||
prompt_token_ids=[1, 2],
|
||||
)
|
||||
content = chunk.choices[0].logprobs.content
|
||||
assert len(content) == 1
|
||||
assert len(content[0].top_logprobs) == expected_candidates
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_logprobs_kept_when_parser_swallows_chunk(
|
||||
self, parsed_derenderer, tokenizer
|
||||
):
|
||||
"""A chunk the parser emits nothing for (buffered markup) still
|
||||
carries its logprobs on an empty delta, so no token's logprobs are
|
||||
lost. Mirrors the coupled endpoint, which emits a chunk whenever
|
||||
logprobs are present even if the delta is empty."""
|
||||
swallowed = [_FakeParser.CONTENT + 1] # unknown to _FakeParser -> None
|
||||
chunk, _ = await parsed_derenderer.derender_chat_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk(
|
||||
swallowed, logprobs=_generate_logprobs(swallowed)
|
||||
),
|
||||
chat_request=_chat_request(),
|
||||
prompt_token_ids=[1, 2],
|
||||
)
|
||||
choice = chunk.choices[0]
|
||||
assert choice.delta.content is None and not choice.delta.tool_calls
|
||||
assert [e.token for e in choice.logprobs.content] == (
|
||||
convert_ids_list_to_tokens(tokenizer, swallowed)
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_include_reasoning_false_drops_logprobs_on_every_chunk(
|
||||
self, parsed_derenderer
|
||||
):
|
||||
"""Hidden reasoning suppresses logprobs on reasoning *and* content
|
||||
chunks (decoded logprob tokens would leak the reasoning), the same
|
||||
rule `/v1/chat/completions` applies with a parser configured."""
|
||||
state = None
|
||||
for ids in ([_FakeParser.REASON], [_FakeParser.CONTENT]):
|
||||
chunk, state = await parsed_derenderer.derender_chat_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk(
|
||||
ids, logprobs=_generate_logprobs(ids)
|
||||
),
|
||||
state=state,
|
||||
chat_request=_chat_request(include_reasoning=False),
|
||||
prompt_token_ids=[1, 2],
|
||||
)
|
||||
assert chunk.choices[0].logprobs is None
|
||||
assert chunk.choices[0].delta.content == "c"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_logprobs_multibyte_across_chunks(self, parsed_derenderer, tokenizer):
|
||||
"""Byte-fallback correction crosses chunk boundaries on the parser
|
||||
path too, via the carried `logprob_context_token_ids`, which starts
|
||||
empty like the engine's and never includes the prompt."""
|
||||
token_ids = tokenizer.encode("👍", add_special_tokens=False)
|
||||
if len(token_ids) < 2 or "�" not in tokenizer.decode([token_ids[-1]]):
|
||||
pytest.skip("Tokenizer does not byte-split this character")
|
||||
state = None
|
||||
last_chunk = None
|
||||
for tid in token_ids:
|
||||
last_chunk, state = await parsed_derenderer.derender_chat_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk(
|
||||
[tid], logprobs=_generate_logprobs([tid])
|
||||
),
|
||||
state=state,
|
||||
chat_request=_chat_request(),
|
||||
prompt_token_ids=[1, 2, 3, 4, 5],
|
||||
)
|
||||
assert last_chunk is not None and state is not None
|
||||
final_entry = last_chunk.choices[0].logprobs.content[0]
|
||||
assert not final_entry.token.endswith("�"), final_entry.token
|
||||
assert state.logprob_context_token_ids == token_ids[-4:]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_logprob_context_advances_without_logprobs(self, parsed_derenderer):
|
||||
"""The context tail tracks sampled ids whether or not the chunk
|
||||
carries logprobs, so a later chunk that does can still be corrected."""
|
||||
ids = [_FakeParser.REASON, _FakeParser.CONTENT, _FakeParser.CONTENT]
|
||||
_, state = await parsed_derenderer.derender_chat_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk(ids),
|
||||
chat_request=_chat_request(),
|
||||
prompt_token_ids=[1, 2],
|
||||
)
|
||||
assert state.logprob_context_token_ids == ids
|
||||
_, state = await parsed_derenderer.derender_chat_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk([], finish_reason="stop"),
|
||||
state=state,
|
||||
chat_request=_chat_request(),
|
||||
prompt_token_ids=[1, 2],
|
||||
)
|
||||
assert state.logprob_context_token_ids == ids
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Harmony / GPT-OSS replay — unit, no server
|
||||
|
||||
@@ -52,6 +52,8 @@ async def stream_chat_derender(
|
||||
prompt_token_ids: list[int],
|
||||
finish_reason: str = "stop",
|
||||
on_chunk: Callable[[list[dict]], None] | None = None,
|
||||
logprobs: list[dict | None] | None = None,
|
||||
raw_choices: list[dict] | None = None,
|
||||
) -> dict:
|
||||
"""Feed `output_ids` through the streaming chat derender endpoint in
|
||||
the given `chunk_sizes`, threading `stream_state` across calls and
|
||||
@@ -64,7 +66,13 @@ async def stream_chat_derender(
|
||||
`tool_calls` assembled so far, letting callers assert properties of the
|
||||
intermediate deltas (e.g. monotonic argument growth) rather than only
|
||||
the final assembled result.
|
||||
|
||||
`logprobs` supplies one `GenerateLogProbs` payload (or `None`) per chunk
|
||||
and `raw_choices`, if given, collects every derendered choice so callers
|
||||
can compare per-chunk fields such as `logprobs`.
|
||||
"""
|
||||
if logprobs is not None:
|
||||
assert len(logprobs) == len(chunk_sizes)
|
||||
state = None
|
||||
choices: list[dict] = []
|
||||
pos = 0
|
||||
@@ -84,6 +92,7 @@ async def stream_chat_derender(
|
||||
"index": 0,
|
||||
"token_ids": tids,
|
||||
"finish_reason": finish_reason if is_last else None,
|
||||
"logprobs": logprobs[i] if logprobs is not None else None,
|
||||
}
|
||||
],
|
||||
},
|
||||
@@ -101,6 +110,8 @@ async def stream_chat_derender(
|
||||
assert data["chunk"]["choices"][0]["token_ids"] == tids
|
||||
state = data["stream_state"]
|
||||
choices.extend(data["chunk"]["choices"])
|
||||
if raw_choices is not None:
|
||||
raw_choices.extend(data["chunk"]["choices"])
|
||||
if on_chunk is not None:
|
||||
on_chunk(assemble_stream(choices)["tool_calls"])
|
||||
|
||||
|
||||
Reference in New Issue
Block a user