diff --git a/docs/serving/online_serving/derenderer.md b/docs/serving/online_serving/derenderer.md
index b7b5e5b8f0..afc7961dd2 100644
--- a/docs/serving/online_serving/derenderer.md
+++ b/docs/serving/online_serving/derenderer.md
@@ -105,7 +105,7 @@ The server keeps no state between calls. Everything the next call needs is in `s
- **Send `prompt_token_ids` on the first chunk, even without a parser.** It seeds detokenization so the first token's leading space matches the coupled endpoint (see [Request format](#request-format)). Later chunks already carry that context in `stream_state`.
- **Don't forward `[DONE]`.** It marks the end of the generate stream and isn't a chunk.
-`/inference/v1/generate` returns `GenerateLogProbs` with integer `token_id`s (the generate server has no tokenizer), and derender turns those into `ChatCompletionLogProbs` / `CompletionLogProbs`, filling `token` and `bytes` from the tokenizer with the usual U+FFFD byte-fallback correction. Generate returns every top-k candidate the engine produced; derender keeps the ones the coupled endpoint would, with the same helpers: the first `top_logprobs` on chat (all of them for `top_logprobs=-1`) and the first `logprobs + 1` on completions (all of them for `logprobs=-1`). It reads those values from `chat_request` / `completion_request`, so without the original request, or with one that does not ask for logprobs, it returns every candidate. Requests that use `logprob_token_ids` get no output logprobs from generate (their `sampling_params.logprobs` is unset), so derender has none to return for them either. Streaming derender does the same per chunk on the plain detokenization path (see [Streaming state and logprobs](#streaming-state-and-logprobs)). The parser path doesn't resolve them yet and drops them.
+`/inference/v1/generate` returns `GenerateLogProbs` with integer `token_id`s (the generate server has no tokenizer), and derender turns those into `ChatCompletionLogProbs` / `CompletionLogProbs`, filling `token` and `bytes` from the tokenizer with the usual U+FFFD byte-fallback correction. Generate returns every top-k candidate the engine produced; derender keeps the ones the coupled endpoint would, with the same helpers: the first `top_logprobs` on chat (all of them for `top_logprobs=-1`) and the first `logprobs + 1` on completions (all of them for `logprobs=-1`). It reads those values from `chat_request` / `completion_request`, so without the original request, or with one that does not ask for logprobs, it returns every candidate. Requests that use `logprob_token_ids` get no output logprobs from generate (their `sampling_params.logprobs` is unset), so derender has none to return for them either. Streaming derender does the same per chunk on both the plain detokenization path and the parser path (see [Streaming state and logprobs](#streaming-state-and-logprobs)). As on `/v1/chat/completions`, a request with `include_reasoning: false` and a parser configured gets no logprobs at all, batch or streaming, since the decoded logprob tokens would reveal the hidden reasoning.
## Streaming cost
@@ -129,7 +129,7 @@ Streaming derender is stateless on the server side: all mutable state lives in t
- `logprob_context_token_ids`: the trailing sampled token IDs (at most 4) from previous chunks, used to seed byte-fallback (U+FFFD) correction so multi-byte characters whose tokens split across chunk boundaries still resolve to real strings
- `logprob_text_offset`: the cumulative emitted text length, so `text_offset` in completion streaming logprobs stays absolute across chunks instead of restarting at 0
-When a streamed `GenerateTokensStreamChoice` carries `logprobs`, the integer `token_id`s are decoded per chunk and the resolved logprobs are attached to the corresponding streamed choice. For chat it is as `ChatCompletionLogProbs` and for completions it is converted to the flat `CompletionLogProbs` lists. Chunks without `logprobs` produce choices with `logprobs: null`.
+When a streamed `GenerateTokensStreamChoice` carries `logprobs`, the integer `token_id`s are decoded per chunk and the resolved logprobs are attached to the corresponding streamed choice. For chat it is as `ChatCompletionLogProbs` and for completions it is converted to the flat `CompletionLogProbs` lists. Chunks without `logprobs` produce choices with `logprobs: null`. On the parser path the chunk's logprobs are attached to whatever delta the parser emits for that chunk (reasoning, content or a tool call delta, or an empty delta while the parser is still buffering markup), which is what the coupled endpoint does; replay never touches them, since each chunk's `logprobs` cover exactly that chunk's tokens.
## Example
diff --git a/tests/entrypoints/scale_out/derender/test_derender.py b/tests/entrypoints/scale_out/derender/test_derender.py
index 771213309f..fc4278d3e8 100644
--- a/tests/entrypoints/scale_out/derender/test_derender.py
+++ b/tests/entrypoints/scale_out/derender/test_derender.py
@@ -1133,6 +1133,50 @@ async def test_e2e_parsed_reasoning(parser_client, parser_tokenizer):
assert "" not in msg["content"]
+@pytest.mark.asyncio
+@pytest.mark.parametrize("include_reasoning", [True, False])
+async def test_e2e_parsed_logprobs_follow_include_reasoning(
+ parser_client, parser_tokenizer, include_reasoning
+):
+ """With a parser configured, batch derender returns logprobs only when
+ reasoning is shown. Hidden reasoning drops them, as the coupled chat
+ endpoint does, because decoded logprob tokens would reveal it."""
+ messages = [{"role": "user", "content": "What is 2+3?"}]
+ gen_req = await _e2e_render_chat(parser_client, PARSER_MODEL, messages)
+ output_ids = _require_markers_survive(
+ parser_tokenizer, "Two plus three.Five.", ""
+ )
+ generate_response = _e2e_generate_response(output_ids)
+ generate_response["choices"][0]["logprobs"] = {
+ "content": [
+ {"token_id": tid, "logprob": -0.5, "rank": 1, "top_logprobs": []}
+ for tid in output_ids
+ ]
+ }
+ resp = await parser_client.post(
+ "/v1/chat/completions/derender",
+ json={
+ "model": PARSER_MODEL,
+ "generate_response": generate_response,
+ "prompt_tokens": len(gen_req["token_ids"]),
+ "chat_request": {
+ "model": PARSER_MODEL,
+ "messages": messages,
+ "include_reasoning": include_reasoning,
+ },
+ },
+ )
+ assert resp.status_code == 200, resp.text
+ choice = resp.json()["choices"][0]
+ assert choice["message"]["content"] == "Five."
+ if include_reasoning:
+ assert choice["message"]["reasoning"]
+ assert len(choice["logprobs"]["content"]) == len(output_ids)
+ else:
+ assert choice["message"]["reasoning"] is None
+ assert choice["logprobs"] is None
+
+
@pytest.mark.asyncio
async def test_e2e_parsed_tool_call(parser_client, parser_tokenizer):
""" extracted into tool_calls field."""
diff --git a/tests/entrypoints/scale_out/derender/test_derender_parity.py b/tests/entrypoints/scale_out/derender/test_derender_parity.py
index f331873874..e2de9433dc 100644
--- a/tests/entrypoints/scale_out/derender/test_derender_parity.py
+++ b/tests/entrypoints/scale_out/derender/test_derender_parity.py
@@ -316,13 +316,18 @@ async def test_parity_logprobs(client):
async def _coupled_stream(
- client: httpx.AsyncClient, messages: list[dict], **extra
-) -> tuple[list[dict], list[int]]:
+ client: httpx.AsyncClient,
+ messages: list[dict],
+ require_token_ids: bool = True,
+ **extra,
+) -> tuple[list[dict], list[int] | None]:
"""Stream `/v1/chat/completions` and return the choices that carry
`token_ids` along with the prompt token IDs.
The leading role only chunk has no `token_ids` and no counterpart on
- `/inference/v1/generate`, so it is dropped.
+ `/inference/v1/generate`, so it is dropped. With `require_token_ids`
+ off every choice is kept and `prompt_token_ids` may be `None`, for
+ requests where the coupled server hides token ids (hidden reasoning).
"""
choices: list[dict] = []
prompt_token_ids: list[int] | None = None
@@ -351,9 +356,12 @@ async def _coupled_stream(
if chunk.get("prompt_token_ids") is not None:
prompt_token_ids = chunk["prompt_token_ids"]
choices.extend(
- ch for ch in chunk["choices"] if ch.get("token_ids") is not None
+ ch
+ for ch in chunk["choices"]
+ if not require_token_ids or ch.get("token_ids") is not None
)
- assert prompt_token_ids is not None
+ if require_token_ids:
+ assert prompt_token_ids is not None
return choices, prompt_token_ids
@@ -473,6 +481,151 @@ async def test_stream_parity_reasoning_and_tool_call(client, chunking):
pytest.skip("Model did not emit both a block and a tool call")
+# ---------------------------------------------------------------------------
+# Streaming logprobs on the parser path
+# ---------------------------------------------------------------------------
+
+
+def _logprob_tokens(choice: dict) -> list[tuple[str, list[int] | None]] | None:
+ lp = choice.get("logprobs")
+ if lp is None or lp.get("content") is None:
+ return None
+ return [(e["token"], e.get("bytes")) for e in lp["content"]]
+
+
+def _sampled_generate_logprobs(choice: dict) -> dict | None:
+ """`GenerateLogProbs` for one coupled stream chunk from its `token_ids`
+ and per-token logprob values, without `top_logprobs`.
+
+ The coupled stream's `token_ids` are the integer ids a GPU-less worker
+ would have sent, so one stream is both the derender input and the string
+ reference. A second greedy run is not needed, and greedy decoding is not
+ bit-reproducible across calls on every platform anyway.
+ """
+ lp = choice.get("logprobs")
+ if lp is None or lp.get("content") is None:
+ return None
+ content = lp["content"]
+ assert len(content) == len(choice["token_ids"]), choice
+ return {
+ "content": [
+ {"token_id": tid, "logprob": entry["logprob"], "top_logprobs": []}
+ for tid, entry in zip(choice["token_ids"], content)
+ ]
+ }
+
+
+async def _run_stream_logprob_parity_case(
+ client: httpx.AsyncClient, messages: list[dict], **extra
+) -> tuple[dict, list[dict], list[dict]]:
+ """Stream with logprobs through the coupled server, replay the same
+ chunks through the streaming derender endpoint and compare per chunk.
+ Returns the assembled coupled message, the coupled choices and the
+ derender choices.
+ """
+ extra = {"logprobs": True, **extra}
+ coupled_choices, prompt_token_ids = await _coupled_stream(client, messages, **extra)
+ finish_reason = coupled_choices[-1]["finish_reason"]
+ derender_choices: list[dict] = []
+ disagg = await stream_chat_derender(
+ client,
+ [tid for ch in coupled_choices for tid in ch["token_ids"]],
+ [len(ch["token_ids"]) for ch in coupled_choices],
+ {"model": MODEL, "messages": messages, **extra},
+ len(prompt_token_ids),
+ prompt_token_ids,
+ finish_reason="stop" if finish_reason == "tool_calls" else finish_reason,
+ logprobs=[_sampled_generate_logprobs(ch) for ch in coupled_choices],
+ raw_choices=derender_choices,
+ )
+ coupled = assemble_stream(coupled_choices)
+ for message in (coupled, disagg):
+ for tc in message["tool_calls"]:
+ tc["id"] = _tool_call_id_shape(tc["id"])
+ assert disagg == coupled
+ assert len(derender_choices) == len(coupled_choices)
+ for ref, der in zip(coupled_choices, derender_choices):
+ assert _logprob_tokens(der) == _logprob_tokens(ref), (ref, der)
+ return coupled, coupled_choices, derender_choices
+
+
+@pytest.mark.asyncio
+async def test_stream_parity_logprobs_reasoning(client):
+ """Per-chunk logprobs on the parser path match the coupled stream across
+ the reasoning -> content transition."""
+ messages = [{"role": "user", "content": "What is 17 times 23? Think it through."}]
+ coupled, _, derender_choices = await _run_stream_logprob_parity_case(
+ client, messages, include_reasoning=True, max_tokens=256
+ )
+ if not coupled["reasoning"]:
+ pytest.skip("Model did not emit a block")
+ assert any(
+ ch["delta"].get("reasoning") and ch.get("logprobs") for ch in derender_choices
+ )
+
+
+@pytest.mark.asyncio
+async def test_stream_parity_logprobs_tool_call(client):
+ """Chunks the parser swallows (tool call markup) still carry their
+ logprobs, on both sides, so no token's logprobs are lost."""
+ messages = [{"role": "user", "content": "What's the weather in Paris?"}]
+ coupled, _, derender_choices = await _run_stream_logprob_parity_case(
+ client,
+ messages,
+ tools=TOOLS,
+ tool_choice=FORCE_WEATHER_TOOL,
+ include_reasoning=True,
+ max_tokens=512,
+ )
+ if not coupled["tool_calls"]:
+ pytest.skip("Model did not emit a tool call")
+ swallowed = [
+ ch
+ for ch in derender_choices
+ if ch.get("logprobs")
+ and not ch["delta"].get("content")
+ and not ch["delta"].get("reasoning")
+ and not ch["delta"].get("tool_calls")
+ ]
+ assert swallowed, "expected at least one buffered-markup chunk with logprobs"
+
+
+@pytest.mark.asyncio
+async def test_stream_parity_logprobs_hidden_reasoning(client):
+ """With `include_reasoning=False` and a parser configured, neither side
+ returns logprobs on any chunk; derender still emits the content."""
+ messages = [{"role": "user", "content": "What is 17 times 23? Think it through."}]
+ extra = {"logprobs": True, "max_tokens": 256}
+ # Token ids come from a shown-reasoning run: the coupled server hides
+ # them along with the logprobs when reasoning is hidden.
+ shown_choices, prompt_token_ids = await _coupled_stream(
+ client, messages, include_reasoning=True, **extra
+ )
+ hidden_choices, _ = await _coupled_stream(
+ client, messages, require_token_ids=False, include_reasoning=False, **extra
+ )
+ assert all(ch.get("logprobs") is None for ch in hidden_choices)
+ assert assemble_stream(hidden_choices)["reasoning"] is None
+
+ finish_reason = shown_choices[-1]["finish_reason"]
+ derender_choices: list[dict] = []
+ disagg = await stream_chat_derender(
+ client,
+ [tid for ch in shown_choices for tid in ch["token_ids"]],
+ [len(ch["token_ids"]) for ch in shown_choices],
+ {"model": MODEL, "messages": messages, "include_reasoning": False, **extra},
+ len(prompt_token_ids),
+ prompt_token_ids,
+ finish_reason="stop" if finish_reason == "tool_calls" else finish_reason,
+ logprobs=[_sampled_generate_logprobs(ch) for ch in shown_choices],
+ raw_choices=derender_choices,
+ )
+ assert all(ch.get("logprobs") is None for ch in derender_choices)
+ shown = assemble_stream(shown_choices)
+ assert disagg["reasoning"] is None
+ assert disagg["content"] == shown["content"]
+
+
# ---------------------------------------------------------------------------
# Inline text level: generate(output_mode="text") == derender(generate(...))
# ---------------------------------------------------------------------------
diff --git a/tests/entrypoints/scale_out/derender/test_derender_stream.py b/tests/entrypoints/scale_out/derender/test_derender_stream.py
index 372ef31ea3..56260964a0 100644
--- a/tests/entrypoints/scale_out/derender/test_derender_stream.py
+++ b/tests/entrypoints/scale_out/derender/test_derender_stream.py
@@ -1427,6 +1427,159 @@ class TestDerenderChatStreamParsed:
assert delta.reasoning is None
assert delta.content == "c"
+ @pytest.mark.asyncio
+ async def test_logprobs_attached_to_each_parsed_delta(
+ self, parsed_derenderer, tokenizer
+ ):
+ """A chunk's logprobs ride on whatever delta the parser emits for it
+ (reasoning, content or tool call), with token ids decoded, as on
+ `/v1/chat/completions`."""
+ chunks = [
+ [_FakeParser.REASON],
+ [_FakeParser.CONTENT],
+ [_FakeParser.TOOL_START, _FakeParser.TOOL_ARG],
+ ]
+ state = None
+ seen = []
+ for ids in chunks:
+ chunk, state = await parsed_derenderer.derender_chat_stream(
+ model=MODEL_NAME,
+ generate_chunk=_make_stream_chunk(
+ ids, logprobs=_generate_logprobs(ids)
+ ),
+ state=state,
+ chat_request=_chat_request(),
+ prompt_token_ids=[1, 2],
+ )
+ choice = chunk.choices[0]
+ assert choice.logprobs is not None and choice.logprobs.content is not None
+ assert [e.token for e in choice.logprobs.content] == (
+ convert_ids_list_to_tokens(tokenizer, ids)
+ )
+ seen.append(choice.delta)
+ assert seen[0].reasoning == "r"
+ assert seen[1].content == "c"
+ assert (
+ seen[2].tool_calls and seen[2].tool_calls[0].function.name == "get_weather"
+ )
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ ("request_kwargs", "expected_candidates"),
+ [({"logprobs": True, "top_logprobs": 2}, 2), ({}, 4)],
+ )
+ async def test_top_logprobs_follow_the_chat_cut(
+ self, parsed_derenderer, request_kwargs, expected_candidates
+ ):
+ """The parsed path trims generate's candidate list to the request's
+ `top_logprobs` like the plain path does, and keeps every candidate
+ when the request does not ask for logprobs."""
+ ids = [_FakeParser.CONTENT]
+ logprobs = _generate_logprobs(ids)
+ logprobs["content"][0]["top_logprobs"] = [
+ {"token_id": tid, "logprob": -float(rank)}
+ for rank, tid in enumerate([ids[0], 11, 12, 13])
+ ]
+ chunk, _ = await parsed_derenderer.derender_chat_stream(
+ model=MODEL_NAME,
+ generate_chunk=_make_stream_chunk(ids, logprobs=logprobs),
+ chat_request=_chat_request(**request_kwargs),
+ prompt_token_ids=[1, 2],
+ )
+ content = chunk.choices[0].logprobs.content
+ assert len(content) == 1
+ assert len(content[0].top_logprobs) == expected_candidates
+
+ @pytest.mark.asyncio
+ async def test_logprobs_kept_when_parser_swallows_chunk(
+ self, parsed_derenderer, tokenizer
+ ):
+ """A chunk the parser emits nothing for (buffered markup) still
+ carries its logprobs on an empty delta, so no token's logprobs are
+ lost. Mirrors the coupled endpoint, which emits a chunk whenever
+ logprobs are present even if the delta is empty."""
+ swallowed = [_FakeParser.CONTENT + 1] # unknown to _FakeParser -> None
+ chunk, _ = await parsed_derenderer.derender_chat_stream(
+ model=MODEL_NAME,
+ generate_chunk=_make_stream_chunk(
+ swallowed, logprobs=_generate_logprobs(swallowed)
+ ),
+ chat_request=_chat_request(),
+ prompt_token_ids=[1, 2],
+ )
+ choice = chunk.choices[0]
+ assert choice.delta.content is None and not choice.delta.tool_calls
+ assert [e.token for e in choice.logprobs.content] == (
+ convert_ids_list_to_tokens(tokenizer, swallowed)
+ )
+
+ @pytest.mark.asyncio
+ async def test_include_reasoning_false_drops_logprobs_on_every_chunk(
+ self, parsed_derenderer
+ ):
+ """Hidden reasoning suppresses logprobs on reasoning *and* content
+ chunks (decoded logprob tokens would leak the reasoning), the same
+ rule `/v1/chat/completions` applies with a parser configured."""
+ state = None
+ for ids in ([_FakeParser.REASON], [_FakeParser.CONTENT]):
+ chunk, state = await parsed_derenderer.derender_chat_stream(
+ model=MODEL_NAME,
+ generate_chunk=_make_stream_chunk(
+ ids, logprobs=_generate_logprobs(ids)
+ ),
+ state=state,
+ chat_request=_chat_request(include_reasoning=False),
+ prompt_token_ids=[1, 2],
+ )
+ assert chunk.choices[0].logprobs is None
+ assert chunk.choices[0].delta.content == "c"
+
+ @pytest.mark.asyncio
+ async def test_logprobs_multibyte_across_chunks(self, parsed_derenderer, tokenizer):
+ """Byte-fallback correction crosses chunk boundaries on the parser
+ path too, via the carried `logprob_context_token_ids`, which starts
+ empty like the engine's and never includes the prompt."""
+ token_ids = tokenizer.encode("👍", add_special_tokens=False)
+ if len(token_ids) < 2 or "�" not in tokenizer.decode([token_ids[-1]]):
+ pytest.skip("Tokenizer does not byte-split this character")
+ state = None
+ last_chunk = None
+ for tid in token_ids:
+ last_chunk, state = await parsed_derenderer.derender_chat_stream(
+ model=MODEL_NAME,
+ generate_chunk=_make_stream_chunk(
+ [tid], logprobs=_generate_logprobs([tid])
+ ),
+ state=state,
+ chat_request=_chat_request(),
+ prompt_token_ids=[1, 2, 3, 4, 5],
+ )
+ assert last_chunk is not None and state is not None
+ final_entry = last_chunk.choices[0].logprobs.content[0]
+ assert not final_entry.token.endswith("�"), final_entry.token
+ assert state.logprob_context_token_ids == token_ids[-4:]
+
+ @pytest.mark.asyncio
+ async def test_logprob_context_advances_without_logprobs(self, parsed_derenderer):
+ """The context tail tracks sampled ids whether or not the chunk
+ carries logprobs, so a later chunk that does can still be corrected."""
+ ids = [_FakeParser.REASON, _FakeParser.CONTENT, _FakeParser.CONTENT]
+ _, state = await parsed_derenderer.derender_chat_stream(
+ model=MODEL_NAME,
+ generate_chunk=_make_stream_chunk(ids),
+ chat_request=_chat_request(),
+ prompt_token_ids=[1, 2],
+ )
+ assert state.logprob_context_token_ids == ids
+ _, state = await parsed_derenderer.derender_chat_stream(
+ model=MODEL_NAME,
+ generate_chunk=_make_stream_chunk([], finish_reason="stop"),
+ state=state,
+ chat_request=_chat_request(),
+ prompt_token_ids=[1, 2],
+ )
+ assert state.logprob_context_token_ids == ids
+
# ---------------------------------------------------------------------------
# Harmony / GPT-OSS replay — unit, no server
diff --git a/tests/entrypoints/scale_out/derender/utils.py b/tests/entrypoints/scale_out/derender/utils.py
index c249e2053a..b6dcf2b668 100644
--- a/tests/entrypoints/scale_out/derender/utils.py
+++ b/tests/entrypoints/scale_out/derender/utils.py
@@ -52,6 +52,8 @@ async def stream_chat_derender(
prompt_token_ids: list[int],
finish_reason: str = "stop",
on_chunk: Callable[[list[dict]], None] | None = None,
+ logprobs: list[dict | None] | None = None,
+ raw_choices: list[dict] | None = None,
) -> dict:
"""Feed `output_ids` through the streaming chat derender endpoint in
the given `chunk_sizes`, threading `stream_state` across calls and
@@ -64,7 +66,13 @@ async def stream_chat_derender(
`tool_calls` assembled so far, letting callers assert properties of the
intermediate deltas (e.g. monotonic argument growth) rather than only
the final assembled result.
+
+ `logprobs` supplies one `GenerateLogProbs` payload (or `None`) per chunk
+ and `raw_choices`, if given, collects every derendered choice so callers
+ can compare per-chunk fields such as `logprobs`.
"""
+ if logprobs is not None:
+ assert len(logprobs) == len(chunk_sizes)
state = None
choices: list[dict] = []
pos = 0
@@ -84,6 +92,7 @@ async def stream_chat_derender(
"index": 0,
"token_ids": tids,
"finish_reason": finish_reason if is_last else None,
+ "logprobs": logprobs[i] if logprobs is not None else None,
}
],
},
@@ -101,6 +110,8 @@ async def stream_chat_derender(
assert data["chunk"]["choices"][0]["token_ids"] == tids
state = data["stream_state"]
choices.extend(data["chunk"]["choices"])
+ if raw_choices is not None:
+ raw_choices.extend(data["chunk"]["choices"])
if on_chunk is not None:
on_chunk(assemble_stream(choices)["tool_calls"])
diff --git a/vllm/renderers/online_derenderer.py b/vllm/renderers/online_derenderer.py
index 0c9412816a..3578c65c91 100644
--- a/vllm/renderers/online_derenderer.py
+++ b/vllm/renderers/online_derenderer.py
@@ -202,13 +202,21 @@ class OnlineDerenderer:
if has_parser and chat_request is not None
else {}
)
-
+ # Like the coupled chat endpoint, hide per-token metadata (logprobs,
+ # token ids) when a parser is configured and reasoning is hidden:
+ # either would otherwise leak the reasoning text.
+ hide_metadata = (
+ has_parser
+ and chat_request is not None
+ and not chat_request.include_reasoning
+ )
return_token_ids = (
chat_request is not None
and chat_request.return_token_ids
- and (not has_parser or chat_request.include_reasoning)
+ and not hide_metadata
)
top_limit = _chat_top_logprobs_limit(chat_request)
+
for choice in generate_response.choices:
if not choice.token_ids:
raise ValueError(f"choice {choice.index} has empty or null token_ids")
@@ -219,7 +227,7 @@ class OnlineDerenderer:
tokenizer,
top_limit=top_limit,
)
- if choice.logprobs is not None
+ if choice.logprobs is not None and not hide_metadata
else None
)
@@ -508,13 +516,6 @@ class OnlineDerenderer:
spaces_between_special_tokens=spaces_between,
)
- # NOTE: parser-configured servers dispatch to
- # _derender_chat_stream_parsed above and never reach this plain
- # path. That parsed path does not resolve logprobs yet; when it
- # does, mirror the generate chat streaming path, which suppresses
- # logprobs entirely when a parser is configured and reasoning is
- # hidden, because decoded logprob token text would leak hidden
- # reasoning.
resolved_logprobs = None
if choice.logprobs is not None:
resolved_logprobs = _resolve_logprobs(
@@ -595,8 +596,17 @@ class OnlineDerenderer:
Text comes from a fresh incremental detokenizer with special tokens
preserved (``skip_special_tokens=False``), seeded from the prompt
tail on every call since replay starts from scratch.
+
+ Logprobs need no replay: a generate chunk's ``logprobs`` cover exactly
+ that chunk's tokens, so they are resolved once and attached to
+ whatever delta the parser emits for the chunk, as the coupled chat
+ endpoint does. Only the byte-fallback context crosses chunks, via
+ ``state.logprob_context_token_ids``. With ``include_reasoning=False``
+ logprobs are dropped on every chunk, again like the coupled endpoint.
"""
tokenizer = self.renderer.get_tokenizer()
+ hide_metadata = not chat_request.include_reasoning
+ top_limit = _chat_top_logprobs_limit(chat_request)
parser = parser_cls(
tokenizer,
@@ -651,10 +661,9 @@ class OnlineDerenderer:
role_sent = state.role_sent
tools_streamed = state.tools_streamed
last_tool_call_ids = list(state.last_tool_call_ids)
+ logprob_context = list(state.logprob_context_token_ids)
- return_token_ids = (
- chat_request.return_token_ids and chat_request.include_reasoning
- )
+ return_token_ids = chat_request.return_token_ids and not hide_metadata
# At most one choice: the caller (derender_chat_stream) already
# rejects >1 before dispatching here. role_sent/tools_streamed/
@@ -707,6 +716,18 @@ class OnlineDerenderer:
output_token_ids.extend(delta_tids)
output_chunk_lens.append(len(delta_tids))
+ resolved_logprobs = None
+ if choice.logprobs is not None and not hide_metadata:
+ resolved_logprobs = _resolve_logprobs(
+ choice.logprobs,
+ tokenizer,
+ initial_context_token_ids=state.logprob_context_token_ids,
+ top_limit=top_limit,
+ )
+ logprob_context = _logprob_context_tail(
+ state.logprob_context_token_ids, delta_tids
+ )
+
if delta_message is None:
delta_message = DeltaMessage()
@@ -735,6 +756,7 @@ class OnlineDerenderer:
stream_choice = ChatCompletionResponseStreamChoice(
index=choice.index,
delta=delta_message,
+ logprobs=resolved_logprobs,
finish_reason=finish_reason,
stop_reason=choice.stop_reason,
token_ids=choice.token_ids if return_token_ids else None,
@@ -750,6 +772,7 @@ class OnlineDerenderer:
"role_sent": role_sent,
"tools_streamed": tools_streamed,
"last_tool_call_ids": last_tool_call_ids,
+ "logprob_context_token_ids": logprob_context,
}
)