[Bugfix][Frontend] Keep the parser's tool call IDs in batch derender (#61852)

Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
Martin Hickey
2026-10-16 10:27:17 +00:00
committed by GitHub
parent 4ab99ff136
commit 64175d3827
3 changed files with 82 additions and 17 deletions
@@ -10,6 +10,7 @@ import pytest
import pytest_asyncio
from tests.utils import RemoteLaunchRenderServer
from vllm.entrypoints.generate.base.protocol import FunctionCall
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
GenerateTokensResponse,
@@ -1490,19 +1491,12 @@ _HERMES_TOOL_CALL_TEXT = (
)
@pytest.mark.parametrize(
("tool_choice", "text", "engine_finish_reason", "expected_finish_reason"),
[
("auto", _HERMES_TOOL_CALL_TEXT, "stop", "tool_calls"),
("auto", _HERMES_TOOL_CALL_TEXT, "length", "length"),
("required", "Hello there", "stop", "stop"),
],
)
def test_derender_chat_finish_reason_matches_chat_completion(
tool_choice, text, engine_finish_reason, expected_finish_reason
def _derender_hermes_chat(
text: str,
tool_choice: str = "auto",
engine_finish_reason: str = "stop",
parser_cls: type | None = None,
):
"""/derender applies the same finish_reason rule as the coupled chat
path: ``tool_calls`` only for a real call at a natural stop."""
tokenizer = get_tokenizer(MODEL_NAME)
renderer = MagicMock()
renderer.get_tokenizer.return_value = tokenizer
@@ -1518,6 +1512,8 @@ def test_derender_chat_finish_reason_matches_chat_completion(
enable_auto_tools=True,
tool_parser="hermes",
)
if parser_cls is not None:
derenderer.parser = parser_cls
request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "Weather in Paris?"}],
@@ -1532,6 +1528,54 @@ def test_derender_chat_finish_reason_matches_chat_completion(
)
(choice,) = derenderer._derender_chat(response, request)
return choice
@pytest.mark.parametrize(
("tool_choice", "text", "engine_finish_reason", "expected_finish_reason"),
[
("auto", _HERMES_TOOL_CALL_TEXT, "stop", "tool_calls"),
("auto", _HERMES_TOOL_CALL_TEXT, "length", "length"),
("required", "Hello there", "stop", "stop"),
],
)
def test_derender_chat_finish_reason_matches_chat_completion(
tool_choice, text, engine_finish_reason, expected_finish_reason
):
"""/derender applies the same finish_reason rule as the coupled chat
path: ``tool_calls`` only for a real call at a natural stop."""
choice = _derender_hermes_chat(text, tool_choice, engine_finish_reason)
assert choice.finish_reason == expected_finish_reason
assert bool(choice.message.tool_calls) == (text == _HERMES_TOOL_CALL_TEXT)
def test_derender_chat_tool_call_id_matches_chat_completion():
"""Batch /derender uses the coupled chat's default tool call ID format."""
choice = _derender_hermes_chat(_HERMES_TOOL_CALL_TEXT)
(tool_call,) = choice.message.tool_calls
assert tool_call.id.startswith("chatcmpl-tool-")
def test_derender_chat_keeps_parser_tool_call_id():
"""Batch /derender keeps the parser's tool call ID (Kimi K2 style
``functions.<name>:<idx>``), since chat templates reference it in
multi-turn history."""
class _KimiIdParser:
def __init__(self, *args, **kwargs):
pass
def parse(self, model_output, request, **kwargs):
call = FunctionCall(
id="functions.get_weather:1",
name="get_weather",
arguments='{"city": "Paris"}',
)
return None, None, [call]
choice = _derender_hermes_chat(_HERMES_TOOL_CALL_TEXT, parser_cls=_KimiIdParser)
(tool_call,) = choice.message.tool_calls
assert tool_call.id == "functions.get_weather:1"
@@ -142,12 +142,28 @@ def _tool_sig(response_choice: dict) -> list[tuple[str, dict]]:
]
def _tool_id_shapes(response_choice: dict) -> list[str | None]:
return [
_tool_call_id_shape(tc["id"])
for tc in (response_choice["message"].get("tool_calls") or [])
]
def _engine_finish_reason(coupled_finish_reason: str) -> str:
"""Undo the chat level `"tool_calls"` rewrite of `"stop"` so derender
gets what the engine reported and has to reapply the rewrite itself."""
if coupled_finish_reason == "tool_calls":
return "stop"
return coupled_finish_reason
def _assert_parity(coupled: dict, disagg: dict) -> None:
"""Both paths saw the same tokens, so they must agree unconditionally."""
c, d = coupled["choices"][0], disagg["choices"][0]
assert d["message"]["content"] == c["message"]["content"]
assert d["message"].get("reasoning") == c["message"].get("reasoning")
assert _tool_sig(d) == _tool_sig(c)
assert _tool_id_shapes(d) == _tool_id_shapes(c)
assert d["finish_reason"] == c["finish_reason"]
assert d["token_ids"] == c["token_ids"]
assert disagg["usage"]["prompt_tokens"] == coupled["usage"]["prompt_tokens"]
@@ -166,7 +182,7 @@ async def _run_parity_case(
client,
ch["token_ids"],
coupled["usage"]["prompt_tokens"],
ch["finish_reason"],
_engine_finish_reason(ch["finish_reason"]),
chat_request,
)
return coupled, disagg
@@ -398,7 +414,7 @@ async def _run_stream_parity_case(
{"model": MODEL, "messages": messages, "return_token_ids": True, **extra},
len(prompt_token_ids),
prompt_token_ids,
finish_reason="stop" if finish_reason == "tool_calls" else finish_reason,
finish_reason=_engine_finish_reason(finish_reason),
)
coupled = assemble_stream(coupled_choices)
for message in (coupled, disagg):
+8 -3
View File
@@ -47,14 +47,16 @@ from vllm.entrypoints.serve.utils.tool_calls_utils import (
from vllm.logger import init_logger
from vllm.parser import Parser, ParserManager
from vllm.renderers import BaseRenderer
from vllm.renderers.chat_utils import ChatTemplateContentFormatOption
from vllm.renderers.chat_utils import (
ChatTemplateContentFormatOption,
make_tool_call_id,
)
from vllm.tokenizers import TokenizerLike
from vllm.tokenizers.detokenizer_utils import (
convert_prompt_ids_to_tokens,
detokenize_incrementally,
get_leading_space_marker,
)
from vllm.utils import random_uuid
from vllm.utils.async_utils import make_async
from vllm.v1.engine.detokenizer import uses_fast_detokenizer
@@ -334,7 +336,10 @@ class OnlineDerenderer:
reasoning = None
tc_items = (
[ToolCall(id=random_uuid(), function=tc) for tc in tool_calls]
[
ToolCall(id=tc.id or make_tool_call_id(), function=tc)
for tc in tool_calls
]
if tool_calls
else []
)