[Bugfix][Frontend] Keep the parser's tool call IDs in batch derender (#61852)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
@@ -10,6 +10,7 @@ import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteLaunchRenderServer
|
||||
from vllm.entrypoints.generate.base.protocol import FunctionCall
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
GenerateTokensResponse,
|
||||
@@ -1490,19 +1491,12 @@ _HERMES_TOOL_CALL_TEXT = (
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tool_choice", "text", "engine_finish_reason", "expected_finish_reason"),
|
||||
[
|
||||
("auto", _HERMES_TOOL_CALL_TEXT, "stop", "tool_calls"),
|
||||
("auto", _HERMES_TOOL_CALL_TEXT, "length", "length"),
|
||||
("required", "Hello there", "stop", "stop"),
|
||||
],
|
||||
)
|
||||
def test_derender_chat_finish_reason_matches_chat_completion(
|
||||
tool_choice, text, engine_finish_reason, expected_finish_reason
|
||||
def _derender_hermes_chat(
|
||||
text: str,
|
||||
tool_choice: str = "auto",
|
||||
engine_finish_reason: str = "stop",
|
||||
parser_cls: type | None = None,
|
||||
):
|
||||
"""/derender applies the same finish_reason rule as the coupled chat
|
||||
path: ``tool_calls`` only for a real call at a natural stop."""
|
||||
tokenizer = get_tokenizer(MODEL_NAME)
|
||||
renderer = MagicMock()
|
||||
renderer.get_tokenizer.return_value = tokenizer
|
||||
@@ -1518,6 +1512,8 @@ def test_derender_chat_finish_reason_matches_chat_completion(
|
||||
enable_auto_tools=True,
|
||||
tool_parser="hermes",
|
||||
)
|
||||
if parser_cls is not None:
|
||||
derenderer.parser = parser_cls
|
||||
request = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "Weather in Paris?"}],
|
||||
@@ -1532,6 +1528,54 @@ def test_derender_chat_finish_reason_matches_chat_completion(
|
||||
)
|
||||
|
||||
(choice,) = derenderer._derender_chat(response, request)
|
||||
return choice
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tool_choice", "text", "engine_finish_reason", "expected_finish_reason"),
|
||||
[
|
||||
("auto", _HERMES_TOOL_CALL_TEXT, "stop", "tool_calls"),
|
||||
("auto", _HERMES_TOOL_CALL_TEXT, "length", "length"),
|
||||
("required", "Hello there", "stop", "stop"),
|
||||
],
|
||||
)
|
||||
def test_derender_chat_finish_reason_matches_chat_completion(
|
||||
tool_choice, text, engine_finish_reason, expected_finish_reason
|
||||
):
|
||||
"""/derender applies the same finish_reason rule as the coupled chat
|
||||
path: ``tool_calls`` only for a real call at a natural stop."""
|
||||
choice = _derender_hermes_chat(text, tool_choice, engine_finish_reason)
|
||||
|
||||
assert choice.finish_reason == expected_finish_reason
|
||||
assert bool(choice.message.tool_calls) == (text == _HERMES_TOOL_CALL_TEXT)
|
||||
|
||||
|
||||
def test_derender_chat_tool_call_id_matches_chat_completion():
|
||||
"""Batch /derender uses the coupled chat's default tool call ID format."""
|
||||
choice = _derender_hermes_chat(_HERMES_TOOL_CALL_TEXT)
|
||||
|
||||
(tool_call,) = choice.message.tool_calls
|
||||
assert tool_call.id.startswith("chatcmpl-tool-")
|
||||
|
||||
|
||||
def test_derender_chat_keeps_parser_tool_call_id():
|
||||
"""Batch /derender keeps the parser's tool call ID (Kimi K2 style
|
||||
``functions.<name>:<idx>``), since chat templates reference it in
|
||||
multi-turn history."""
|
||||
|
||||
class _KimiIdParser:
|
||||
def __init__(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
def parse(self, model_output, request, **kwargs):
|
||||
call = FunctionCall(
|
||||
id="functions.get_weather:1",
|
||||
name="get_weather",
|
||||
arguments='{"city": "Paris"}',
|
||||
)
|
||||
return None, None, [call]
|
||||
|
||||
choice = _derender_hermes_chat(_HERMES_TOOL_CALL_TEXT, parser_cls=_KimiIdParser)
|
||||
|
||||
(tool_call,) = choice.message.tool_calls
|
||||
assert tool_call.id == "functions.get_weather:1"
|
||||
|
||||
@@ -142,12 +142,28 @@ def _tool_sig(response_choice: dict) -> list[tuple[str, dict]]:
|
||||
]
|
||||
|
||||
|
||||
def _tool_id_shapes(response_choice: dict) -> list[str | None]:
|
||||
return [
|
||||
_tool_call_id_shape(tc["id"])
|
||||
for tc in (response_choice["message"].get("tool_calls") or [])
|
||||
]
|
||||
|
||||
|
||||
def _engine_finish_reason(coupled_finish_reason: str) -> str:
|
||||
"""Undo the chat level `"tool_calls"` rewrite of `"stop"` so derender
|
||||
gets what the engine reported and has to reapply the rewrite itself."""
|
||||
if coupled_finish_reason == "tool_calls":
|
||||
return "stop"
|
||||
return coupled_finish_reason
|
||||
|
||||
|
||||
def _assert_parity(coupled: dict, disagg: dict) -> None:
|
||||
"""Both paths saw the same tokens, so they must agree unconditionally."""
|
||||
c, d = coupled["choices"][0], disagg["choices"][0]
|
||||
assert d["message"]["content"] == c["message"]["content"]
|
||||
assert d["message"].get("reasoning") == c["message"].get("reasoning")
|
||||
assert _tool_sig(d) == _tool_sig(c)
|
||||
assert _tool_id_shapes(d) == _tool_id_shapes(c)
|
||||
assert d["finish_reason"] == c["finish_reason"]
|
||||
assert d["token_ids"] == c["token_ids"]
|
||||
assert disagg["usage"]["prompt_tokens"] == coupled["usage"]["prompt_tokens"]
|
||||
@@ -166,7 +182,7 @@ async def _run_parity_case(
|
||||
client,
|
||||
ch["token_ids"],
|
||||
coupled["usage"]["prompt_tokens"],
|
||||
ch["finish_reason"],
|
||||
_engine_finish_reason(ch["finish_reason"]),
|
||||
chat_request,
|
||||
)
|
||||
return coupled, disagg
|
||||
@@ -398,7 +414,7 @@ async def _run_stream_parity_case(
|
||||
{"model": MODEL, "messages": messages, "return_token_ids": True, **extra},
|
||||
len(prompt_token_ids),
|
||||
prompt_token_ids,
|
||||
finish_reason="stop" if finish_reason == "tool_calls" else finish_reason,
|
||||
finish_reason=_engine_finish_reason(finish_reason),
|
||||
)
|
||||
coupled = assemble_stream(coupled_choices)
|
||||
for message in (coupled, disagg):
|
||||
|
||||
@@ -47,14 +47,16 @@ from vllm.entrypoints.serve.utils.tool_calls_utils import (
|
||||
from vllm.logger import init_logger
|
||||
from vllm.parser import Parser, ParserManager
|
||||
from vllm.renderers import BaseRenderer
|
||||
from vllm.renderers.chat_utils import ChatTemplateContentFormatOption
|
||||
from vllm.renderers.chat_utils import (
|
||||
ChatTemplateContentFormatOption,
|
||||
make_tool_call_id,
|
||||
)
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tokenizers.detokenizer_utils import (
|
||||
convert_prompt_ids_to_tokens,
|
||||
detokenize_incrementally,
|
||||
get_leading_space_marker,
|
||||
)
|
||||
from vllm.utils import random_uuid
|
||||
from vllm.utils.async_utils import make_async
|
||||
from vllm.v1.engine.detokenizer import uses_fast_detokenizer
|
||||
|
||||
@@ -334,7 +336,10 @@ class OnlineDerenderer:
|
||||
reasoning = None
|
||||
|
||||
tc_items = (
|
||||
[ToolCall(id=random_uuid(), function=tc) for tc in tool_calls]
|
||||
[
|
||||
ToolCall(id=tc.id or make_tool_call_id(), function=tc)
|
||||
for tc in tool_calls
|
||||
]
|
||||
if tool_calls
|
||||
else []
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user