[Frontend] Parse tool calls and reasoning from checkpoint response templates (#58604)

Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Yoni Gozlan <yonigozlan@users.noreply.github.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Yoni Gozlan
2026-10-02 02:33:00 +00:00
committed by GitHub
co-authored by Cursor Bugen Zhao Claude Opus 5.5
parent 5a661487e2
commit 9bd993564f
18 changed files with 2298 additions and 21 deletions
@@ -246,11 +246,15 @@ async def test_batched_chat_completions_logprob_token_ids(
}
def _make_request_output(prompt_idx: int, text: str) -> RequestOutput:
def _make_request_output(
prompt_idx: int,
text: str,
prompt_token_ids: list[int] | None = None,
) -> RequestOutput:
return RequestOutput(
request_id=f"req-{prompt_idx}",
prompt=None,
prompt_token_ids=[1, 2, 3],
prompt_token_ids=prompt_token_ids or [1, 2, 3],
prompt_logprobs=None,
outputs=[
CompletionOutput(
@@ -266,8 +270,12 @@ def _make_request_output(prompt_idx: int, text: str) -> RequestOutput:
)
async def _generator(prompt_idx: int, text: str) -> AsyncGenerator[RequestOutput, None]:
yield _make_request_output(prompt_idx, text)
async def _generator(
prompt_idx: int,
text: str,
prompt_token_ids: list[int] | None = None,
) -> AsyncGenerator[RequestOutput, None]:
yield _make_request_output(prompt_idx, text, prompt_token_ids)
@pytest.mark.asyncio
@@ -402,3 +410,47 @@ def test_batch_rejects_kv_transfer_prompt_token_ids():
],
kv_transfer_params={"prompt_token_ids": [10, 20, 30]},
)
@pytest.mark.asyncio
@pytest.mark.skip_global_cleanup
async def test_batched_parser_receives_each_prompt_prefix() -> None:
prefixes = []
class RecordingParser:
def __init__(self, *args, **kwargs):
self.prompt_token_ids = None
def set_prompt_token_ids(self, prompt_token_ids):
self.prompt_token_ids = prompt_token_ids
def parse(self, model_output, **kwargs):
prefixes.append(self.prompt_token_ids)
return None, model_output, None
serving = OpenAIServingChatBatch.__new__(OpenAIServingChatBatch)
serving.response_role = "assistant"
serving.system_fingerprint = None
serving.chat_template = None
serving.chat_template_content_format = "auto"
serving.default_chat_template_kwargs = {}
request = BatchChatCompletionRequest(
model="test-model",
messages=[
[{"role": "user", "content": "first"}],
[{"role": "user", "content": "second"}],
],
)
await serving.chat_completion_full_generator_batch(
request=request,
generators=[_generator(0, "one", [1]), _generator(1, "two", [2])],
request_id="req-prefix",
model_name="test-model",
all_conversations=request.messages,
tokenizer=object(),
request_metadata=RequestResponseMetadata(request_id="req-prefix"),
parser_cls=RecordingParser,
)
assert prefixes == [[1], [2]]
@@ -790,6 +790,64 @@ async def test_chat_per_request_metrics_suppressed_for_n_greater_than_one():
assert response.metrics is None
@pytest.mark.asyncio
async def test_chat_full_generator_gives_each_choice_parser_the_prompt():
prompts: list[list[int] | None] = []
class RecordingParser:
tool_parser_cls = None
def __init__(self, *args, **kwargs):
self.prompt_token_ids = None
def set_prompt_token_ids(self, prompt_token_ids):
self.prompt_token_ids = list(prompt_token_ids)
def parse(self, model_output, request, **kwargs):
prompts.append(self.prompt_token_ids)
return None, model_output, None
def count_reasoning_tokens(self, token_ids):
return 0
serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=False)
serving.parser_cls = RecordingParser
serving.model_config = None
serving.chat_template = None
serving.chat_template_content_format = "auto"
serving.default_chat_template_kwargs = {}
request_output = _make_metrics_request_output()
request_output.prompt_token_ids = [7, 8]
request_output.outputs.append(
CompletionOutput(
index=1,
text="Hi",
token_ids=[102],
cumulative_logprob=None,
logprobs=None,
finish_reason="stop",
)
)
await serving.chat_completion_full_generator(
ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "Test prompt"}],
max_tokens=10,
n=2,
),
_single_request_output(request_output),
"chatcmpl-test-id",
"test-model",
conversation=[{"role": "user", "content": "Test"}],
tokenizer=MagicMock(),
request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"),
parser=RecordingParser(),
)
assert prompts == [[7, 8], [7, 8]]
@pytest.mark.asyncio
async def test_chat_streaming_metrics_ride_on_usage_chunk():
serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=True)
@@ -1608,6 +1666,21 @@ async def test_serving_chat_truncation_side_controls_prompt_truncation():
assert left_token_ids == full_token_ids[-4:]
def test_serving_chat_starts_without_tokenizer():
"""Regression test: building the serving objects must not require a
tokenizer, which is absent under `skip_tokenizer_init`."""
mock_engine = _build_mock_engine()
mock_engine.model_config = MockModelConfig(skip_tokenizer_init=True)
mock_engine.renderer = HfRenderer(
MockVllmConfig(mock_engine.model_config, parallel_config=MockParallelConfig()),
None,
)
serving_chat = _build_serving_chat(mock_engine)
assert serving_chat.parser_cls is None
@pytest.mark.asyncio
async def test_serving_chat_mistral_token_ids_prompt_is_validated():
"""Regression test: when the Mistral tokenizer path returns token IDs
@@ -296,6 +296,45 @@ def parsed_derenderer(tokenizer):
return dr
def test_non_streaming_derender_initializes_parser_prefix(
derenderer, tokenizer, monkeypatch
):
from unittest.mock import MagicMock
from vllm.entrypoints.generate.base.protocol import FunctionCall
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
parser = MagicMock()
parser.parse.return_value = (
None,
None,
[FunctionCall(name="tool", arguments="{}")],
)
monkeypatch.setattr(derenderer, "parser", MagicMock(return_value=parser))
generated_ids = tokenizer.encode("answer", add_special_tokens=False)
response = GenerateResponse(
request_id="test",
prompt_token_ids=[11, 12],
choices=[
GenerateResponseChoice(
index=0,
token_ids=generated_ids,
finish_reason="length",
)
],
)
request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "question"}],
)
choices = derenderer._derender_chat(response, request)
parser.set_prompt_token_ids.assert_called_once_with([11, 12])
assert choices[0].message.tool_calls[0].function.name == "tool"
assert choices[0].finish_reason == "length"
class TestDetokenizeDelta:
"""_detokenize_delta: chunked decode must equal one shot decode."""
File diff suppressed because it is too large Load Diff