[Frontend] Parse tool calls and reasoning from checkpoint response templates (#58604)

Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Yoni Gozlan <yonigozlan@users.noreply.github.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Yoni Gozlan
2026-10-02 02:33:00 +00:00
committed by GitHub
co-authored by Cursor Bugen Zhao Claude Opus 5.5
parent 5a661487e2
commit 9bd993564f
18 changed files with 2298 additions and 21 deletions
@@ -790,6 +790,64 @@ async def test_chat_per_request_metrics_suppressed_for_n_greater_than_one():
assert response.metrics is None
@pytest.mark.asyncio
async def test_chat_full_generator_gives_each_choice_parser_the_prompt():
prompts: list[list[int] | None] = []
class RecordingParser:
tool_parser_cls = None
def __init__(self, *args, **kwargs):
self.prompt_token_ids = None
def set_prompt_token_ids(self, prompt_token_ids):
self.prompt_token_ids = list(prompt_token_ids)
def parse(self, model_output, request, **kwargs):
prompts.append(self.prompt_token_ids)
return None, model_output, None
def count_reasoning_tokens(self, token_ids):
return 0
serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=False)
serving.parser_cls = RecordingParser
serving.model_config = None
serving.chat_template = None
serving.chat_template_content_format = "auto"
serving.default_chat_template_kwargs = {}
request_output = _make_metrics_request_output()
request_output.prompt_token_ids = [7, 8]
request_output.outputs.append(
CompletionOutput(
index=1,
text="Hi",
token_ids=[102],
cumulative_logprob=None,
logprobs=None,
finish_reason="stop",
)
)
await serving.chat_completion_full_generator(
ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "Test prompt"}],
max_tokens=10,
n=2,
),
_single_request_output(request_output),
"chatcmpl-test-id",
"test-model",
conversation=[{"role": "user", "content": "Test"}],
tokenizer=MagicMock(),
request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"),
parser=RecordingParser(),
)
assert prompts == [[7, 8], [7, 8]]
@pytest.mark.asyncio
async def test_chat_streaming_metrics_ride_on_usage_chunk():
serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=True)
@@ -1608,6 +1666,21 @@ async def test_serving_chat_truncation_side_controls_prompt_truncation():
assert left_token_ids == full_token_ids[-4:]
def test_serving_chat_starts_without_tokenizer():
"""Regression test: building the serving objects must not require a
tokenizer, which is absent under `skip_tokenizer_init`."""
mock_engine = _build_mock_engine()
mock_engine.model_config = MockModelConfig(skip_tokenizer_init=True)
mock_engine.renderer = HfRenderer(
MockVllmConfig(mock_engine.model_config, parallel_config=MockParallelConfig()),
None,
)
serving_chat = _build_serving_chat(mock_engine)
assert serving_chat.parser_cls is None
@pytest.mark.asyncio
async def test_serving_chat_mistral_token_ids_prompt_is_validated():
"""Regression test: when the Mistral tokenizer path returns token IDs