[Frontend] Parse tool calls and reasoning from checkpoint response templates (#58604)
Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Yoni Gozlan <yonigozlan@users.noreply.github.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Cursor
Bugen Zhao
Claude Opus 5.5
parent
5a661487e2
commit
9bd993564f
@@ -246,11 +246,15 @@ async def test_batched_chat_completions_logprob_token_ids(
|
||||
}
|
||||
|
||||
|
||||
def _make_request_output(prompt_idx: int, text: str) -> RequestOutput:
|
||||
def _make_request_output(
|
||||
prompt_idx: int,
|
||||
text: str,
|
||||
prompt_token_ids: list[int] | None = None,
|
||||
) -> RequestOutput:
|
||||
return RequestOutput(
|
||||
request_id=f"req-{prompt_idx}",
|
||||
prompt=None,
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
prompt_token_ids=prompt_token_ids or [1, 2, 3],
|
||||
prompt_logprobs=None,
|
||||
outputs=[
|
||||
CompletionOutput(
|
||||
@@ -266,8 +270,12 @@ def _make_request_output(prompt_idx: int, text: str) -> RequestOutput:
|
||||
)
|
||||
|
||||
|
||||
async def _generator(prompt_idx: int, text: str) -> AsyncGenerator[RequestOutput, None]:
|
||||
yield _make_request_output(prompt_idx, text)
|
||||
async def _generator(
|
||||
prompt_idx: int,
|
||||
text: str,
|
||||
prompt_token_ids: list[int] | None = None,
|
||||
) -> AsyncGenerator[RequestOutput, None]:
|
||||
yield _make_request_output(prompt_idx, text, prompt_token_ids)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -402,3 +410,47 @@ def test_batch_rejects_kv_transfer_prompt_token_ids():
|
||||
],
|
||||
kv_transfer_params={"prompt_token_ids": [10, 20, 30]},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip_global_cleanup
|
||||
async def test_batched_parser_receives_each_prompt_prefix() -> None:
|
||||
prefixes = []
|
||||
|
||||
class RecordingParser:
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.prompt_token_ids = None
|
||||
|
||||
def set_prompt_token_ids(self, prompt_token_ids):
|
||||
self.prompt_token_ids = prompt_token_ids
|
||||
|
||||
def parse(self, model_output, **kwargs):
|
||||
prefixes.append(self.prompt_token_ids)
|
||||
return None, model_output, None
|
||||
|
||||
serving = OpenAIServingChatBatch.__new__(OpenAIServingChatBatch)
|
||||
serving.response_role = "assistant"
|
||||
serving.system_fingerprint = None
|
||||
serving.chat_template = None
|
||||
serving.chat_template_content_format = "auto"
|
||||
serving.default_chat_template_kwargs = {}
|
||||
|
||||
request = BatchChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[
|
||||
[{"role": "user", "content": "first"}],
|
||||
[{"role": "user", "content": "second"}],
|
||||
],
|
||||
)
|
||||
await serving.chat_completion_full_generator_batch(
|
||||
request=request,
|
||||
generators=[_generator(0, "one", [1]), _generator(1, "two", [2])],
|
||||
request_id="req-prefix",
|
||||
model_name="test-model",
|
||||
all_conversations=request.messages,
|
||||
tokenizer=object(),
|
||||
request_metadata=RequestResponseMetadata(request_id="req-prefix"),
|
||||
parser_cls=RecordingParser,
|
||||
)
|
||||
|
||||
assert prefixes == [[1], [2]]
|
||||
|
||||
@@ -790,6 +790,64 @@ async def test_chat_per_request_metrics_suppressed_for_n_greater_than_one():
|
||||
assert response.metrics is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_full_generator_gives_each_choice_parser_the_prompt():
|
||||
prompts: list[list[int] | None] = []
|
||||
|
||||
class RecordingParser:
|
||||
tool_parser_cls = None
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.prompt_token_ids = None
|
||||
|
||||
def set_prompt_token_ids(self, prompt_token_ids):
|
||||
self.prompt_token_ids = list(prompt_token_ids)
|
||||
|
||||
def parse(self, model_output, request, **kwargs):
|
||||
prompts.append(self.prompt_token_ids)
|
||||
return None, model_output, None
|
||||
|
||||
def count_reasoning_tokens(self, token_ids):
|
||||
return 0
|
||||
|
||||
serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=False)
|
||||
serving.parser_cls = RecordingParser
|
||||
serving.model_config = None
|
||||
serving.chat_template = None
|
||||
serving.chat_template_content_format = "auto"
|
||||
serving.default_chat_template_kwargs = {}
|
||||
request_output = _make_metrics_request_output()
|
||||
request_output.prompt_token_ids = [7, 8]
|
||||
request_output.outputs.append(
|
||||
CompletionOutput(
|
||||
index=1,
|
||||
text="Hi",
|
||||
token_ids=[102],
|
||||
cumulative_logprob=None,
|
||||
logprobs=None,
|
||||
finish_reason="stop",
|
||||
)
|
||||
)
|
||||
|
||||
await serving.chat_completion_full_generator(
|
||||
ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[{"role": "user", "content": "Test prompt"}],
|
||||
max_tokens=10,
|
||||
n=2,
|
||||
),
|
||||
_single_request_output(request_output),
|
||||
"chatcmpl-test-id",
|
||||
"test-model",
|
||||
conversation=[{"role": "user", "content": "Test"}],
|
||||
tokenizer=MagicMock(),
|
||||
request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"),
|
||||
parser=RecordingParser(),
|
||||
)
|
||||
|
||||
assert prompts == [[7, 8], [7, 8]]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_streaming_metrics_ride_on_usage_chunk():
|
||||
serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=True)
|
||||
@@ -1608,6 +1666,21 @@ async def test_serving_chat_truncation_side_controls_prompt_truncation():
|
||||
assert left_token_ids == full_token_ids[-4:]
|
||||
|
||||
|
||||
def test_serving_chat_starts_without_tokenizer():
|
||||
"""Regression test: building the serving objects must not require a
|
||||
tokenizer, which is absent under `skip_tokenizer_init`."""
|
||||
mock_engine = _build_mock_engine()
|
||||
mock_engine.model_config = MockModelConfig(skip_tokenizer_init=True)
|
||||
mock_engine.renderer = HfRenderer(
|
||||
MockVllmConfig(mock_engine.model_config, parallel_config=MockParallelConfig()),
|
||||
None,
|
||||
)
|
||||
|
||||
serving_chat = _build_serving_chat(mock_engine)
|
||||
|
||||
assert serving_chat.parser_cls is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_serving_chat_mistral_token_ids_prompt_is_validated():
|
||||
"""Regression test: when the Mistral tokenizer path returns token IDs
|
||||
|
||||
@@ -296,6 +296,45 @@ def parsed_derenderer(tokenizer):
|
||||
return dr
|
||||
|
||||
|
||||
def test_non_streaming_derender_initializes_parser_prefix(
|
||||
derenderer, tokenizer, monkeypatch
|
||||
):
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import FunctionCall
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
|
||||
parser = MagicMock()
|
||||
parser.parse.return_value = (
|
||||
None,
|
||||
None,
|
||||
[FunctionCall(name="tool", arguments="{}")],
|
||||
)
|
||||
monkeypatch.setattr(derenderer, "parser", MagicMock(return_value=parser))
|
||||
generated_ids = tokenizer.encode("answer", add_special_tokens=False)
|
||||
response = GenerateResponse(
|
||||
request_id="test",
|
||||
prompt_token_ids=[11, 12],
|
||||
choices=[
|
||||
GenerateResponseChoice(
|
||||
index=0,
|
||||
token_ids=generated_ids,
|
||||
finish_reason="length",
|
||||
)
|
||||
],
|
||||
)
|
||||
request = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "question"}],
|
||||
)
|
||||
|
||||
choices = derenderer._derender_chat(response, request)
|
||||
|
||||
parser.set_prompt_token_ids.assert_called_once_with([11, 12])
|
||||
assert choices[0].message.tool_calls[0].function.name == "tool"
|
||||
assert choices[0].finish_reason == "length"
|
||||
|
||||
|
||||
class TestDetokenizeDelta:
|
||||
"""_detokenize_delta: chunked decode must equal one shot decode."""
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user