From 5769a7382cb111288a9c2927342ca908943b9793 Mon Sep 17 00:00:00 2001 From: akii96 Date: Tue, 7 Jul 2026 02:06:19 +0300 Subject: [PATCH] [ROCm][CI][Bugfix] Fix flaky parallel tool-call streaming (test assertion + Mistral/Granite parsers) (#47550) Signed-off-by: Aakif Nawaz Signed-off-by: Andreas Karatzas Co-authored-by: Andreas Karatzas --- .../test_thinking_token_budget.py | 32 +++++--- .../tool_parsers/test_granite_tool_parser.py | 45 +++++++++++- .../tool_parsers/test_mistral_tool_parser.py | 73 +++++++++++++++---- tests/tool_use/test_parallel_tool_calls.py | 10 +-- vllm/tool_parsers/granite_tool_parser.py | 10 ++- vllm/tool_parsers/mistral_tool_parser.py | 17 ++--- 6 files changed, 141 insertions(+), 46 deletions(-) diff --git a/tests/entrypoints/openai/chat_completion/test_thinking_token_budget.py b/tests/entrypoints/openai/chat_completion/test_thinking_token_budget.py index 3e9e4850f0..4c57467381 100644 --- a/tests/entrypoints/openai/chat_completion/test_thinking_token_budget.py +++ b/tests/entrypoints/openai/chat_completion/test_thinking_token_budget.py @@ -183,27 +183,39 @@ async def test_thinking_token_budget_mixed_requests(client: openai.AsyncOpenAI): async def test_thinking_token_budget_limits_reasoning(client: openai.AsyncOpenAI): """Test that thinking_token_budget limits the number of reasoning tokens. - Counts non-empty streaming ``delta.reasoning`` chunks (coarse proxy; each - chunk may represent multiple decode tokens — see - ``_count_reasoning_decode_token_ids_between_markers`` and the Qwen3.5 MTP - test for id-based checks). + Counts reasoning decode tokens by id, which is robust to how tokens are + grouped into streamed chunks (a single chunk can carry several tokens under + async scheduling / stream_interval > 1). Counting chunks under-counts. """ - reasoning_token_count = 0 + tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME) + start_ids = list(tokenizer.encode(REASONING_START_STR, add_special_tokens=False)) + end_ids = list(tokenizer.encode(REASONING_END_STR, add_special_tokens=False)) + + prompt_token_ids: list[int] = [] + decode_token_ids: list[int] = [] stream = await client.chat.completions.create( model=MODEL_NAME, messages=MESSAGES, max_tokens=100, stream=True, - extra_body={"thinking_token_budget": THINK_BUDGET}, + extra_body={"thinking_token_budget": THINK_BUDGET, "return_token_ids": True}, ) async for chunk in stream: - delta = chunk.choices[0].delta - if getattr(delta, "reasoning", None): - reasoning_token_count += 1 + if not chunk.choices: + continue + if getattr(chunk, "prompt_token_ids", None): + prompt_token_ids = list(chunk.prompt_token_ids) + delta_ids = getattr(chunk.choices[0], "token_ids", None) + if delta_ids: + decode_token_ids.extend(delta_ids) + reasoning_token_count = _count_reasoning_decode_token_ids_between_markers( + prompt_token_ids + decode_token_ids, start_ids, end_ids + ) + assert reasoning_token_count is not None, "missing reasoning start marker in ids" assert reasoning_token_count == THINK_BUDGET, ( - f"reasoning tokens ({reasoning_token_count}) exceeded " + f"reasoning tokens ({reasoning_token_count}) != " f"thinking_token_budget ({THINK_BUDGET})" ) diff --git a/tests/tool_parsers/test_granite_tool_parser.py b/tests/tool_parsers/test_granite_tool_parser.py index 2046c11c5d..af3386112f 100644 --- a/tests/tool_parsers/test_granite_tool_parser.py +++ b/tests/tool_parsers/test_granite_tool_parser.py @@ -2,13 +2,21 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import json + import pytest from tests.tool_parsers.common_tests import ( ToolParserTestConfig, ToolParserTests, ) -from tests.tool_parsers.utils import run_tool_extraction +from tests.tool_parsers.utils import ( + run_tool_extraction, + run_tool_extraction_streaming, + split_string_into_token_deltas, +) +from vllm.tokenizers import get_tokenizer +from vllm.tool_parsers.granite_tool_parser import GraniteToolParser class TestGraniteToolParser(ToolParserTests): @@ -116,3 +124,38 @@ I'll get that information.""", f"Expected 1 tool call from string format, got {len(tool_calls)}" ) assert tool_calls[0].function.name == "get_weather" + + +# granite emits arguments before name and its own tokenizer (not gpt2) is used +# here so the token boundaries match production; get_tokenizer only fetches the +# small tokenizer files, not the model weights. +@pytest.fixture(scope="module") +def granite_tokenizer(): + return get_tokenizer(tokenizer_name="ibm-granite/granite-3.1-8b-instruct") + + +@pytest.mark.parametrize("chunk_size", [2, 3, 4, 5]) +def test_streaming_parallel_calls_batched_deltas(granite_tokenizer, chunk_size): + """A batched delta (multiple tokens) spanning the boundary between two + parallel calls must not drop the first call's name. granite streams + arguments before name, so the name only completes as the next call appears. + """ + parser = GraniteToolParser(granite_tokenizer) + model_output = ( + '<|tool_call|> [{"arguments": {"city": "Tokyo"}, "name": "get_weather"}, ' + '{"arguments": {"timezone": "Asia/Tokyo"}, "name": "get_time"}]' + ) + token_deltas = split_string_into_token_deltas(granite_tokenizer, model_output) + batched = [ + "".join(token_deltas[i : i + chunk_size]) + for i in range(0, len(token_deltas), chunk_size) + ] + reconstructor = run_tool_extraction_streaming( + parser, batched, assert_one_tool_per_delta=False + ) + names = [tc.function.name for tc in reconstructor.tool_calls] + assert names == ["get_weather", "get_time"] + # trailing args of the final call are flushed by the serving layer + assert json.loads(reconstructor.tool_calls[0].function.arguments) == { + "city": "Tokyo" + } diff --git a/tests/tool_parsers/test_mistral_tool_parser.py b/tests/tool_parsers/test_mistral_tool_parser.py index 03a10ef099..4f75e2f576 100644 --- a/tests/tool_parsers/test_mistral_tool_parser.py +++ b/tests/tool_parsers/test_mistral_tool_parser.py @@ -144,6 +144,7 @@ def stream_delta_message_generator( mistral_tokenizer: TokenizerLike, model_output: str | None, tools: list[tuple[str, str]] | None, + chunk_size: int = 1, ) -> Generator[DeltaMessage, None, None]: if ( isinstance(mistral_tokenizer, MistralTokenizer) @@ -182,15 +183,13 @@ def stream_delta_message_generator( previous_tokens = None prefix_offset = 0 read_offset = 0 + pending_text = "" + pending_token_ids: list[int] = [] for i, delta_token in enumerate(all_token_ids): - delta_token_ids = [delta_token] - previous_token_ids = all_token_ids[:i] - current_token_ids = all_token_ids[: i + 1] - (new_tokens, delta_text, new_prefix_offset, new_read_offset) = ( detokenize_incrementally( tokenizer=mistral_tokenizer, - all_input_ids=current_token_ids, + all_input_ids=all_token_ids[: i + 1], prev_tokens=previous_tokens, prefix_offset=prefix_offset, read_offset=read_offset, @@ -198,27 +197,39 @@ def stream_delta_message_generator( spaces_between_special_tokens=True, ) ) + previous_tokens = ( + previous_tokens + new_tokens if previous_tokens else new_tokens + ) + prefix_offset = new_prefix_offset + read_offset = new_read_offset - current_text = previous_text + delta_text + # Buffer tokens so each streamed delta can carry ``chunk_size`` tokens, + # reproducing the multi-token deltas produced by async scheduling / + # stream_interval > 1. + pending_text += delta_text + pending_token_ids.append(delta_token) + if len(pending_token_ids) < chunk_size and i != len(all_token_ids) - 1: + continue + + previous_token_ids = all_token_ids[: i + 1 - len(pending_token_ids)] + current_token_ids = all_token_ids[: i + 1] + current_text = previous_text + pending_text delta_message = mistral_tool_parser.extract_tool_calls_streaming( previous_text, current_text, - delta_text, + pending_text, previous_token_ids, current_token_ids, - delta_token_ids, + pending_token_ids, request=_DUMMY_REQUEST, ) if delta_message: yield delta_message previous_text = current_text - previous_tokens = ( - previous_tokens + new_tokens if previous_tokens else new_tokens - ) - prefix_offset = new_prefix_offset - read_offset = new_read_offset + pending_text = "" + pending_token_ids = [] @pytest.mark.parametrize( @@ -1572,3 +1583,39 @@ def test_grammar_from_tool_parser_set_by_adjust_request( request = _make_request() result = mistral_tool_parser.adjust_request(request) assert result._grammar_from_tool_parser is True + + +@pytest.mark.parametrize("chunk_size", [2, 3, 4, 5]) +def test_streaming_pre_v11_parallel_calls_batched_deltas( + mistral_pre_v11_tool_parser, mistral_pre_v11_tokenizer, chunk_size +): + """A batched delta spanning the boundary between two parallel calls must + keep them on distinct indices (the bug collapsed both onto index 0).""" + model_output = ( + '[TOOL_CALLS] [{"name": "add", "arguments": {"a": 3.5, "b": 4}}, ' + '{"name": "get_current_weather", "arguments": ' + '{"city": "San Francisco", "state": "CA", "unit": "celsius"}}]' + ) + names: list[str] = [] + args: list[str] = [] + idx = -1 + for delta_message in stream_delta_message_generator( + mistral_pre_v11_tool_parser, + mistral_pre_v11_tokenizer, + model_output, + tools=None, + chunk_size=chunk_size, + ): + for tool_call in delta_message.tool_calls or []: + if tool_call.index != idx: + idx = tool_call.index + args.append("") + if tool_call.function and tool_call.function.name: + names.append(tool_call.function.name) + if tool_call.function and tool_call.function.arguments: + args[tool_call.index] += tool_call.function.arguments + + assert names == ["add", "get_current_weather"] + assert len(args) == 2 + # trailing args of the final call are flushed by the serving layer + assert json.loads(args[0]) == {"a": 3.5, "b": 4} diff --git a/tests/tool_use/test_parallel_tool_calls.py b/tests/tool_use/test_parallel_tool_calls.py index 0f7f689316..4cfd165f1a 100644 --- a/tests/tool_use/test_parallel_tool_calls.py +++ b/tests/tool_use/test_parallel_tool_calls.py @@ -115,14 +115,12 @@ async def test_parallel_tool_calls( assert not role_name or role_name == "assistant" role_name = "assistant" - # if a tool call is streamed make sure there's exactly one - # (based on the request parameters + # a chunk may carry >1 tool-call delta at a parallel-call boundary streamed_tool_calls = chunk.choices[0].delta.tool_calls - if streamed_tool_calls and len(streamed_tool_calls) > 0: - # make sure only one diff is present - correct even for parallel - assert len(streamed_tool_calls) == 1 - tool_call = streamed_tool_calls[0] + for tool_call in streamed_tool_calls or []: + # deltas arrive in non-decreasing index order + assert tool_call.index >= tool_call_idx # if a new tool is being called, set up empty arguments if tool_call.index != tool_call_idx: diff --git a/vllm/tool_parsers/granite_tool_parser.py b/vllm/tool_parsers/granite_tool_parser.py index d586db3267..174e288427 100644 --- a/vllm/tool_parsers/granite_tool_parser.py +++ b/vllm/tool_parsers/granite_tool_parser.py @@ -154,9 +154,11 @@ class GraniteToolParser(ToolParser): current_tool_call: dict = tool_call_arr[self.current_tool_id] delta = None - # case: we are starting a new tool in the array - # -> array has > 0 length AND length has moved past cursor - if len(tool_call_arr) > self.current_tool_id + 1: + # Only advance once the current tool name is streamed; granite + # emits arguments before name, so advancing early would drop it. + if len(tool_call_arr) > self.current_tool_id + 1 and ( + self.current_tool_id < 0 or self.current_tool_name_sent + ): # if we're moving on to a new call, first make sure we # haven't missed anything in the previous one that was # auto-generated due to JSON completions, but wasn't @@ -184,7 +186,7 @@ class GraniteToolParser(ToolParser): ) # re-set stuff pertaining to progress in the current tool - self.current_tool_id = len(tool_call_arr) - 1 + self.current_tool_id += 1 self.current_tool_name_sent = False self.streamed_args_for_tool.append("") logger.debug("starting on new tool %d", self.current_tool_id) diff --git a/vllm/tool_parsers/mistral_tool_parser.py b/vllm/tool_parsers/mistral_tool_parser.py index 1d605557b1..026098a873 100644 --- a/vllm/tool_parsers/mistral_tool_parser.py +++ b/vllm/tool_parsers/mistral_tool_parser.py @@ -533,6 +533,7 @@ class MistralToolParser(ToolParser): if prefix == "item" and event == "start_map": self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY + self.starting_new_tool = True if prefix == "item" and event == "map_key" and value == "name": self.streaming_state = StreamingState.PARSING_NAME if prefix == "item.name" and event == "string": @@ -640,18 +641,10 @@ class MistralToolParser(ToolParser): # Given the parsed text and the possible streaming state change, # let's add to the tool delta - if ( - (streaming_state_before_parse != self.streaming_state) - and streaming_state_before_parse - in [StreamingState.WAITING_FOR_TOOL_START, StreamingState.TOOL_COMPLETE] - and self.streaming_state - not in [ - StreamingState.ALL_TOOLS_COMPLETE, - StreamingState.TOOL_COMPLETE, - StreamingState.WAITING_FOR_TOOL_START, - ] - ): - # starting a new tool call + # start_map is the authoritative new-tool signal and survives + # batched deltas, unlike comparing pre/post streaming states + if self.starting_new_tool: + self.starting_new_tool = False if current_tool_call_modified: if self.current_tool_mistral_id is not None: current_tool_call.id = self.current_tool_mistral_id