[ROCm][CI][Bugfix] Fix flaky parallel tool-call streaming (test assertion + Mistral/Granite parsers) (#47550)

Signed-off-by: Aakif Nawaz <aakif.nawaz@amd.com>
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
akii96
2026-07-06 19:06:19 -04:00
committed by GitHub
co-authored by Andreas Karatzas
parent 8484ca5d45
commit 5769a7382c
6 changed files with 141 additions and 46 deletions
@@ -183,27 +183,39 @@ async def test_thinking_token_budget_mixed_requests(client: openai.AsyncOpenAI):
async def test_thinking_token_budget_limits_reasoning(client: openai.AsyncOpenAI):
"""Test that thinking_token_budget limits the number of reasoning tokens.
Counts non-empty streaming ``delta.reasoning`` chunks (coarse proxy; each
chunk may represent multiple decode tokens — see
``_count_reasoning_decode_token_ids_between_markers`` and the Qwen3.5 MTP
test for id-based checks).
Counts reasoning decode tokens by id, which is robust to how tokens are
grouped into streamed chunks (a single chunk can carry several tokens under
async scheduling / stream_interval > 1). Counting chunks under-counts.
"""
reasoning_token_count = 0
tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
start_ids = list(tokenizer.encode(REASONING_START_STR, add_special_tokens=False))
end_ids = list(tokenizer.encode(REASONING_END_STR, add_special_tokens=False))
prompt_token_ids: list[int] = []
decode_token_ids: list[int] = []
stream = await client.chat.completions.create(
model=MODEL_NAME,
messages=MESSAGES,
max_tokens=100,
stream=True,
extra_body={"thinking_token_budget": THINK_BUDGET},
extra_body={"thinking_token_budget": THINK_BUDGET, "return_token_ids": True},
)
async for chunk in stream:
delta = chunk.choices[0].delta
if getattr(delta, "reasoning", None):
reasoning_token_count += 1
if not chunk.choices:
continue
if getattr(chunk, "prompt_token_ids", None):
prompt_token_ids = list(chunk.prompt_token_ids)
delta_ids = getattr(chunk.choices[0], "token_ids", None)
if delta_ids:
decode_token_ids.extend(delta_ids)
reasoning_token_count = _count_reasoning_decode_token_ids_between_markers(
prompt_token_ids + decode_token_ids, start_ids, end_ids
)
assert reasoning_token_count is not None, "missing reasoning start marker in ids"
assert reasoning_token_count == THINK_BUDGET, (
f"reasoning tokens ({reasoning_token_count}) exceeded "
f"reasoning tokens ({reasoning_token_count}) != "
f"thinking_token_budget ({THINK_BUDGET})"
)
+44 -1
View File
@@ -2,13 +2,21 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import pytest
from tests.tool_parsers.common_tests import (
ToolParserTestConfig,
ToolParserTests,
)
from tests.tool_parsers.utils import run_tool_extraction
from tests.tool_parsers.utils import (
run_tool_extraction,
run_tool_extraction_streaming,
split_string_into_token_deltas,
)
from vllm.tokenizers import get_tokenizer
from vllm.tool_parsers.granite_tool_parser import GraniteToolParser
class TestGraniteToolParser(ToolParserTests):
@@ -116,3 +124,38 @@ I'll get that information.""",
f"Expected 1 tool call from string format, got {len(tool_calls)}"
)
assert tool_calls[0].function.name == "get_weather"
# granite emits arguments before name and its own tokenizer (not gpt2) is used
# here so the token boundaries match production; get_tokenizer only fetches the
# small tokenizer files, not the model weights.
@pytest.fixture(scope="module")
def granite_tokenizer():
return get_tokenizer(tokenizer_name="ibm-granite/granite-3.1-8b-instruct")
@pytest.mark.parametrize("chunk_size", [2, 3, 4, 5])
def test_streaming_parallel_calls_batched_deltas(granite_tokenizer, chunk_size):
"""A batched delta (multiple tokens) spanning the boundary between two
parallel calls must not drop the first call's name. granite streams
arguments before name, so the name only completes as the next call appears.
"""
parser = GraniteToolParser(granite_tokenizer)
model_output = (
'<|tool_call|> [{"arguments": {"city": "Tokyo"}, "name": "get_weather"}, '
'{"arguments": {"timezone": "Asia/Tokyo"}, "name": "get_time"}]'
)
token_deltas = split_string_into_token_deltas(granite_tokenizer, model_output)
batched = [
"".join(token_deltas[i : i + chunk_size])
for i in range(0, len(token_deltas), chunk_size)
]
reconstructor = run_tool_extraction_streaming(
parser, batched, assert_one_tool_per_delta=False
)
names = [tc.function.name for tc in reconstructor.tool_calls]
assert names == ["get_weather", "get_time"]
# trailing args of the final call are flushed by the serving layer
assert json.loads(reconstructor.tool_calls[0].function.arguments) == {
"city": "Tokyo"
}
+60 -13
View File
@@ -144,6 +144,7 @@ def stream_delta_message_generator(
mistral_tokenizer: TokenizerLike,
model_output: str | None,
tools: list[tuple[str, str]] | None,
chunk_size: int = 1,
) -> Generator[DeltaMessage, None, None]:
if (
isinstance(mistral_tokenizer, MistralTokenizer)
@@ -182,15 +183,13 @@ def stream_delta_message_generator(
previous_tokens = None
prefix_offset = 0
read_offset = 0
pending_text = ""
pending_token_ids: list[int] = []
for i, delta_token in enumerate(all_token_ids):
delta_token_ids = [delta_token]
previous_token_ids = all_token_ids[:i]
current_token_ids = all_token_ids[: i + 1]
(new_tokens, delta_text, new_prefix_offset, new_read_offset) = (
detokenize_incrementally(
tokenizer=mistral_tokenizer,
all_input_ids=current_token_ids,
all_input_ids=all_token_ids[: i + 1],
prev_tokens=previous_tokens,
prefix_offset=prefix_offset,
read_offset=read_offset,
@@ -198,27 +197,39 @@ def stream_delta_message_generator(
spaces_between_special_tokens=True,
)
)
previous_tokens = (
previous_tokens + new_tokens if previous_tokens else new_tokens
)
prefix_offset = new_prefix_offset
read_offset = new_read_offset
current_text = previous_text + delta_text
# Buffer tokens so each streamed delta can carry ``chunk_size`` tokens,
# reproducing the multi-token deltas produced by async scheduling /
# stream_interval > 1.
pending_text += delta_text
pending_token_ids.append(delta_token)
if len(pending_token_ids) < chunk_size and i != len(all_token_ids) - 1:
continue
previous_token_ids = all_token_ids[: i + 1 - len(pending_token_ids)]
current_token_ids = all_token_ids[: i + 1]
current_text = previous_text + pending_text
delta_message = mistral_tool_parser.extract_tool_calls_streaming(
previous_text,
current_text,
delta_text,
pending_text,
previous_token_ids,
current_token_ids,
delta_token_ids,
pending_token_ids,
request=_DUMMY_REQUEST,
)
if delta_message:
yield delta_message
previous_text = current_text
previous_tokens = (
previous_tokens + new_tokens if previous_tokens else new_tokens
)
prefix_offset = new_prefix_offset
read_offset = new_read_offset
pending_text = ""
pending_token_ids = []
@pytest.mark.parametrize(
@@ -1572,3 +1583,39 @@ def test_grammar_from_tool_parser_set_by_adjust_request(
request = _make_request()
result = mistral_tool_parser.adjust_request(request)
assert result._grammar_from_tool_parser is True
@pytest.mark.parametrize("chunk_size", [2, 3, 4, 5])
def test_streaming_pre_v11_parallel_calls_batched_deltas(
mistral_pre_v11_tool_parser, mistral_pre_v11_tokenizer, chunk_size
):
"""A batched delta spanning the boundary between two parallel calls must
keep them on distinct indices (the bug collapsed both onto index 0)."""
model_output = (
'[TOOL_CALLS] [{"name": "add", "arguments": {"a": 3.5, "b": 4}}, '
'{"name": "get_current_weather", "arguments": '
'{"city": "San Francisco", "state": "CA", "unit": "celsius"}}]'
)
names: list[str] = []
args: list[str] = []
idx = -1
for delta_message in stream_delta_message_generator(
mistral_pre_v11_tool_parser,
mistral_pre_v11_tokenizer,
model_output,
tools=None,
chunk_size=chunk_size,
):
for tool_call in delta_message.tool_calls or []:
if tool_call.index != idx:
idx = tool_call.index
args.append("")
if tool_call.function and tool_call.function.name:
names.append(tool_call.function.name)
if tool_call.function and tool_call.function.arguments:
args[tool_call.index] += tool_call.function.arguments
assert names == ["add", "get_current_weather"]
assert len(args) == 2
# trailing args of the final call are flushed by the serving layer
assert json.loads(args[0]) == {"a": 3.5, "b": 4}
+4 -6
View File
@@ -115,14 +115,12 @@ async def test_parallel_tool_calls(
assert not role_name or role_name == "assistant"
role_name = "assistant"
# if a tool call is streamed make sure there's exactly one
# (based on the request parameters
# a chunk may carry >1 tool-call delta at a parallel-call boundary
streamed_tool_calls = chunk.choices[0].delta.tool_calls
if streamed_tool_calls and len(streamed_tool_calls) > 0:
# make sure only one diff is present - correct even for parallel
assert len(streamed_tool_calls) == 1
tool_call = streamed_tool_calls[0]
for tool_call in streamed_tool_calls or []:
# deltas arrive in non-decreasing index order
assert tool_call.index >= tool_call_idx
# if a new tool is being called, set up empty arguments
if tool_call.index != tool_call_idx:
+6 -4
View File
@@ -154,9 +154,11 @@ class GraniteToolParser(ToolParser):
current_tool_call: dict = tool_call_arr[self.current_tool_id]
delta = None
# case: we are starting a new tool in the array
# -> array has > 0 length AND length has moved past cursor
if len(tool_call_arr) > self.current_tool_id + 1:
# Only advance once the current tool name is streamed; granite
# emits arguments before name, so advancing early would drop it.
if len(tool_call_arr) > self.current_tool_id + 1 and (
self.current_tool_id < 0 or self.current_tool_name_sent
):
# if we're moving on to a new call, first make sure we
# haven't missed anything in the previous one that was
# auto-generated due to JSON completions, but wasn't
@@ -184,7 +186,7 @@ class GraniteToolParser(ToolParser):
)
# re-set stuff pertaining to progress in the current tool
self.current_tool_id = len(tool_call_arr) - 1
self.current_tool_id += 1
self.current_tool_name_sent = False
self.streamed_args_for_tool.append("")
logger.debug("starting on new tool %d", self.current_tool_id)
+5 -12
View File
@@ -533,6 +533,7 @@ class MistralToolParser(ToolParser):
if prefix == "item" and event == "start_map":
self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY
self.starting_new_tool = True
if prefix == "item" and event == "map_key" and value == "name":
self.streaming_state = StreamingState.PARSING_NAME
if prefix == "item.name" and event == "string":
@@ -640,18 +641,10 @@ class MistralToolParser(ToolParser):
# Given the parsed text and the possible streaming state change,
# let's add to the tool delta
if (
(streaming_state_before_parse != self.streaming_state)
and streaming_state_before_parse
in [StreamingState.WAITING_FOR_TOOL_START, StreamingState.TOOL_COMPLETE]
and self.streaming_state
not in [
StreamingState.ALL_TOOLS_COMPLETE,
StreamingState.TOOL_COMPLETE,
StreamingState.WAITING_FOR_TOOL_START,
]
):
# starting a new tool call
# start_map is the authoritative new-tool signal and survives
# batched deltas, unlike comparing pre/post streaming states
if self.starting_new_tool:
self.starting_new_tool = False
if current_tool_call_modified:
if self.current_tool_mistral_id is not None:
current_tool_call.id = self.current_tool_mistral_id