[ROCm][CI][Bugfix] Fix flaky parallel tool-call streaming (test assertion + Mistral/Granite parsers) (#47550)
Signed-off-by: Aakif Nawaz <aakif.nawaz@amd.com> Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Andreas Karatzas
parent
8484ca5d45
commit
5769a7382c
@@ -183,27 +183,39 @@ async def test_thinking_token_budget_mixed_requests(client: openai.AsyncOpenAI):
|
||||
async def test_thinking_token_budget_limits_reasoning(client: openai.AsyncOpenAI):
|
||||
"""Test that thinking_token_budget limits the number of reasoning tokens.
|
||||
|
||||
Counts non-empty streaming ``delta.reasoning`` chunks (coarse proxy; each
|
||||
chunk may represent multiple decode tokens — see
|
||||
``_count_reasoning_decode_token_ids_between_markers`` and the Qwen3.5 MTP
|
||||
test for id-based checks).
|
||||
Counts reasoning decode tokens by id, which is robust to how tokens are
|
||||
grouped into streamed chunks (a single chunk can carry several tokens under
|
||||
async scheduling / stream_interval > 1). Counting chunks under-counts.
|
||||
"""
|
||||
|
||||
reasoning_token_count = 0
|
||||
tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
|
||||
start_ids = list(tokenizer.encode(REASONING_START_STR, add_special_tokens=False))
|
||||
end_ids = list(tokenizer.encode(REASONING_END_STR, add_special_tokens=False))
|
||||
|
||||
prompt_token_ids: list[int] = []
|
||||
decode_token_ids: list[int] = []
|
||||
stream = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=MESSAGES,
|
||||
max_tokens=100,
|
||||
stream=True,
|
||||
extra_body={"thinking_token_budget": THINK_BUDGET},
|
||||
extra_body={"thinking_token_budget": THINK_BUDGET, "return_token_ids": True},
|
||||
)
|
||||
async for chunk in stream:
|
||||
delta = chunk.choices[0].delta
|
||||
if getattr(delta, "reasoning", None):
|
||||
reasoning_token_count += 1
|
||||
if not chunk.choices:
|
||||
continue
|
||||
if getattr(chunk, "prompt_token_ids", None):
|
||||
prompt_token_ids = list(chunk.prompt_token_ids)
|
||||
delta_ids = getattr(chunk.choices[0], "token_ids", None)
|
||||
if delta_ids:
|
||||
decode_token_ids.extend(delta_ids)
|
||||
|
||||
reasoning_token_count = _count_reasoning_decode_token_ids_between_markers(
|
||||
prompt_token_ids + decode_token_ids, start_ids, end_ids
|
||||
)
|
||||
assert reasoning_token_count is not None, "missing reasoning start marker in ids"
|
||||
assert reasoning_token_count == THINK_BUDGET, (
|
||||
f"reasoning tokens ({reasoning_token_count}) exceeded "
|
||||
f"reasoning tokens ({reasoning_token_count}) != "
|
||||
f"thinking_token_budget ({THINK_BUDGET})"
|
||||
)
|
||||
|
||||
|
||||
@@ -2,13 +2,21 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.tool_parsers.common_tests import (
|
||||
ToolParserTestConfig,
|
||||
ToolParserTests,
|
||||
)
|
||||
from tests.tool_parsers.utils import run_tool_extraction
|
||||
from tests.tool_parsers.utils import (
|
||||
run_tool_extraction,
|
||||
run_tool_extraction_streaming,
|
||||
split_string_into_token_deltas,
|
||||
)
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
from vllm.tool_parsers.granite_tool_parser import GraniteToolParser
|
||||
|
||||
|
||||
class TestGraniteToolParser(ToolParserTests):
|
||||
@@ -116,3 +124,38 @@ I'll get that information.""",
|
||||
f"Expected 1 tool call from string format, got {len(tool_calls)}"
|
||||
)
|
||||
assert tool_calls[0].function.name == "get_weather"
|
||||
|
||||
|
||||
# granite emits arguments before name and its own tokenizer (not gpt2) is used
|
||||
# here so the token boundaries match production; get_tokenizer only fetches the
|
||||
# small tokenizer files, not the model weights.
|
||||
@pytest.fixture(scope="module")
|
||||
def granite_tokenizer():
|
||||
return get_tokenizer(tokenizer_name="ibm-granite/granite-3.1-8b-instruct")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("chunk_size", [2, 3, 4, 5])
|
||||
def test_streaming_parallel_calls_batched_deltas(granite_tokenizer, chunk_size):
|
||||
"""A batched delta (multiple tokens) spanning the boundary between two
|
||||
parallel calls must not drop the first call's name. granite streams
|
||||
arguments before name, so the name only completes as the next call appears.
|
||||
"""
|
||||
parser = GraniteToolParser(granite_tokenizer)
|
||||
model_output = (
|
||||
'<|tool_call|> [{"arguments": {"city": "Tokyo"}, "name": "get_weather"}, '
|
||||
'{"arguments": {"timezone": "Asia/Tokyo"}, "name": "get_time"}]'
|
||||
)
|
||||
token_deltas = split_string_into_token_deltas(granite_tokenizer, model_output)
|
||||
batched = [
|
||||
"".join(token_deltas[i : i + chunk_size])
|
||||
for i in range(0, len(token_deltas), chunk_size)
|
||||
]
|
||||
reconstructor = run_tool_extraction_streaming(
|
||||
parser, batched, assert_one_tool_per_delta=False
|
||||
)
|
||||
names = [tc.function.name for tc in reconstructor.tool_calls]
|
||||
assert names == ["get_weather", "get_time"]
|
||||
# trailing args of the final call are flushed by the serving layer
|
||||
assert json.loads(reconstructor.tool_calls[0].function.arguments) == {
|
||||
"city": "Tokyo"
|
||||
}
|
||||
|
||||
@@ -144,6 +144,7 @@ def stream_delta_message_generator(
|
||||
mistral_tokenizer: TokenizerLike,
|
||||
model_output: str | None,
|
||||
tools: list[tuple[str, str]] | None,
|
||||
chunk_size: int = 1,
|
||||
) -> Generator[DeltaMessage, None, None]:
|
||||
if (
|
||||
isinstance(mistral_tokenizer, MistralTokenizer)
|
||||
@@ -182,15 +183,13 @@ def stream_delta_message_generator(
|
||||
previous_tokens = None
|
||||
prefix_offset = 0
|
||||
read_offset = 0
|
||||
pending_text = ""
|
||||
pending_token_ids: list[int] = []
|
||||
for i, delta_token in enumerate(all_token_ids):
|
||||
delta_token_ids = [delta_token]
|
||||
previous_token_ids = all_token_ids[:i]
|
||||
current_token_ids = all_token_ids[: i + 1]
|
||||
|
||||
(new_tokens, delta_text, new_prefix_offset, new_read_offset) = (
|
||||
detokenize_incrementally(
|
||||
tokenizer=mistral_tokenizer,
|
||||
all_input_ids=current_token_ids,
|
||||
all_input_ids=all_token_ids[: i + 1],
|
||||
prev_tokens=previous_tokens,
|
||||
prefix_offset=prefix_offset,
|
||||
read_offset=read_offset,
|
||||
@@ -198,27 +197,39 @@ def stream_delta_message_generator(
|
||||
spaces_between_special_tokens=True,
|
||||
)
|
||||
)
|
||||
previous_tokens = (
|
||||
previous_tokens + new_tokens if previous_tokens else new_tokens
|
||||
)
|
||||
prefix_offset = new_prefix_offset
|
||||
read_offset = new_read_offset
|
||||
|
||||
current_text = previous_text + delta_text
|
||||
# Buffer tokens so each streamed delta can carry ``chunk_size`` tokens,
|
||||
# reproducing the multi-token deltas produced by async scheduling /
|
||||
# stream_interval > 1.
|
||||
pending_text += delta_text
|
||||
pending_token_ids.append(delta_token)
|
||||
if len(pending_token_ids) < chunk_size and i != len(all_token_ids) - 1:
|
||||
continue
|
||||
|
||||
previous_token_ids = all_token_ids[: i + 1 - len(pending_token_ids)]
|
||||
current_token_ids = all_token_ids[: i + 1]
|
||||
current_text = previous_text + pending_text
|
||||
|
||||
delta_message = mistral_tool_parser.extract_tool_calls_streaming(
|
||||
previous_text,
|
||||
current_text,
|
||||
delta_text,
|
||||
pending_text,
|
||||
previous_token_ids,
|
||||
current_token_ids,
|
||||
delta_token_ids,
|
||||
pending_token_ids,
|
||||
request=_DUMMY_REQUEST,
|
||||
)
|
||||
if delta_message:
|
||||
yield delta_message
|
||||
|
||||
previous_text = current_text
|
||||
previous_tokens = (
|
||||
previous_tokens + new_tokens if previous_tokens else new_tokens
|
||||
)
|
||||
prefix_offset = new_prefix_offset
|
||||
read_offset = new_read_offset
|
||||
pending_text = ""
|
||||
pending_token_ids = []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -1572,3 +1583,39 @@ def test_grammar_from_tool_parser_set_by_adjust_request(
|
||||
request = _make_request()
|
||||
result = mistral_tool_parser.adjust_request(request)
|
||||
assert result._grammar_from_tool_parser is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("chunk_size", [2, 3, 4, 5])
|
||||
def test_streaming_pre_v11_parallel_calls_batched_deltas(
|
||||
mistral_pre_v11_tool_parser, mistral_pre_v11_tokenizer, chunk_size
|
||||
):
|
||||
"""A batched delta spanning the boundary between two parallel calls must
|
||||
keep them on distinct indices (the bug collapsed both onto index 0)."""
|
||||
model_output = (
|
||||
'[TOOL_CALLS] [{"name": "add", "arguments": {"a": 3.5, "b": 4}}, '
|
||||
'{"name": "get_current_weather", "arguments": '
|
||||
'{"city": "San Francisco", "state": "CA", "unit": "celsius"}}]'
|
||||
)
|
||||
names: list[str] = []
|
||||
args: list[str] = []
|
||||
idx = -1
|
||||
for delta_message in stream_delta_message_generator(
|
||||
mistral_pre_v11_tool_parser,
|
||||
mistral_pre_v11_tokenizer,
|
||||
model_output,
|
||||
tools=None,
|
||||
chunk_size=chunk_size,
|
||||
):
|
||||
for tool_call in delta_message.tool_calls or []:
|
||||
if tool_call.index != idx:
|
||||
idx = tool_call.index
|
||||
args.append("")
|
||||
if tool_call.function and tool_call.function.name:
|
||||
names.append(tool_call.function.name)
|
||||
if tool_call.function and tool_call.function.arguments:
|
||||
args[tool_call.index] += tool_call.function.arguments
|
||||
|
||||
assert names == ["add", "get_current_weather"]
|
||||
assert len(args) == 2
|
||||
# trailing args of the final call are flushed by the serving layer
|
||||
assert json.loads(args[0]) == {"a": 3.5, "b": 4}
|
||||
|
||||
@@ -115,14 +115,12 @@ async def test_parallel_tool_calls(
|
||||
assert not role_name or role_name == "assistant"
|
||||
role_name = "assistant"
|
||||
|
||||
# if a tool call is streamed make sure there's exactly one
|
||||
# (based on the request parameters
|
||||
# a chunk may carry >1 tool-call delta at a parallel-call boundary
|
||||
streamed_tool_calls = chunk.choices[0].delta.tool_calls
|
||||
|
||||
if streamed_tool_calls and len(streamed_tool_calls) > 0:
|
||||
# make sure only one diff is present - correct even for parallel
|
||||
assert len(streamed_tool_calls) == 1
|
||||
tool_call = streamed_tool_calls[0]
|
||||
for tool_call in streamed_tool_calls or []:
|
||||
# deltas arrive in non-decreasing index order
|
||||
assert tool_call.index >= tool_call_idx
|
||||
|
||||
# if a new tool is being called, set up empty arguments
|
||||
if tool_call.index != tool_call_idx:
|
||||
|
||||
@@ -154,9 +154,11 @@ class GraniteToolParser(ToolParser):
|
||||
current_tool_call: dict = tool_call_arr[self.current_tool_id]
|
||||
|
||||
delta = None
|
||||
# case: we are starting a new tool in the array
|
||||
# -> array has > 0 length AND length has moved past cursor
|
||||
if len(tool_call_arr) > self.current_tool_id + 1:
|
||||
# Only advance once the current tool name is streamed; granite
|
||||
# emits arguments before name, so advancing early would drop it.
|
||||
if len(tool_call_arr) > self.current_tool_id + 1 and (
|
||||
self.current_tool_id < 0 or self.current_tool_name_sent
|
||||
):
|
||||
# if we're moving on to a new call, first make sure we
|
||||
# haven't missed anything in the previous one that was
|
||||
# auto-generated due to JSON completions, but wasn't
|
||||
@@ -184,7 +186,7 @@ class GraniteToolParser(ToolParser):
|
||||
)
|
||||
|
||||
# re-set stuff pertaining to progress in the current tool
|
||||
self.current_tool_id = len(tool_call_arr) - 1
|
||||
self.current_tool_id += 1
|
||||
self.current_tool_name_sent = False
|
||||
self.streamed_args_for_tool.append("")
|
||||
logger.debug("starting on new tool %d", self.current_tool_id)
|
||||
|
||||
@@ -533,6 +533,7 @@ class MistralToolParser(ToolParser):
|
||||
|
||||
if prefix == "item" and event == "start_map":
|
||||
self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY
|
||||
self.starting_new_tool = True
|
||||
if prefix == "item" and event == "map_key" and value == "name":
|
||||
self.streaming_state = StreamingState.PARSING_NAME
|
||||
if prefix == "item.name" and event == "string":
|
||||
@@ -640,18 +641,10 @@ class MistralToolParser(ToolParser):
|
||||
|
||||
# Given the parsed text and the possible streaming state change,
|
||||
# let's add to the tool delta
|
||||
if (
|
||||
(streaming_state_before_parse != self.streaming_state)
|
||||
and streaming_state_before_parse
|
||||
in [StreamingState.WAITING_FOR_TOOL_START, StreamingState.TOOL_COMPLETE]
|
||||
and self.streaming_state
|
||||
not in [
|
||||
StreamingState.ALL_TOOLS_COMPLETE,
|
||||
StreamingState.TOOL_COMPLETE,
|
||||
StreamingState.WAITING_FOR_TOOL_START,
|
||||
]
|
||||
):
|
||||
# starting a new tool call
|
||||
# start_map is the authoritative new-tool signal and survives
|
||||
# batched deltas, unlike comparing pre/post streaming states
|
||||
if self.starting_new_tool:
|
||||
self.starting_new_tool = False
|
||||
if current_tool_call_modified:
|
||||
if self.current_tool_mistral_id is not None:
|
||||
current_tool_call.id = self.current_tool_mistral_id
|
||||
|
||||
Reference in New Issue
Block a user