[Bugfix][Gemma4] Fix infinite loop and array boundary issues in tool parser (#41991)
Signed-off-by: David Oy <david.oy@baseten.co> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -135,6 +135,11 @@ class TestParseGemma4Args:
|
|||||||
result = _parse_gemma4_args('name:<|"|>test<|"|>,flag:', partial=True)
|
result = _parse_gemma4_args('name:<|"|>test<|"|>,flag:', partial=True)
|
||||||
assert result == {"name": "test"}
|
assert result == {"name": "test"}
|
||||||
|
|
||||||
|
@pytest.mark.timeout(5)
|
||||||
|
def test_malformed_partial_array(self):
|
||||||
|
result = _parse_gemma4_args(":[t:[]")
|
||||||
|
assert isinstance(result, dict)
|
||||||
|
|
||||||
|
|
||||||
class TestParseGemma4Array:
|
class TestParseGemma4Array:
|
||||||
def test_string_array(self):
|
def test_string_array(self):
|
||||||
@@ -149,6 +154,16 @@ class TestParseGemma4Array:
|
|||||||
result = _parse_gemma4_array("42,true,3.14")
|
result = _parse_gemma4_array("42,true,3.14")
|
||||||
assert result == [42, True, 3.14]
|
assert result == [42, True, 3.14]
|
||||||
|
|
||||||
|
@pytest.mark.timeout(5)
|
||||||
|
def test_string_element_with_closing_bracket(self):
|
||||||
|
result = _parse_gemma4_array('[<|"|>a]b<|"|>,<|"|>c<|"|>],<|"|>tail<|"|>')
|
||||||
|
assert result == [["a]b", "c"], "tail"]
|
||||||
|
|
||||||
|
@pytest.mark.timeout(5)
|
||||||
|
def test_stray_closing_bracket(self):
|
||||||
|
result = _parse_gemma4_array("42,]trailing")
|
||||||
|
assert result == [42]
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Non-streaming extraction tests
|
# Non-streaming extraction tests
|
||||||
|
|||||||
@@ -204,6 +204,13 @@ def _parse_gemma4_args(args_str: str, *, partial: bool = False) -> dict:
|
|||||||
# Value may be incomplete (e.g. partial boolean) —
|
# Value may be incomplete (e.g. partial boolean) —
|
||||||
# withhold to avoid type instability during streaming.
|
# withhold to avoid type instability during streaming.
|
||||||
break
|
break
|
||||||
|
if i == val_start:
|
||||||
|
logger.warning(
|
||||||
|
"Gemma4 args parser made no progress at position %d; "
|
||||||
|
"aborting on malformed input.",
|
||||||
|
i,
|
||||||
|
)
|
||||||
|
break
|
||||||
result[key] = _parse_gemma4_value(args_str[val_start:i])
|
result[key] = _parse_gemma4_value(args_str[val_start:i])
|
||||||
|
|
||||||
return result
|
return result
|
||||||
@@ -258,6 +265,11 @@ def _parse_gemma4_array(arr_str: str, *, partial: bool = False) -> list:
|
|||||||
sub_start = i + 1
|
sub_start = i + 1
|
||||||
i += 1
|
i += 1
|
||||||
while i < n and depth > 0:
|
while i < n and depth > 0:
|
||||||
|
if arr_str[i:].startswith(STRING_DELIM):
|
||||||
|
i += len(STRING_DELIM)
|
||||||
|
nd = arr_str.find(STRING_DELIM, i)
|
||||||
|
i = nd + len(STRING_DELIM) if nd != -1 else n
|
||||||
|
continue
|
||||||
if arr_str[i] == "[":
|
if arr_str[i] == "[":
|
||||||
depth += 1
|
depth += 1
|
||||||
elif arr_str[i] == "]":
|
elif arr_str[i] == "]":
|
||||||
@@ -275,6 +287,13 @@ def _parse_gemma4_array(arr_str: str, *, partial: bool = False) -> list:
|
|||||||
i += 1
|
i += 1
|
||||||
if partial and i >= n:
|
if partial and i >= n:
|
||||||
break
|
break
|
||||||
|
if i == val_start:
|
||||||
|
logger.warning(
|
||||||
|
"Gemma4 array parser made no progress at position %d; "
|
||||||
|
"aborting on malformed input.",
|
||||||
|
i,
|
||||||
|
)
|
||||||
|
break
|
||||||
items.append(_parse_gemma4_value(arr_str[val_start:i]))
|
items.append(_parse_gemma4_value(arr_str[val_start:i]))
|
||||||
|
|
||||||
return items
|
return items
|
||||||
|
|||||||
Reference in New Issue
Block a user