[Bugfix][Responses API] Complete cache-write token accounting (#60973)

Signed-off-by: Kay Yan <kay.yan@daocloud.io>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Kay Yan
2026-10-10 09:08:36 +00:00
committed by GitHub
co-authored by Codex
parent a98247ab4d
commit c41b2639e2
4 changed files with 34 additions and 2 deletions
+18 -1
View File
@@ -20,6 +20,7 @@ def create_mock_request_output(
output_token_ids=None,
num_cached_tokens=0,
finished=True,
num_cache_creation_tokens=None,
):
"""Helper function to create a mock RequestOutput object for testing."""
outputs = []
@@ -44,6 +45,7 @@ def create_mock_request_output(
outputs=outputs,
finished=finished,
num_cached_tokens=num_cached_tokens,
num_cache_creation_tokens=num_cache_creation_tokens,
)
@@ -117,7 +119,8 @@ def make_harmony_context(
return context, fake_parser
def test_single_turn_token_counting():
@pytest.mark.parametrize("num_cache_creation_tokens", [None, 0, 3])
def test_single_turn_token_counting(num_cache_creation_tokens):
"""Test token counting behavior for a single turn."""
# Create a context
context, _ = make_harmony_context()
@@ -127,6 +130,7 @@ def test_single_turn_token_counting():
prompt_token_ids=[1, 2, 3, 4, 5], # 5 prompt tokens
output_token_ids=[6, 7, 8], # 3 output tokens
num_cached_tokens=2, # 2 cached tokens
num_cache_creation_tokens=num_cache_creation_tokens,
)
# Append the output to the context
@@ -136,6 +140,7 @@ def test_single_turn_token_counting():
assert context.num_prompt_tokens == 5
assert context.num_output_tokens == 3
assert context.num_cached_tokens == 2
assert context.num_cache_creation_tokens == (num_cache_creation_tokens or 0)
assert context.num_tool_output_tokens == 0 # No tool tokens in first turn
# Verify internal state tracking
@@ -398,6 +403,7 @@ async def test_streaming_multi_turn_token_counting():
num_prompt_tokens = [3, 8, 13]
num_output_tokens = [3, 3, 2]
num_cached_tokens = [0, 3, 8]
num_cache_creation_tokens = [3, 4, 2]
# Simulate three turns of conversation:
# Turn 1: stream tokens one by one, then finish the message
@@ -411,6 +417,7 @@ async def test_streaming_multi_turn_token_counting():
prompt_token_ids=[1, 2, 3], # 3 prompt tokens
output_token_ids=[101], # Single token
num_cached_tokens=num_cached_tokens[0],
num_cache_creation_tokens=num_cache_creation_tokens[0],
finished=False, # Not end of message yet
)
)
@@ -419,6 +426,7 @@ async def test_streaming_multi_turn_token_counting():
context.append_output(
create_mock_request_output(
output_token_ids=[102],
num_cache_creation_tokens=num_cache_creation_tokens[0],
finished=False,
)
)
@@ -427,6 +435,7 @@ async def test_streaming_multi_turn_token_counting():
context.append_output(
create_mock_request_output(
output_token_ids=[103],
num_cache_creation_tokens=num_cache_creation_tokens[0],
finished=True, # End of message
)
)
@@ -435,6 +444,7 @@ async def test_streaming_multi_turn_token_counting():
assert context.num_prompt_tokens == 3 # Initial prompt tokens
assert context.num_output_tokens == 3 # Three output tokens
assert context.num_cached_tokens == 0
assert context.num_cache_creation_tokens == 3
assert context.num_tool_output_tokens == 0 # No tool output in first turn
assert context.first_tok_of_message is True # Ready for next message
@@ -454,6 +464,7 @@ async def test_streaming_multi_turn_token_counting():
], # 8 tokens (includes previous)
output_token_ids=[201],
num_cached_tokens=num_cached_tokens[1], # Some tokens cached
num_cache_creation_tokens=num_cache_creation_tokens[1],
finished=False,
)
)
@@ -463,6 +474,7 @@ async def test_streaming_multi_turn_token_counting():
context.append_output(
create_mock_request_output(
output_token_ids=[202],
num_cache_creation_tokens=num_cache_creation_tokens[1],
finished=False,
)
)
@@ -471,6 +483,7 @@ async def test_streaming_multi_turn_token_counting():
context.append_output(
create_mock_request_output(
output_token_ids=[203],
num_cache_creation_tokens=num_cache_creation_tokens[1],
finished=True, # End of reasoning message
)
)
@@ -480,6 +493,7 @@ async def test_streaming_multi_turn_token_counting():
assert context.num_output_tokens == 3 + 3 # First turn + second turn
assert context.num_reasoning_tokens == 3 # All tokens in analysis channel
assert context.num_cached_tokens == 3 # Cached tokens from second turn
assert context.num_cache_creation_tokens == 3 + 4
# Formula: this turn prompt tokens - last turn prompt - last turn output
expected_tool_tokens = 8 - 3 - 3 # = 2
@@ -505,6 +519,7 @@ async def test_streaming_multi_turn_token_counting():
], # 13 tokens
output_token_ids=[301],
num_cached_tokens=num_cached_tokens[2], # More cached tokens
num_cache_creation_tokens=num_cache_creation_tokens[2],
finished=False,
)
)
@@ -512,6 +527,7 @@ async def test_streaming_multi_turn_token_counting():
context.append_output(
create_mock_request_output(
output_token_ids=[302],
num_cache_creation_tokens=num_cache_creation_tokens[2],
finished=True,
)
)
@@ -523,6 +539,7 @@ async def test_streaming_multi_turn_token_counting():
assert context.num_cached_tokens == sum(
num_cached_tokens
) # Accumulated cached tokens
assert context.num_cache_creation_tokens == sum(num_cache_creation_tokens)
# Additional tool tokens from third turn
# Formula: this turn prompt - last turn prompt - last turn output