Bound generation inputs before expensive work (#51447)

Signed-off-by: Clinton Thomas <1033162+KernelClint@users.noreply.github.com>
Co-authored-by: Lucas Bourtoule <35483370+dhalf@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Clinton Thomas
2026-08-11 22:03:50 +08:00
committed by GitHub
co-authored by Lucas Bourtoule mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 5b184f775a
commit b64a2708b0
11 changed files with 421 additions and 37 deletions
+15 -2
View File
@@ -199,7 +199,14 @@ def find_last_user_index(messages: List[Dict[str, Any]]) -> int:
# Message Rendering
# ============================================================
def render_message(index: int, messages: List[Dict[str, Any]], thinking_mode: str, drop_thinking: bool = True, reasoning_effort: Optional[str] = None) -> str:
def render_message(
index: int,
messages: List[Dict[str, Any]],
thinking_mode: str,
drop_thinking: bool = True,
reasoning_effort: Optional[str] = None,
last_user_idx: Optional[int] = None,
) -> str:
"""
Render a single message at the given index into its encoded string form.
@@ -213,6 +220,7 @@ def render_message(index: int, messages: List[Dict[str, Any]], thinking_mode: st
drop_thinking: Whether to drop reasoning content from earlier turns.
reasoning_effort: Reasoning effort level, one of "low", "high", "max".
None is treated as "low".
last_user_idx: Cached index of the last user/developer message.
Returns:
Encoded string for this message.
@@ -222,7 +230,9 @@ def render_message(index: int, messages: List[Dict[str, Any]], thinking_mode: st
prompt = ""
msg = messages[index]
last_user_idx = find_last_user_index(messages)
last_user_idx = (
find_last_user_index(messages) if last_user_idx is None else last_user_idx
)
role = msg.get("role")
content = msg.get("content")
@@ -544,6 +554,8 @@ def encode_messages(
num_to_render = len(messages)
context_len = len(context)
last_user_idx = find_last_user_index(full_messages)
for idx in range(num_to_render):
prompt += render_message(
idx + context_len,
@@ -551,6 +563,7 @@ def encode_messages(
thinking_mode=thinking_mode,
drop_thinking=effective_drop_thinking,
reasoning_effort=reasoning_effort,
last_user_idx=last_user_idx,
)
return prompt