[Frontend] Preserve bare Inkling text in Python and Rust parsers (#50403)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -61,7 +61,7 @@ enum InklingEvent {
|
||||
TextStart,
|
||||
ReasoningStart,
|
||||
MessageStart,
|
||||
Header,
|
||||
Header { len: usize },
|
||||
ToolJsonStart,
|
||||
ToolJsonHeader { name: String },
|
||||
ToolJsonArgs { len: usize, complete: bool },
|
||||
@@ -72,7 +72,9 @@ enum InklingEvent {
|
||||
enum InklingMode {
|
||||
#[default]
|
||||
Idle,
|
||||
MessageHeader,
|
||||
MessageHeader {
|
||||
pending: String,
|
||||
},
|
||||
Text,
|
||||
Reasoning,
|
||||
ToolJsonHeader,
|
||||
@@ -117,7 +119,9 @@ impl InklingUnifiedParser {
|
||||
self.mode = InklingMode::Idle;
|
||||
for token_id in prompt_token_ids.iter().rev().copied() {
|
||||
if token_id == self.message_model_token_id {
|
||||
self.mode = InklingMode::MessageHeader;
|
||||
self.mode = InklingMode::MessageHeader {
|
||||
pending: String::new(),
|
||||
};
|
||||
return;
|
||||
}
|
||||
if token_id == self.content_thinking_token_id {
|
||||
@@ -140,8 +144,19 @@ impl InklingUnifiedParser {
|
||||
InklingEvent::Reasoning { len } => {
|
||||
output.push_reasoning(self.buffer[..len].to_string());
|
||||
}
|
||||
InklingEvent::MessageStart => self.mode = InklingMode::MessageHeader,
|
||||
InklingEvent::Header => {}
|
||||
InklingEvent::MessageStart => {
|
||||
self.mode = InklingMode::MessageHeader {
|
||||
pending: String::new(),
|
||||
};
|
||||
}
|
||||
InklingEvent::Header { len } => {
|
||||
let InklingMode::MessageHeader { pending } = &mut self.mode else {
|
||||
return Err(parsing_failed!(
|
||||
"Inkling header text outside a message header"
|
||||
));
|
||||
};
|
||||
pending.push_str(&self.buffer[..len]);
|
||||
}
|
||||
InklingEvent::TextStart => self.mode = InklingMode::Text,
|
||||
InklingEvent::ReasoningStart => self.mode = InklingMode::Reasoning,
|
||||
InklingEvent::ToolJsonStart => self.mode = InklingMode::ToolJsonHeader,
|
||||
@@ -174,7 +189,10 @@ impl InklingUnifiedParser {
|
||||
}
|
||||
}
|
||||
InklingEvent::BlockEnd => {
|
||||
self.mode = InklingMode::Idle;
|
||||
let mode = std::mem::take(&mut self.mode);
|
||||
if let InklingMode::MessageHeader { pending } = mode {
|
||||
output.push_text(pending);
|
||||
}
|
||||
self.active_tool_index = None;
|
||||
}
|
||||
}
|
||||
@@ -182,10 +200,14 @@ impl InklingUnifiedParser {
|
||||
}
|
||||
|
||||
fn reset(&mut self) -> String {
|
||||
self.mode = InklingMode::Idle;
|
||||
let mut uncommitted = match std::mem::take(&mut self.mode) {
|
||||
InklingMode::MessageHeader { pending } => pending,
|
||||
_ => String::new(),
|
||||
};
|
||||
self.active_tool_index = None;
|
||||
self.emitted_tool_count = 0;
|
||||
std::mem::take(&mut self.buffer)
|
||||
uncommitted.push_str(&std::mem::take(&mut self.buffer));
|
||||
uncommitted
|
||||
}
|
||||
}
|
||||
|
||||
@@ -225,11 +247,15 @@ impl UnifiedParser for InklingUnifiedParser {
|
||||
fn finish(&mut self) -> Result<UnifiedParserOutput> {
|
||||
let mut output = UnifiedParserOutput::default();
|
||||
|
||||
match &self.mode {
|
||||
match &mut self.mode {
|
||||
InklingMode::Idle | InklingMode::Text => {
|
||||
output.push_text(std::mem::take(&mut self.buffer))
|
||||
}
|
||||
InklingMode::MessageHeader => self.buffer.clear(),
|
||||
InklingMode::MessageHeader { pending } => {
|
||||
pending.push_str(&self.buffer);
|
||||
self.buffer.clear();
|
||||
output.push_text(std::mem::take(pending));
|
||||
}
|
||||
InklingMode::Reasoning => output.push_reasoning(std::mem::take(&mut self.buffer)),
|
||||
InklingMode::ToolJsonHeader
|
||||
| InklingMode::ToolJsonArgs { .. }
|
||||
@@ -254,7 +280,7 @@ fn parse_next_inkling_event(
|
||||
) -> ModalResult<InklingEvent> {
|
||||
match mode {
|
||||
InklingMode::Idle => parse_idle_event(input),
|
||||
InklingMode::MessageHeader => parse_message_header_event(input),
|
||||
InklingMode::MessageHeader { .. } => parse_message_header_event(input),
|
||||
InklingMode::Text => parse_text_event(input),
|
||||
InklingMode::Reasoning => parse_reasoning_event(input),
|
||||
InklingMode::ToolJsonHeader => parse_tool_json_header_event(input),
|
||||
@@ -346,7 +372,7 @@ fn safe_idle_text_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEven
|
||||
|
||||
/// Parse safe header text before the next Inkling marker.
|
||||
fn safe_header_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
|
||||
safe_text_len_mul(input, IDLE_MARKERS).map(|_| InklingEvent::Header)
|
||||
safe_text_len_mul(input, IDLE_MARKERS).map(|len| InklingEvent::Header { len })
|
||||
}
|
||||
|
||||
/// Parse safe text before the end of a Inkling text block.
|
||||
@@ -672,17 +698,60 @@ mod tests {
|
||||
let mut parser = test_parser();
|
||||
parser.initialize(&[200001]).unwrap();
|
||||
|
||||
let output = parser
|
||||
.parse_complete(concat!(
|
||||
"get_weather<|content_invoke_tool_json|>",
|
||||
"{\"name\":\"get_weather\",\"args\":{}}<|end_message|>"
|
||||
))
|
||||
.unwrap();
|
||||
let mut output = parser.parse_chunk("get_").unwrap();
|
||||
output.append(
|
||||
parser
|
||||
.parse_complete(concat!(
|
||||
"weather<|content_invoke_tool_json|>",
|
||||
"{\"name\":\"get_weather\",\"args\":{}}<|end_message|>"
|
||||
))
|
||||
.unwrap(),
|
||||
);
|
||||
|
||||
assert!(output.normal_text().is_empty());
|
||||
assert_eq!(output.calls()[0].name.as_deref(), Some("get_weather"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inkling_initialize_model_opener_flushes_bare_text_at_finish() {
|
||||
let mut parser = test_parser();
|
||||
parser.initialize(&[200001]).unwrap();
|
||||
|
||||
let mut output = parser.parse_chunk("plain ").unwrap();
|
||||
output.append(parser.parse_chunk("answer").unwrap());
|
||||
assert!(output.normal_text().is_empty());
|
||||
|
||||
output.append(parser.finish().unwrap());
|
||||
|
||||
assert_eq!(output.normal_text(), "plain answer");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inkling_initialize_model_opener_flushes_bare_text_before_end_marker() {
|
||||
let mut parser = test_parser();
|
||||
parser.initialize(&[200001]).unwrap();
|
||||
|
||||
let output = parser
|
||||
.parse_complete(concat!(
|
||||
"plain answer",
|
||||
"<|end_message|>",
|
||||
"<|content_model_end_sampling|>"
|
||||
))
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(output.normal_text(), "plain answer");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inkling_reset_returns_pending_message_header() {
|
||||
let mut parser = test_parser();
|
||||
parser.initialize(&[200001]).unwrap();
|
||||
parser.parse_chunk("plain ").unwrap();
|
||||
parser.parse_chunk("answer").unwrap();
|
||||
|
||||
assert_eq!(parser.reset(), "plain answer");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inkling_plain_text_falls_through_as_text() {
|
||||
let output = collect_stream(&["plain ", "answer"]);
|
||||
|
||||
@@ -178,6 +178,13 @@ class TestArgConverter:
|
||||
|
||||
|
||||
class TestNonStreaming:
|
||||
@pytest.mark.parametrize("suffix", ["", END_MESSAGE, END_SAMPLING])
|
||||
def test_bare_text_after_model_opener(self, parser, mock_request, suffix):
|
||||
reasoning, content, tools = parser.parse(f"hello world{suffix}", mock_request)
|
||||
assert reasoning is None
|
||||
assert content == "hello world"
|
||||
assert tools is None
|
||||
|
||||
def test_plain_text(self, parser, mock_request):
|
||||
reasoning, content, tools = parser.parse(
|
||||
f"{TEXT_START}hello world{END_MESSAGE}", mock_request
|
||||
@@ -426,6 +433,28 @@ class TestPromptSeededState:
|
||||
assert delta.content is None
|
||||
assert delta.tool_calls[0].function.name == "get_weather"
|
||||
|
||||
def test_generation_prompt_header_flushes_bare_text_at_finish(
|
||||
self, parser, mock_request
|
||||
):
|
||||
prompt_token_ids = [_TML_VOCAB[END_MESSAGE], _TML_VOCAB[MSG_MODEL]]
|
||||
first = parser.parse_delta(
|
||||
"plain ",
|
||||
[ord(char) for char in "plain "],
|
||||
mock_request,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
finished=False,
|
||||
)
|
||||
assert first is None
|
||||
|
||||
second = parser.parse_delta(
|
||||
"answer",
|
||||
[ord(char) for char in "answer"],
|
||||
mock_request,
|
||||
finished=True,
|
||||
)
|
||||
assert second is not None
|
||||
assert second.content == "plain answer"
|
||||
|
||||
|
||||
class TestToolCallFiltering:
|
||||
"""Inkling equivalents of the generic tool-call-filtering replay tests
|
||||
|
||||
@@ -185,6 +185,7 @@ class StreamingParserEngine:
|
||||
# implicit-reasoning-end (content returns None).
|
||||
self._scanner.reset()
|
||||
self._lexer.reset()
|
||||
self._message_header_buffer = ""
|
||||
self._reset_args_state()
|
||||
|
||||
def feed(
|
||||
@@ -270,6 +271,15 @@ class StreamingParserEngine:
|
||||
)
|
||||
self.state = ParserState.CONTENT
|
||||
elif self.state == ParserState.MESSAGE_HEADER:
|
||||
if self._message_header_buffer:
|
||||
events.append(
|
||||
SemanticEvent(
|
||||
EventType.TEXT_CHUNK,
|
||||
value=self._message_header_buffer,
|
||||
tool_index=self.tool_index,
|
||||
)
|
||||
)
|
||||
self._message_header_buffer = ""
|
||||
self.state = ParserState.CONTENT
|
||||
|
||||
return events
|
||||
@@ -311,6 +321,7 @@ class StreamingParserEngine:
|
||||
if self.skip_tool_parsing and terminal in self._tool_terminals:
|
||||
if self.state == ParserState.MESSAGE_HEADER:
|
||||
self.state = ParserState.CONTENT
|
||||
self._message_header_buffer = ""
|
||||
return [
|
||||
SemanticEvent(
|
||||
EventType.TEXT_CHUNK,
|
||||
@@ -345,6 +356,9 @@ class StreamingParserEngine:
|
||||
return self._apply_transition(transition, value)
|
||||
|
||||
def _emit_for_state(self, text: str) -> list[SemanticEvent]:
|
||||
if self.state == ParserState.MESSAGE_HEADER:
|
||||
self._message_header_buffer += text
|
||||
return []
|
||||
if self.state == ParserState.TOOL_ARGS:
|
||||
if self.config.tool_args_json:
|
||||
return self._feed_args_text(text)
|
||||
@@ -371,6 +385,8 @@ class StreamingParserEngine:
|
||||
value: str,
|
||||
) -> list[SemanticEvent]:
|
||||
events: list[SemanticEvent] = []
|
||||
previous_state = self.state
|
||||
message_header = ""
|
||||
|
||||
if (
|
||||
self.state == ParserState.TOOL_ARGS
|
||||
@@ -386,15 +402,27 @@ class StreamingParserEngine:
|
||||
)
|
||||
self._args_buffer = ""
|
||||
|
||||
if previous_state == ParserState.MESSAGE_HEADER:
|
||||
message_header = self._message_header_buffer
|
||||
self._message_header_buffer = ""
|
||||
|
||||
self.state = transition.next_state
|
||||
|
||||
for event_type in transition.events:
|
||||
if event_type == EventType.TOOL_CALL_START:
|
||||
self.tool_index += 1
|
||||
event_value = (
|
||||
message_header
|
||||
if previous_state == ParserState.MESSAGE_HEADER
|
||||
and event_type == EventType.TEXT_CHUNK
|
||||
else value
|
||||
)
|
||||
if event_type == EventType.TEXT_CHUNK and not event_value:
|
||||
continue
|
||||
events.append(
|
||||
SemanticEvent(
|
||||
event_type,
|
||||
value=value,
|
||||
value=event_value,
|
||||
tool_index=self.tool_index,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -268,7 +268,7 @@ def inkling_config() -> ParserEngineConfig:
|
||||
)
|
||||
transitions[(ParserState.MESSAGE_HEADER, end)] = Transition(
|
||||
ParserState.CONTENT,
|
||||
(),
|
||||
(EventType.TEXT_CHUNK,),
|
||||
)
|
||||
|
||||
return ParserEngineConfig(
|
||||
|
||||
Reference in New Issue
Block a user