diff --git a/rust/src/parser/src/unified/inkling.rs b/rust/src/parser/src/unified/inkling.rs index 1ce69d8bfb..d35fb48d9c 100644 --- a/rust/src/parser/src/unified/inkling.rs +++ b/rust/src/parser/src/unified/inkling.rs @@ -61,7 +61,7 @@ enum InklingEvent { TextStart, ReasoningStart, MessageStart, - Header, + Header { len: usize }, ToolJsonStart, ToolJsonHeader { name: String }, ToolJsonArgs { len: usize, complete: bool }, @@ -72,7 +72,9 @@ enum InklingEvent { enum InklingMode { #[default] Idle, - MessageHeader, + MessageHeader { + pending: String, + }, Text, Reasoning, ToolJsonHeader, @@ -117,7 +119,9 @@ impl InklingUnifiedParser { self.mode = InklingMode::Idle; for token_id in prompt_token_ids.iter().rev().copied() { if token_id == self.message_model_token_id { - self.mode = InklingMode::MessageHeader; + self.mode = InklingMode::MessageHeader { + pending: String::new(), + }; return; } if token_id == self.content_thinking_token_id { @@ -140,8 +144,19 @@ impl InklingUnifiedParser { InklingEvent::Reasoning { len } => { output.push_reasoning(self.buffer[..len].to_string()); } - InklingEvent::MessageStart => self.mode = InklingMode::MessageHeader, - InklingEvent::Header => {} + InklingEvent::MessageStart => { + self.mode = InklingMode::MessageHeader { + pending: String::new(), + }; + } + InklingEvent::Header { len } => { + let InklingMode::MessageHeader { pending } = &mut self.mode else { + return Err(parsing_failed!( + "Inkling header text outside a message header" + )); + }; + pending.push_str(&self.buffer[..len]); + } InklingEvent::TextStart => self.mode = InklingMode::Text, InklingEvent::ReasoningStart => self.mode = InklingMode::Reasoning, InklingEvent::ToolJsonStart => self.mode = InklingMode::ToolJsonHeader, @@ -174,7 +189,10 @@ impl InklingUnifiedParser { } } InklingEvent::BlockEnd => { - self.mode = InklingMode::Idle; + let mode = std::mem::take(&mut self.mode); + if let InklingMode::MessageHeader { pending } = mode { + output.push_text(pending); + } self.active_tool_index = None; } } @@ -182,10 +200,14 @@ impl InklingUnifiedParser { } fn reset(&mut self) -> String { - self.mode = InklingMode::Idle; + let mut uncommitted = match std::mem::take(&mut self.mode) { + InklingMode::MessageHeader { pending } => pending, + _ => String::new(), + }; self.active_tool_index = None; self.emitted_tool_count = 0; - std::mem::take(&mut self.buffer) + uncommitted.push_str(&std::mem::take(&mut self.buffer)); + uncommitted } } @@ -225,11 +247,15 @@ impl UnifiedParser for InklingUnifiedParser { fn finish(&mut self) -> Result { let mut output = UnifiedParserOutput::default(); - match &self.mode { + match &mut self.mode { InklingMode::Idle | InklingMode::Text => { output.push_text(std::mem::take(&mut self.buffer)) } - InklingMode::MessageHeader => self.buffer.clear(), + InklingMode::MessageHeader { pending } => { + pending.push_str(&self.buffer); + self.buffer.clear(); + output.push_text(std::mem::take(pending)); + } InklingMode::Reasoning => output.push_reasoning(std::mem::take(&mut self.buffer)), InklingMode::ToolJsonHeader | InklingMode::ToolJsonArgs { .. } @@ -254,7 +280,7 @@ fn parse_next_inkling_event( ) -> ModalResult { match mode { InklingMode::Idle => parse_idle_event(input), - InklingMode::MessageHeader => parse_message_header_event(input), + InklingMode::MessageHeader { .. } => parse_message_header_event(input), InklingMode::Text => parse_text_event(input), InklingMode::Reasoning => parse_reasoning_event(input), InklingMode::ToolJsonHeader => parse_tool_json_header_event(input), @@ -346,7 +372,7 @@ fn safe_idle_text_event(input: &mut InklingInput<'_>) -> ModalResult) -> ModalResult { - safe_text_len_mul(input, IDLE_MARKERS).map(|_| InklingEvent::Header) + safe_text_len_mul(input, IDLE_MARKERS).map(|len| InklingEvent::Header { len }) } /// Parse safe text before the end of a Inkling text block. @@ -672,17 +698,60 @@ mod tests { let mut parser = test_parser(); parser.initialize(&[200001]).unwrap(); - let output = parser - .parse_complete(concat!( - "get_weather<|content_invoke_tool_json|>", - "{\"name\":\"get_weather\",\"args\":{}}<|end_message|>" - )) - .unwrap(); + let mut output = parser.parse_chunk("get_").unwrap(); + output.append( + parser + .parse_complete(concat!( + "weather<|content_invoke_tool_json|>", + "{\"name\":\"get_weather\",\"args\":{}}<|end_message|>" + )) + .unwrap(), + ); assert!(output.normal_text().is_empty()); assert_eq!(output.calls()[0].name.as_deref(), Some("get_weather")); } + #[test] + fn inkling_initialize_model_opener_flushes_bare_text_at_finish() { + let mut parser = test_parser(); + parser.initialize(&[200001]).unwrap(); + + let mut output = parser.parse_chunk("plain ").unwrap(); + output.append(parser.parse_chunk("answer").unwrap()); + assert!(output.normal_text().is_empty()); + + output.append(parser.finish().unwrap()); + + assert_eq!(output.normal_text(), "plain answer"); + } + + #[test] + fn inkling_initialize_model_opener_flushes_bare_text_before_end_marker() { + let mut parser = test_parser(); + parser.initialize(&[200001]).unwrap(); + + let output = parser + .parse_complete(concat!( + "plain answer", + "<|end_message|>", + "<|content_model_end_sampling|>" + )) + .unwrap(); + + assert_eq!(output.normal_text(), "plain answer"); + } + + #[test] + fn inkling_reset_returns_pending_message_header() { + let mut parser = test_parser(); + parser.initialize(&[200001]).unwrap(); + parser.parse_chunk("plain ").unwrap(); + parser.parse_chunk("answer").unwrap(); + + assert_eq!(parser.reset(), "plain answer"); + } + #[test] fn inkling_plain_text_falls_through_as_text() { let output = collect_stream(&["plain ", "answer"]); diff --git a/tests/parser/engine/test_inkling.py b/tests/parser/engine/test_inkling.py index c0b9877db3..582783fd5d 100644 --- a/tests/parser/engine/test_inkling.py +++ b/tests/parser/engine/test_inkling.py @@ -178,6 +178,13 @@ class TestArgConverter: class TestNonStreaming: + @pytest.mark.parametrize("suffix", ["", END_MESSAGE, END_SAMPLING]) + def test_bare_text_after_model_opener(self, parser, mock_request, suffix): + reasoning, content, tools = parser.parse(f"hello world{suffix}", mock_request) + assert reasoning is None + assert content == "hello world" + assert tools is None + def test_plain_text(self, parser, mock_request): reasoning, content, tools = parser.parse( f"{TEXT_START}hello world{END_MESSAGE}", mock_request @@ -426,6 +433,28 @@ class TestPromptSeededState: assert delta.content is None assert delta.tool_calls[0].function.name == "get_weather" + def test_generation_prompt_header_flushes_bare_text_at_finish( + self, parser, mock_request + ): + prompt_token_ids = [_TML_VOCAB[END_MESSAGE], _TML_VOCAB[MSG_MODEL]] + first = parser.parse_delta( + "plain ", + [ord(char) for char in "plain "], + mock_request, + prompt_token_ids=prompt_token_ids, + finished=False, + ) + assert first is None + + second = parser.parse_delta( + "answer", + [ord(char) for char in "answer"], + mock_request, + finished=True, + ) + assert second is not None + assert second.content == "plain answer" + class TestToolCallFiltering: """Inkling equivalents of the generic tool-call-filtering replay tests diff --git a/vllm/parser/engine/streaming_parser_engine.py b/vllm/parser/engine/streaming_parser_engine.py index b459b75b34..f5c395f76d 100644 --- a/vllm/parser/engine/streaming_parser_engine.py +++ b/vllm/parser/engine/streaming_parser_engine.py @@ -185,6 +185,7 @@ class StreamingParserEngine: # implicit-reasoning-end (content returns None). self._scanner.reset() self._lexer.reset() + self._message_header_buffer = "" self._reset_args_state() def feed( @@ -270,6 +271,15 @@ class StreamingParserEngine: ) self.state = ParserState.CONTENT elif self.state == ParserState.MESSAGE_HEADER: + if self._message_header_buffer: + events.append( + SemanticEvent( + EventType.TEXT_CHUNK, + value=self._message_header_buffer, + tool_index=self.tool_index, + ) + ) + self._message_header_buffer = "" self.state = ParserState.CONTENT return events @@ -311,6 +321,7 @@ class StreamingParserEngine: if self.skip_tool_parsing and terminal in self._tool_terminals: if self.state == ParserState.MESSAGE_HEADER: self.state = ParserState.CONTENT + self._message_header_buffer = "" return [ SemanticEvent( EventType.TEXT_CHUNK, @@ -345,6 +356,9 @@ class StreamingParserEngine: return self._apply_transition(transition, value) def _emit_for_state(self, text: str) -> list[SemanticEvent]: + if self.state == ParserState.MESSAGE_HEADER: + self._message_header_buffer += text + return [] if self.state == ParserState.TOOL_ARGS: if self.config.tool_args_json: return self._feed_args_text(text) @@ -371,6 +385,8 @@ class StreamingParserEngine: value: str, ) -> list[SemanticEvent]: events: list[SemanticEvent] = [] + previous_state = self.state + message_header = "" if ( self.state == ParserState.TOOL_ARGS @@ -386,15 +402,27 @@ class StreamingParserEngine: ) self._args_buffer = "" + if previous_state == ParserState.MESSAGE_HEADER: + message_header = self._message_header_buffer + self._message_header_buffer = "" + self.state = transition.next_state for event_type in transition.events: if event_type == EventType.TOOL_CALL_START: self.tool_index += 1 + event_value = ( + message_header + if previous_state == ParserState.MESSAGE_HEADER + and event_type == EventType.TEXT_CHUNK + else value + ) + if event_type == EventType.TEXT_CHUNK and not event_value: + continue events.append( SemanticEvent( event_type, - value=value, + value=event_value, tool_index=self.tool_index, ) ) diff --git a/vllm/parser/inkling.py b/vllm/parser/inkling.py index 84502be018..994382539e 100644 --- a/vllm/parser/inkling.py +++ b/vllm/parser/inkling.py @@ -268,7 +268,7 @@ def inkling_config() -> ParserEngineConfig: ) transitions[(ParserState.MESSAGE_HEADER, end)] = Transition( ParserState.CONTENT, - (), + (EventType.TEXT_CHUNK,), ) return ParserEngineConfig(