diff --git a/tests/v1/structured_output/test_structured_output_manager.py b/tests/v1/structured_output/test_structured_output_manager.py index 17bda09b3e..a1da205fe0 100644 --- a/tests/v1/structured_output/test_structured_output_manager.py +++ b/tests/v1/structured_output/test_structured_output_manager.py @@ -19,6 +19,8 @@ THINK_END = "\n" # reasoning-end marker (single GPT-2 token) EOS = "<|eos|>" # resolved to tokenizer.eos_token_id JSON_SCHEMA = '{"type": "object"}' BACKENDS = ("xgrammar", "guidance") +REGEX_BACKENDS = (*BACKENDS, "outlines") +DIGITS_REGEX = "[0-9]+" MAX_WAIT_SECONDS = 5 NUM_SPEC_TOKENS = 8 @@ -88,6 +90,7 @@ def _build_harness( reasoning_ended: bool | None = None, enable_in_reasoning: bool = False, reasoning_parser_kwargs: dict | None = None, + structured_outputs: StructuredOutputsParams | None = None, ) -> tuple[StructuredOutputManager, Request]: vllm_config = VllmConfig( model_config=ModelConfig(tokenizer=TOKENIZER), @@ -106,7 +109,8 @@ def _build_harness( manager.reasoner_cls = MockReasoner sampling_params = SamplingParams( - structured_outputs=StructuredOutputsParams(json=JSON_SCHEMA) + structured_outputs=structured_outputs + or StructuredOutputsParams(json=JSON_SCHEMA) ) sampling_params.structured_outputs._backend = backend sampling_params.update_from_generation_config({}, tokenizer.eos_token_id) @@ -412,3 +416,96 @@ def test_initial_constraint_activation( structured_req = request.structured_output_request assert structured_req is not None assert structured_req.reasoner is None + + +@pytest.mark.parametrize("backend", REGEX_BACKENDS) +@pytest.mark.parametrize( + ("raw_drafts", "expected_validated", "expected_row_pattern", "terminated"), + [ + pytest.param(("1", "2"), None, "CCC", False, id="all_valid"), + # EOS is only valid once the regex has matched; rows after EOS are + # unconstrained since the request stops there. + pytest.param(("1", EOS, "2"), ("1", EOS), "CCUU", True, id="terminates"), + pytest.param((EOS, "1"), (), "CUU", False, id="early_eos"), + ], +) +def test_regex_flow( + tokenizer, + backend: str, + raw_drafts: tuple[str, ...], + expected_validated: tuple[str, ...] | None, + expected_row_pattern: str, + terminated: bool, +): + manager, request = _build_harness( + tokenizer, + backend, + use_reasoner=False, + structured_outputs=StructuredOutputsParams(regex=DIGITS_REGEX), + ) + validated = _to_token_ids( + tokenizer, raw_drafts if expected_validated is None else expected_validated + ) + _run_real_flow( + manager, + request, + raw_drafts=_to_token_ids(tokenizer, raw_drafts), + expected_validated=validated, + expected_row_pattern=expected_row_pattern, + # accept_tokens() latches reasoning_ended once it accepts any token. + expected_reasoning=True if validated else None, + expect_terminated=terminated, + ) + + +@pytest.mark.parametrize("backend", REGEX_BACKENDS) +def test_rejected_draft_keeps_later_rows_constrained(tokenizer, backend: str): + """A draft the grammar rejects during bitmask fill must not unconstrain + the remaining rows or the bonus row.""" + manager, request = _build_harness( + tokenizer, + backend, + use_reasoner=False, + structured_outputs=StructuredOutputsParams(regex=DIGITS_REGEX), + ) + drafts = _to_token_ids(tokenizer, ("1", "z", "2")) + bitmask = manager.grammar_bitmask( + requests={request.request_id: request}, + structured_output_request_ids=[request.request_id], + scheduled_spec_decode_tokens={request.request_id: drafts}, + ) + assert bitmask is not None + assert _row_pattern(bitmask) == "CCCC" + assert (bitmask[2] == bitmask[1]).all() + assert (bitmask[3] == bitmask[1]).all() + + +def test_outlines_termination(tokenizer): + """outlines_core never advances on EOS; termination must still be + tracked exactly once per accepted EOS and undone by rollback.""" + manager, request = _build_harness( + tokenizer, + "outlines", + use_reasoner=False, + structured_outputs=StructuredOutputsParams(regex=DIGITS_REGEX), + ) + structured_req = request.structured_output_request + assert structured_req is not None + grammar = structured_req.grammar + assert grammar is not None + one, eos = _to_token_ids(tokenizer, ("1", EOS)) + + assert not grammar.accept_tokens(request.request_id, [eos]) + assert grammar.accept_tokens(request.request_id, [one]) + # Polling must not change the result. + assert not any(grammar.is_terminated() for _ in range(3)) + assert grammar.accept_tokens(request.request_id, [eos]) + assert all(grammar.is_terminated() for _ in range(3)) + assert grammar.num_processed_tokens == 2 + + grammar.rollback(2) + assert not grammar.is_terminated() + assert grammar.num_processed_tokens == 0 + assert grammar.validate_tokens([eos]) == [] + assert grammar.accept_tokens(request.request_id, [one, eos, one]) + assert grammar.is_terminated() diff --git a/vllm/v1/structured_output/__init__.py b/vllm/v1/structured_output/__init__.py index 7fd311c873..3b35d39d14 100644 --- a/vllm/v1/structured_output/__init__.py +++ b/vllm/v1/structured_output/__init__.py @@ -389,11 +389,16 @@ class StructuredOutputManager: ) state_advancements = 0 seen_padding = False - failed = False + # Row filled from the last valid grammar state before a draft + # was rejected; later rows reuse it rather than unconstraining. + failed_index: int | None = None + bitmask = self._grammar_bitmask for i, token in enumerate(req_tokens): - apply_bitmask = ( - not failed and not seen_padding and i >= constraint_start - ) + if failed_index is not None: + bitmask[cumulative_index].copy_(bitmask[failed_index]) + cumulative_index += 1 + continue + apply_bitmask = not seen_padding and i >= constraint_start self._fill_bitmasks(((grammar, cumulative_index, apply_bitmask),)) if token == -1: seen_padding = True @@ -403,7 +408,7 @@ class StructuredOutputManager: ): state_advancements += 1 else: - failed = True + failed_index = cumulative_index logger.error( "Unexpected: grammar terminated or rejected draft " "token %s for request %s during bitmask fill.", @@ -415,12 +420,13 @@ class StructuredOutputManager: # Diffusion LLMs don't sample a bonus token after the # scheduled positions, so skip its bitmask in that case. if not (self.vllm_config.model_config.is_diffusion and req_tokens): - bonus_apply = ( - not failed - and not seen_padding - and constraint_start <= len(req_tokens) - ) - self._fill_bitmasks(((grammar, cumulative_index, bonus_apply),)) + if failed_index is not None: + bitmask[cumulative_index].copy_(bitmask[failed_index]) + else: + bonus_apply = not seen_padding and constraint_start <= len( + req_tokens + ) + self._fill_bitmasks(((grammar, cumulative_index, bonus_apply),)) cumulative_index += 1 if state_advancements > 0: diff --git a/vllm/v1/structured_output/backend_outlines.py b/vllm/v1/structured_output/backend_outlines.py index bb49fb189b..cc89ef3e84 100644 --- a/vllm/v1/structured_output/backend_outlines.py +++ b/vllm/v1/structured_output/backend_outlines.py @@ -97,6 +97,7 @@ class OutlinesBackend(StructuredOutputBackend): ) return OutlinesGrammar( vocab_size=self.vocab_size, + eos_token_id=self.vocabulary.inner.get_eos_token_id(), guide=oc.Guide(index, max_rollback=max_rollback_tokens), ) @@ -115,60 +116,94 @@ class OutlinesBackend(StructuredOutputBackend): @dataclass class OutlinesGrammar(StructuredOutputGrammar): vocab_size: int + eos_token_id: int guide: oc.Guide = field(hash=False) num_processed_tokens: int = field( default_factory=lambda: 0, repr=False, hash=False, init=False ) - # outlines_core signals done on DFA accept; vLLM expects done after EOS. - # We delay the finished flag by one step so EOS can still be emitted. - _prev_finished: bool = field(default=False, init=False, repr=False, hash=False) + # outlines_core signals done on DFA accept and never advances on EOS + # (its mask allows only EOS in a final state, but advance(EOS) fails), + # so accepting EOS in a final state is tracked here instead. + _is_terminated: bool = field(default=False, init=False, repr=False, hash=False) def accept_tokens(self, request_id: str, tokens: list[int]) -> bool: """Accepts a list of tokens and advances the FSM. - Returns True if the FSM was advanced successfully. - Returns False if the FSM failed to advance. + Returns True if all grammar-constrained tokens were accepted. + Tokens after termination (EOS) are ignored. Returns False if the FSM + failed to advance. """ - if self.guide.accepts_tokens(tokens): - # Advance can fail when the next state reached after advancing with - # the current tokens is a dead state. This is because Guide.accepts_tokens() - # only checks whether the current tokens can be accepted, - # whereas guide.advance() additionally checks the next state - # after all tokens are accepted. - # We need to be aware that the FSM must be prepared without dead states. - for t in tokens: - self.guide.advance(t) - self.num_processed_tokens += 1 + if self._is_terminated: return True - return False + eos_index = next( + (i for i, t in enumerate(tokens) if t == self.eos_token_id), None + ) + if eos_index is not None: + tokens = tokens[:eos_index] + # Advance can fail when the next state reached after advancing with + # the current tokens is a dead state. This is because Guide.accepts_tokens() + # only checks whether the current tokens can be accepted, + # whereas guide.advance() additionally checks the next state + # after all tokens are accepted. + # We need to be aware that the FSM must be prepared without dead states. + if tokens and not self.guide.accepts_tokens(tokens): + return False + for t in tokens: + self.guide.advance(t) + if eos_index is not None: + if not self.guide.is_finished(): + if tokens: + self.guide.rollback_state(len(tokens)) + return False + self._is_terminated = True + self.num_processed_tokens += len(tokens) + self._is_terminated + return True def rollback(self, num_tokens: int) -> None: - self.guide.rollback_state(num_tokens) + if num_tokens <= 0: + return self.num_processed_tokens -= num_tokens + if self._is_terminated: + self._is_terminated = False + num_tokens -= 1 + if num_tokens: + self.guide.rollback_state(num_tokens) def validate_tokens(self, tokens: list[int]) -> list[int]: + if self._is_terminated: + return [] accepted: list[int] = [] for tok in tokens: + if tok == self.eos_token_id: + if self._is_finished_after(accepted): + accepted.append(tok) + break accepted.append(tok) if not self.guide.accepts_tokens(accepted): accepted.pop() break return accepted + def _is_finished_after(self, tokens: list[int]) -> bool: + if not tokens: + return self.guide.is_finished() + for t in tokens: + self.guide.advance(t) + finished = self.guide.is_finished() + self.guide.rollback_state(len(tokens)) + return finished + def fill_bitmask(self, bitmask: torch.Tensor, idx: int) -> None: mask = bitmask[idx] self.guide.write_mask_into(mask.data_ptr(), mask.numel(), mask.element_size()) def is_terminated(self) -> bool: - curr = self.guide.is_finished() - prev = self._prev_finished - self._prev_finished = curr - return prev + return self._is_terminated def reset(self): self.num_processed_tokens = 0 - self._prev_finished = False + self._is_terminated = False self.guide.reset()