From 314ab8cffd9b115198d50491c5f1c4a2112ad4a1 Mon Sep 17 00:00:00 2001 From: DimensionSTP <65501090+DimensionSTP@users.noreply.github.com> Date: Mon, 12 Oct 2026 22:25:36 +0900 Subject: [PATCH] [Bugfix][Structured Output] Drain compilation tasks before releasing the backend (#61047) Signed-off-by: DimensionSTP <65501090+DimensionSTP@users.noreply.github.com> Co-authored-by: Artem Perevedentsev --- .../test_structured_output_manager.py | 85 ++++++++++++++++++- vllm/v1/structured_output/__init__.py | 9 ++ 2 files changed, 93 insertions(+), 1 deletion(-) diff --git a/tests/v1/structured_output/test_structured_output_manager.py b/tests/v1/structured_output/test_structured_output_manager.py index 67c1582558..1d1e7b78df 100644 --- a/tests/v1/structured_output/test_structured_output_manager.py +++ b/tests/v1/structured_output/test_structured_output_manager.py @@ -2,12 +2,19 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import time +from concurrent.futures import Future, ThreadPoolExecutor from dataclasses import dataclass +from threading import Event import pytest from transformers import AutoTokenizer -from vllm.config import DeviceConfig, StructuredOutputsConfig, VllmConfig +from vllm.config import ( + DeviceConfig, + SchedulerConfig, + StructuredOutputsConfig, + VllmConfig, +) from vllm.config.model import ModelConfig from vllm.config.speculative import SpeculativeConfig from vllm.config.structured_outputs import StructuredOutputsBackend @@ -576,3 +583,79 @@ def test_outlines_choice_with_non_bmp_characters(tokenizer): grammar = backend.compile_grammar(request_type, grammar_spec) assert grammar.accept_tokens("", tokenizer.encode(choices[0])) + + +def test_clear_backend_waits_for_running_grammar_and_cancels_pending( + tokenizer, monkeypatch +): + """Keep the compiler alive until running work finishes during shutdown.""" + manager, request = _build_harness(tokenizer, "xgrammar", use_reasoner=False) + manager.executor.shutdown(wait=True) + manager.executor = ThreadPoolExecutor(max_workers=1) + started, release = Event(), Event() + assert manager.backend is not None + compile_grammar = manager.backend.compile_grammar + + def delayed_compile(*args, **kwargs): + started.set() + assert release.wait(MAX_WAIT_SECONDS) + return compile_grammar(*args, **kwargs) + + monkeypatch.setattr(manager.backend, "compile_grammar", delayed_compile) + try: + manager.grammar_init(request) + assert request.structured_output_request is not None + running = request.structured_output_request._grammar + assert isinstance(running, Future) + assert started.wait(MAX_WAIT_SECONDS) + pending_request = Request( + request_id="pending-at-shutdown", + prompt_token_ids=[], + sampling_params=request.sampling_params, + pooling_params=None, + ) + manager.grammar_init(pending_request) + assert pending_request.structured_output_request is not None + pending = pending_request.structured_output_request._grammar + assert isinstance(pending, Future) + # Cancellation releases the running task without timing-based sleeps. + pending.add_done_callback(lambda _: release.set()) + manager.clear_backend() + release.set() + assert running.result(timeout=MAX_WAIT_SECONDS) is not None + assert pending.cancelled() + assert manager.backend is None + manager.clear_backend() + finally: + release.set() + manager.executor.shutdown(wait=True, cancel_futures=True) + + +@pytest.mark.parametrize("skip_tokenizer_init", [False, True]) +@pytest.mark.parametrize("max_num_seqs", [1, 256]) +def test_clear_backend_without_grammar(skip_tokenizer_init, max_num_seqs): + """Stop initialized thread pools even if no request needed a backend.""" + manager = StructuredOutputManager( + VllmConfig( + model_config=ModelConfig( + tokenizer=TOKENIZER, skip_tokenizer_init=skip_tokenizer_init + ), + scheduler_config=SchedulerConfig( + max_num_seqs=max_num_seqs, + max_model_len=128, + is_encoder_decoder=False, + ), + device_config=DeviceConfig(device="cpu"), + ) + ) + manager.clear_backend() + manager.clear_backend() + assert manager.backend is None + executors = [] + if not skip_tokenizer_init: + executors.append(manager.executor) + if max_num_seqs > manager.fill_bitmask_parallel_threshold: + executors.append(manager.executor_for_fillmask) + for executor in executors: + with pytest.raises(RuntimeError, match="cannot schedule new futures"): + executor.submit(int, 0) diff --git a/vllm/v1/structured_output/__init__.py b/vllm/v1/structured_output/__init__.py index 2163a800a5..3f461ed47f 100644 --- a/vllm/v1/structured_output/__init__.py +++ b/vllm/v1/structured_output/__init__.py @@ -482,5 +482,14 @@ class StructuredOutputManager: return grammar.accept_tokens(request.request_id, new_token_ids[grammar_start:]) def clear_backend(self) -> None: + # Finish tasks that can still use the backend before releasing it. + if not self.vllm_config.model_config.skip_tokenizer_init: + self.executor.shutdown(wait=True, cancel_futures=True) + if ( + self.fill_bitmask_parallel_threshold + < self.vllm_config.scheduler_config.max_num_seqs + ): + self.executor_for_fillmask.shutdown(wait=True, cancel_futures=True) if self.backend is not None: self.backend.destroy() + self.backend = None