[Bugfix][Structured Output] Drain compilation tasks before releasing the backend (#61047)

Signed-off-by: DimensionSTP <65501090+DimensionSTP@users.noreply.github.com>
Co-authored-by: Artem Perevedentsev <aperevedents@nvidia.com>
This commit is contained in:
DimensionSTP
2026-10-12 16:25:36 +03:00
committed by GitHub
co-authored by Artem Perevedentsev
parent ac28877f0e
commit 314ab8cffd
2 changed files with 93 additions and 1 deletions
@@ -2,12 +2,19 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from concurrent.futures import Future, ThreadPoolExecutor
from dataclasses import dataclass
from threading import Event
import pytest
from transformers import AutoTokenizer
from vllm.config import DeviceConfig, StructuredOutputsConfig, VllmConfig
from vllm.config import (
DeviceConfig,
SchedulerConfig,
StructuredOutputsConfig,
VllmConfig,
)
from vllm.config.model import ModelConfig
from vllm.config.speculative import SpeculativeConfig
from vllm.config.structured_outputs import StructuredOutputsBackend
@@ -576,3 +583,79 @@ def test_outlines_choice_with_non_bmp_characters(tokenizer):
grammar = backend.compile_grammar(request_type, grammar_spec)
assert grammar.accept_tokens("", tokenizer.encode(choices[0]))
def test_clear_backend_waits_for_running_grammar_and_cancels_pending(
tokenizer, monkeypatch
):
"""Keep the compiler alive until running work finishes during shutdown."""
manager, request = _build_harness(tokenizer, "xgrammar", use_reasoner=False)
manager.executor.shutdown(wait=True)
manager.executor = ThreadPoolExecutor(max_workers=1)
started, release = Event(), Event()
assert manager.backend is not None
compile_grammar = manager.backend.compile_grammar
def delayed_compile(*args, **kwargs):
started.set()
assert release.wait(MAX_WAIT_SECONDS)
return compile_grammar(*args, **kwargs)
monkeypatch.setattr(manager.backend, "compile_grammar", delayed_compile)
try:
manager.grammar_init(request)
assert request.structured_output_request is not None
running = request.structured_output_request._grammar
assert isinstance(running, Future)
assert started.wait(MAX_WAIT_SECONDS)
pending_request = Request(
request_id="pending-at-shutdown",
prompt_token_ids=[],
sampling_params=request.sampling_params,
pooling_params=None,
)
manager.grammar_init(pending_request)
assert pending_request.structured_output_request is not None
pending = pending_request.structured_output_request._grammar
assert isinstance(pending, Future)
# Cancellation releases the running task without timing-based sleeps.
pending.add_done_callback(lambda _: release.set())
manager.clear_backend()
release.set()
assert running.result(timeout=MAX_WAIT_SECONDS) is not None
assert pending.cancelled()
assert manager.backend is None
manager.clear_backend()
finally:
release.set()
manager.executor.shutdown(wait=True, cancel_futures=True)
@pytest.mark.parametrize("skip_tokenizer_init", [False, True])
@pytest.mark.parametrize("max_num_seqs", [1, 256])
def test_clear_backend_without_grammar(skip_tokenizer_init, max_num_seqs):
"""Stop initialized thread pools even if no request needed a backend."""
manager = StructuredOutputManager(
VllmConfig(
model_config=ModelConfig(
tokenizer=TOKENIZER, skip_tokenizer_init=skip_tokenizer_init
),
scheduler_config=SchedulerConfig(
max_num_seqs=max_num_seqs,
max_model_len=128,
is_encoder_decoder=False,
),
device_config=DeviceConfig(device="cpu"),
)
)
manager.clear_backend()
manager.clear_backend()
assert manager.backend is None
executors = []
if not skip_tokenizer_init:
executors.append(manager.executor)
if max_num_seqs > manager.fill_bitmask_parallel_threshold:
executors.append(manager.executor_for_fillmask)
for executor in executors:
with pytest.raises(RuntimeError, match="cannot schedule new futures"):
executor.submit(int, 0)
+9
View File
@@ -482,5 +482,14 @@ class StructuredOutputManager:
return grammar.accept_tokens(request.request_id, new_token_ids[grammar_start:])
def clear_backend(self) -> None:
# Finish tasks that can still use the backend before releasing it.
if not self.vllm_config.model_config.skip_tokenizer_init:
self.executor.shutdown(wait=True, cancel_futures=True)
if (
self.fill_bitmask_parallel_threshold
< self.vllm_config.scheduler_config.max_num_seqs
):
self.executor_for_fillmask.shutdown(wait=True, cancel_futures=True)
if self.backend is not None:
self.backend.destroy()
self.backend = None