[BugFix] Adding env variable to disable async grammar compilation (#29996)

Signed-off-by: Alec Solder <alecs@fb.com>
Signed-off-by: Alec S <10566873+alecsolder@users.noreply.github.com>
Co-authored-by: Alec Solder <alecs@fb.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: 22quinn <33176974+22quinn@users.noreply.github.com>
This commit is contained in:
Alec S
2025-12-05 00:49:37 -08:00
committed by GitHub
co-authored by Alec Solder gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> 22quinn
parent 62b3333448
commit 65ee97288a
2 changed files with 89 additions and 2 deletions
+15 -2
View File
@@ -40,6 +40,16 @@ class StructuredOutputManager:
self.reasoner: ReasoningParser | None = None
self.vllm_config = vllm_config
# When in external_launcher mode, async grammar compilation causes deadlocks
# due to external_launcher mode having a scheduler for each TP rank.
# Async grammar compilation causes the WAITING_FOR_FSM → WAITING transition to
# happen at different times on different TP ranks,
# breaking the determinism assumption that external_launcher relies on.
self._use_async_grammar_compilation = (
vllm_config.parallel_config.distributed_executor_backend
!= "external_launcher"
)
self._grammar_bitmask: torch.Tensor | None = None
self._full_mask = torch.tensor(-1, dtype=torch.int32)
@@ -138,10 +148,13 @@ class StructuredOutputManager:
else:
raise ValueError(f"Unsupported structured output backend: {backend}")
grammar = self.executor.submit(self._async_create_grammar, request)
if self._use_async_grammar_compilation:
grammar = self.executor.submit(self._create_grammar, request)
else:
grammar = self._create_grammar(request) # type: ignore[assignment]
request.structured_output_request.grammar = grammar # type: ignore[assignment]
def _async_create_grammar(
def _create_grammar(
self,
request: Request,
) -> StructuredOutputGrammar: