[BugFix] Adding env variable to disable async grammar compilation (#29996)
Signed-off-by: Alec Solder <alecs@fb.com> Signed-off-by: Alec S <10566873+alecsolder@users.noreply.github.com> Co-authored-by: Alec Solder <alecs@fb.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: 22quinn <33176974+22quinn@users.noreply.github.com>
This commit is contained in:
co-authored by
Alec Solder
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
22quinn
parent
62b3333448
commit
65ee97288a
@@ -40,6 +40,16 @@ class StructuredOutputManager:
|
||||
self.reasoner: ReasoningParser | None = None
|
||||
self.vllm_config = vllm_config
|
||||
|
||||
# When in external_launcher mode, async grammar compilation causes deadlocks
|
||||
# due to external_launcher mode having a scheduler for each TP rank.
|
||||
# Async grammar compilation causes the WAITING_FOR_FSM → WAITING transition to
|
||||
# happen at different times on different TP ranks,
|
||||
# breaking the determinism assumption that external_launcher relies on.
|
||||
self._use_async_grammar_compilation = (
|
||||
vllm_config.parallel_config.distributed_executor_backend
|
||||
!= "external_launcher"
|
||||
)
|
||||
|
||||
self._grammar_bitmask: torch.Tensor | None = None
|
||||
self._full_mask = torch.tensor(-1, dtype=torch.int32)
|
||||
|
||||
@@ -138,10 +148,13 @@ class StructuredOutputManager:
|
||||
else:
|
||||
raise ValueError(f"Unsupported structured output backend: {backend}")
|
||||
|
||||
grammar = self.executor.submit(self._async_create_grammar, request)
|
||||
if self._use_async_grammar_compilation:
|
||||
grammar = self.executor.submit(self._create_grammar, request)
|
||||
else:
|
||||
grammar = self._create_grammar(request) # type: ignore[assignment]
|
||||
request.structured_output_request.grammar = grammar # type: ignore[assignment]
|
||||
|
||||
def _async_create_grammar(
|
||||
def _create_grammar(
|
||||
self,
|
||||
request: Request,
|
||||
) -> StructuredOutputGrammar:
|
||||
|
||||
Reference in New Issue
Block a user