[ModelRunner V2] Support spec decode with structured outputs (#33374)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-02-02 00:19:59 +00:00
committed by GitHub
parent e535d90deb
commit cf0a99f84d
3 changed files with 60 additions and 1 deletions
+4
View File
@@ -75,6 +75,9 @@ class InputBatch:
cu_num_logits: torch.Tensor
cu_num_logits_np: np.ndarray
# Whether any requests in batch use structured output.
has_structured_output_reqs: bool
@classmethod
def make_dummy(
cls,
@@ -139,6 +142,7 @@ class InputBatch:
logits_indices=logits_indices,
cu_num_logits=cu_num_logits,
cu_num_logits_np=cu_num_logits_np,
has_structured_output_reqs=False,
)