[Core] Streamline some structured output related code (#26737)
Signed-off-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
@@ -47,7 +47,6 @@ def apply_grammar_bitmask(
|
||||
scheduler_output: SchedulerOutput,
|
||||
input_batch: InputBatch,
|
||||
logits: torch.Tensor,
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
"""
|
||||
Apply grammar bitmask to output logits of the model with xgrammar function.
|
||||
@@ -56,7 +55,6 @@ def apply_grammar_bitmask(
|
||||
scheduler_output (SchedulerOutput): The result of engine scheduling.
|
||||
input_batch (InputBatch): The input of model runner.
|
||||
logits (torch.Tensor): The output logits of model forward.
|
||||
device (torch.device): The device that model runner running on.
|
||||
"""
|
||||
grammar_bitmask = scheduler_output.grammar_bitmask
|
||||
if grammar_bitmask is None:
|
||||
@@ -91,10 +89,7 @@ def apply_grammar_bitmask(
|
||||
dtype=grammar_bitmask.dtype,
|
||||
)
|
||||
cumulative_index = 0
|
||||
seq = sorted(
|
||||
scheduler_output.structured_output_request_ids.items(), key=lambda x: x[1]
|
||||
)
|
||||
for req_id, _ in seq:
|
||||
for req_id in scheduler_output.structured_output_request_ids:
|
||||
num_spec_tokens = len(
|
||||
scheduler_output.scheduled_spec_decode_tokens.get(req_id, [])
|
||||
)
|
||||
@@ -117,7 +112,7 @@ def apply_grammar_bitmask(
|
||||
|
||||
xgr.apply_token_bitmask_inplace(
|
||||
logits,
|
||||
grammar_bitmask.to(device, non_blocking=True),
|
||||
grammar_bitmask.to(logits.device, non_blocking=True),
|
||||
indices=out_indices if not skip_out_indices else None,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user