[Structured Output] Let terminal grammars stop under min_tokens (II) (#54218)

Signed-off-by: Artem Perevedentsev <aperevedents@nvidia.com>
This commit is contained in:
Artem Perevedentsev
2026-08-31 20:39:27 +04:00
committed by GitHub
parent e9dd6d4834
commit 39e276eaeb
7 changed files with 501 additions and 60 deletions
+9 -3
View File
@@ -97,9 +97,6 @@ def check_stop(request: Request, max_model_len: int) -> bool:
sampling_params = request.sampling_params
assert sampling_params is not None
if request.num_output_tokens < sampling_params.min_tokens:
return False
last_token_id = request.output_token_ids[-1]
if last_token_id == sampling_params.eos_token_id:
request.status = RequestStatus.FINISHED_STOPPED
@@ -109,6 +106,7 @@ def check_stop(request: Request, max_model_len: int) -> bool:
request.status = RequestStatus.FINISHED_STOPPED
request.stop_reason = last_token_id
return True
if (
request.num_tokens >= max_model_len
or request.num_output_tokens >= request.max_tokens
@@ -116,6 +114,14 @@ def check_stop(request: Request, max_model_len: int) -> bool:
request.status = RequestStatus.FINISHED_LENGTH_CAPPED
return True
# Note(arpera):
# Order of checks is important for min_tokens
# If you decide to change the order, first look at these PRs please:
# 47489, 49521, and 51299
# These PRs show some reasoning about the order of checks
if request.num_output_tokens < sampling_params.min_tokens:
return False
repetition_detection = sampling_params.repetition_detection
if repetition_detection is not None and (
check_sequence_repetition(
+103 -18
View File
@@ -166,16 +166,22 @@ class MinTokensLogitsProcessor(LogitsProcessor):
def __init__(
self, vllm_config: "VllmConfig", device: torch.device, is_pin_memory: bool
):
# index -> (min_toks, output_token_ids, stop_token_ids)
# index -> (min_toks, output_token_ids, stop_token_ids, uses_structured_output)
self.device = device
self.min_toks: dict[int, tuple[int, Sequence[int], set[int]]] = {}
self.min_toks: dict[int, tuple[int, Sequence[int], set[int], bool]] = {}
# (req_idx_tensor,eos_tok_id_tensor)
# (req_idx_tensor, stop_token_id_tensor)
self.logits_slice: tuple[torch.Tensor, torch.Tensor] = (
self._device_tensor([], torch.int32),
self._device_tensor([], torch.int32),
)
# Structured-output stop-token logits that may need to be restored.
self.restore_logits_slice: tuple[torch.Tensor, torch.Tensor] = (
self._device_tensor([], torch.int32),
self._device_tensor([], torch.int32),
)
self.neg_inf_tensor = torch.tensor(
-float("inf"), dtype=torch.float32, device=self.device
)
@@ -188,11 +194,16 @@ class MinTokensLogitsProcessor(LogitsProcessor):
@staticmethod
def add_request(
params: SamplingParams, _: list[int] | None, output_tok_ids: list[int]
) -> tuple[int, Sequence[int], set[int]] | None:
) -> tuple[int, Sequence[int], set[int], bool] | None:
min_tokens = params.min_tokens
if not min_tokens or len(output_tok_ids) >= min_tokens:
return None
return min_tokens, output_tok_ids, params.all_stop_token_ids
return (
min_tokens,
output_tok_ids,
params.all_stop_token_ids,
params.structured_outputs is not None,
)
def update_state(self, batch_update: BatchUpdate | None):
needs_update = process_dict_updates(
@@ -202,7 +213,7 @@ class MinTokensLogitsProcessor(LogitsProcessor):
# Check for any requests that have attained their min tokens.
to_remove = tuple(
index
for index, (min_toks, out_tok_ids, _) in self.min_toks.items()
for index, (min_toks, out_tok_ids, _, _) in self.min_toks.items()
if len(out_tok_ids) >= min_toks
)
if to_remove:
@@ -214,22 +225,66 @@ class MinTokensLogitsProcessor(LogitsProcessor):
if needs_update:
reqs: list[int] = []
tok_ids: list[int] = []
for req, (_, _, stop_tok_ids) in self.min_toks.items():
restore_reqs: list[int] = []
restore_tok_ids: list[int] = []
for req, (
_,
_,
stop_tok_ids,
uses_structured_output,
) in self.min_toks.items():
reqs.extend([req] * len(stop_tok_ids))
tok_ids.extend(stop_tok_ids)
if uses_structured_output:
restore_reqs.extend([req] * len(stop_tok_ids))
restore_tok_ids.extend(stop_tok_ids)
self.logits_slice = (
self._device_tensor(reqs, torch.int32),
self._device_tensor(tok_ids, torch.int32),
)
self.restore_logits_slice = (
self._device_tensor(restore_reqs, torch.int32),
self._device_tensor(restore_tok_ids, torch.int32),
)
def _device_tensor(self, data: list, dtype: torch.dtype) -> torch.Tensor:
return async_tensor_h2d(data, device=self.device, dtype=dtype)
def _mask_stop_token_logits(
self,
logits: torch.Tensor,
logits_slice: tuple[torch.Tensor, torch.Tensor],
restore_logits_slice: tuple[torch.Tensor, torch.Tensor],
) -> None:
restore_rows, restore_toks = restore_logits_slice
# Advanced indexing already returns a copy, so this survives the
# in-place masking below.
stop_logits = logits[restore_logits_slice]
logits.index_put_(logits_slice, self.neg_inf_tensor)
# Stop-token entries for the same row are emitted together, so this
# checks each row once without sorting the index tensor.
unique_rows, inverse_indices = torch.unique_consecutive(
restore_rows, return_inverse=True
)
row_needs_restore = torch.isneginf(logits[unique_rows]).all(dim=-1)
restore_mask = row_needs_restore[inverse_indices] & torch.isfinite(stop_logits)
restore_slice = (
restore_rows[restore_mask],
restore_toks[restore_mask],
)
logits.index_put_(restore_slice, stop_logits[restore_mask])
def apply(self, logits: torch.Tensor) -> torch.Tensor:
if self.min_toks:
# Inhibit EOS token for requests which have not reached min length
logits.index_put_(self.logits_slice, self.neg_inf_tensor)
# Inhibit stop tokens for requests which have not reached min length.
if self.restore_logits_slice[0].numel() == 0:
logits.index_put_(self.logits_slice, self.neg_inf_tensor)
else:
self._mask_stop_token_logits(
logits, self.logits_slice, self.restore_logits_slice
)
return logits
def apply_with_spec_decode(
@@ -238,10 +293,13 @@ class MinTokensLogitsProcessor(LogitsProcessor):
num_draft_tokens: list[int],
) -> torch.Tensor:
"""Spec-decode version of apply().
Priority: ``min_tokens`` > ``stop_token_ids`` / EOS.
Stop tokens stay masked unless masking them would leave every logit
in a row at -inf.
Example: ``num_draft_tokens = [2, 3, 1]``
→ ``logits`` shape ``[6, V]``, ``cumsum = [0, 2, 5, 6]``
→ request 0 owns rows 0‑1, request 1 rows 2‑4, request 2 row 5.
-> ``logits`` shape ``[6, V]``, ``cumsum = [0, 2, 5, 6]``
-> request 0 owns rows 0-1, request 1 rows 2-4, request 2 row 5.
"""
if not self.min_toks:
return logits
@@ -250,8 +308,19 @@ class MinTokensLogitsProcessor(LogitsProcessor):
cumsum = np.concatenate([[0], np.cumsum(num_draft_arr)])
entries = [
(req_idx, min_tok, len(out_tok_ids), list(stop_tok_ids))
for req_idx, (min_tok, out_tok_ids, stop_tok_ids) in self.min_toks.items()
(
req_idx,
min_tok,
len(out_tok_ids),
list(stop_tok_ids),
uses_structured_output,
)
for req_idx, (
min_tok,
out_tok_ids,
stop_tok_ids,
uses_structured_output,
) in self.min_toks.items()
if stop_tok_ids
]
@@ -260,8 +329,10 @@ class MinTokensLogitsProcessor(LogitsProcessor):
all_rows: list[np.ndarray] = [] # row indices to mask
all_toks: list[np.ndarray] = [] # stop-token ids at those rows
restore_rows: list[np.ndarray] = []
restore_toks: list[np.ndarray] = []
for req_idx, min_tok, current_len, stop_toks in entries:
for req_idx, min_tok, current_len, stop_toks, uses_structured_output in entries:
remaining = min_tok - current_len
# How many leading draft positions still need stop-token masking.
n_mask = int(min(max(remaining, 0), num_draft_arr[req_idx]))
@@ -270,8 +341,13 @@ class MinTokensLogitsProcessor(LogitsProcessor):
offset = cumsum[req_idx]
row_indices = np.arange(offset, offset + n_mask, dtype=np.int64)
n_stop = len(stop_toks)
all_rows.append(np.repeat(row_indices, n_stop))
all_toks.append(np.tile(stop_toks, n_mask))
rows = np.repeat(row_indices, n_stop)
toks = np.tile(stop_toks, n_mask)
all_rows.append(rows)
all_toks.append(toks)
if uses_structured_output:
restore_rows.append(rows)
restore_toks.append(toks)
if all_rows:
rows_arr = np.concatenate(all_rows)
@@ -281,7 +357,16 @@ class MinTokensLogitsProcessor(LogitsProcessor):
async_tensor_h2d(rows_arr, device=self.device),
async_tensor_h2d(toks_arr, device=self.device),
)
logits.index_put_(logits_slice, self.neg_inf_tensor)
if restore_rows:
restore_rows_arr = np.concatenate(restore_rows)
restore_toks_arr = np.concatenate(restore_toks)
restore_logits_slice = (
async_tensor_h2d(restore_rows_arr, device=self.device),
async_tensor_h2d(restore_toks_arr, device=self.device),
)
self._mask_stop_token_logits(logits, logits_slice, restore_logits_slice)
else:
logits.index_put_(logits_slice, self.neg_inf_tensor)
return logits
+73 -5
View File
@@ -40,6 +40,9 @@ class LogitBiasState:
# Min tokens.
self.min_lens = UvaBackedTensor(self.max_num_reqs, dtype=torch.int32)
self.num_stop_token_ids = UvaBackedTensor(self.max_num_reqs, dtype=torch.int32)
self.restore_when_all_masked = UvaBackedTensor(
self.max_num_reqs, dtype=torch.int32
)
self.stop_token_ids = StagedWriteTensor(
(self.max_num_reqs, MAX_NUM_STOP_TOKEN_IDS),
dtype=torch.int32,
@@ -100,9 +103,13 @@ class LogitBiasState:
)
self.num_stop_token_ids.np[req_idx] = num_stop_token_ids
self.stop_token_ids.stage_write(req_idx, 0, stop_token_ids)
self.restore_when_all_masked.np[req_idx] = int(
sampling_params.structured_outputs is not None
)
use_logit_bias = True
else:
self.num_stop_token_ids.np[req_idx] = 0
self.restore_when_all_masked.np[req_idx] = 0
self.use_logit_bias[req_idx] = use_logit_bias
@@ -116,6 +123,7 @@ class LogitBiasState:
self.min_lens.copy_to_uva()
self.num_stop_token_ids.copy_to_uva()
self.restore_when_all_masked.copy_to_uva()
self.stop_token_ids.apply_write()
def apply_logit_bias(
@@ -129,6 +137,10 @@ class LogitBiasState:
# No request uses logit bias. Skip the kernel launch.
return
enable_stop_token_restore = bool(
np.any(self.restore_when_all_masked.np[idx_mapping_np])
)
apply_logit_bias(
logits,
expanded_idx_mapping,
@@ -140,7 +152,9 @@ class LogitBiasState:
self.logit_bias.gpu,
self.min_lens.gpu,
self.num_stop_token_ids.gpu,
self.restore_when_all_masked.gpu,
self.stop_token_ids.gpu,
enable_stop_token_restore,
)
@@ -164,10 +178,12 @@ def _bias_kernel(
pos_ptr,
min_lens_ptr,
num_stop_token_ids_ptr,
restore_when_all_masked_ptr,
stop_token_ids_ptr,
stop_token_ids_stride,
BLOCK_SIZE: tl.constexpr,
LOGITS_BLOCK_SIZE: tl.constexpr,
CHECK_ALL_MASKED_ROWS: tl.constexpr,
):
token_idx = tl.program_id(0).to(tl.int64)
req_state_idx = tl.load(expanded_idx_mapping_ptr + token_idx)
@@ -232,11 +248,59 @@ def _bias_kernel(
stop_token_ids_ptr + req_state_idx * stop_token_ids_stride + block,
mask=mask,
)
tl.store(
logits_ptr + token_idx * logits_stride + stop_token_ids,
-float("inf"),
mask=mask,
)
if CHECK_ALL_MASKED_ROWS:
should_restore_stop_logits = tl.load(
restore_when_all_masked_ptr + req_state_idx
)
if should_restore_stop_logits:
stop_logits = tl.load(
logits_ptr + token_idx * logits_stride + stop_token_ids,
mask=mask,
other=-float("inf"),
)
# Save must read original logits before the -inf overwrite.
tl.debug_barrier()
tl.store(
logits_ptr + token_idx * logits_stride + stop_token_ids,
-float("inf"),
mask=mask,
)
# The -inf overwrite must be visible to the row scan below.
tl.debug_barrier()
row_max = tl.full((), -float("inf"), tl.float32)
for i in range(0, vocab_size, LOGITS_BLOCK_SIZE):
offset = i + tl.arange(0, LOGITS_BLOCK_SIZE)
logits = tl.load(
logits_ptr + token_idx * logits_stride + offset,
mask=offset < vocab_size,
other=-float("inf"),
)
row_max = tl.maximum(row_max, tl.max(logits, axis=0))
if row_max == -float("inf"):
tl.store(
logits_ptr + token_idx * logits_stride + stop_token_ids,
stop_logits,
mask=mask
& (stop_logits > -float("inf"))
& (stop_logits < float("inf")),
)
else:
tl.store(
logits_ptr + token_idx * logits_stride + stop_token_ids,
-float("inf"),
mask=mask,
)
else:
tl.store(
logits_ptr + token_idx * logits_stride + stop_token_ids,
-float("inf"),
mask=mask,
)
def apply_logit_bias(
@@ -250,7 +314,9 @@ def apply_logit_bias(
logit_bias: torch.Tensor,
min_lens: torch.Tensor,
num_stop_token_ids: torch.Tensor,
restore_when_all_masked: torch.Tensor,
stop_token_ids: torch.Tensor,
check_all_masked_rows: bool = False,
) -> None:
num_tokens, vocab_size = logits.shape
BLOCK_SIZE = triton.next_power_of_2(
@@ -277,8 +343,10 @@ def apply_logit_bias(
pos,
min_lens,
num_stop_token_ids,
restore_when_all_masked,
stop_token_ids,
stop_token_ids.stride(0),
BLOCK_SIZE=BLOCK_SIZE,
LOGITS_BLOCK_SIZE=LOGITS_BLOCK_SIZE,
CHECK_ALL_MASKED_ROWS=check_all_masked_rows,
)