[Perf] DiffusionGemma: constrained reads over the request's logprob_token_ids (#58216)

Signed-off-by: Matt Mastracci <matthew@mastracci.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
Matt Mastracci
2026-09-24 13:15:08 -07:00
committed by GitHub
co-authored by Claude Fable 5.1 Lucas Wilkinson mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Misha Goin
parent 92e9c6bf13
commit ab3de6edf2
6 changed files with 262 additions and 6 deletions
@@ -11,8 +11,10 @@ left as noise, one denoise step gives a distribution over each slot. These
| `diffusion_pinned` | `list[int]` of canvas positions | held at their seed value on every denoise step, so a read past one step keeps its template | | `diffusion_pinned` | `list[int]` of canvas positions | held at their seed value on every denoise step, so a read past one step keeps its template |
| `diffusion_max_steps` | `int` | denoise steps before the canvas is emitted | | `diffusion_max_steps` | `int` | denoise steps before the canvas is emitted |
| `diffusion_read_only` | `bool` | emit the argmax canvas as soon as the cap is reached, end the request there, and return temperature-1 logprobs at every position | | `diffusion_read_only` | `bool` | emit the argmax canvas as soon as the cap is reached, end the request there, and return temperature-1 logprobs at every position |
| `diffusion_constrained` | `bool` | run the unembedding, sampler and self-conditioning over the request's `logprob_token_ids` only. Logprobs are normalized over that set. A step uses this only when every read in it has the same set |
`structured_server.py` turns a question schema into those fields. It serves `structured_server.py` turns a question schema into those fields, with
`diffusion_constrained` on for every read (`--no-constrained` turns it off). It serves
`/v1/chat/completions`: the system message is the schema, the user message `/v1/chat/completions`: the system message is the schema, the user message
is the state JSON, and the reply content is one distribution per question is the state JSON, and the reply content is one distribution per question
with a standard error over a few noise draws. with a standard error over a few noise draws.
@@ -399,6 +399,14 @@ def canvas_width(template):
return min(CANVAS_LEN, -(-need // CANVAS_STEP) * CANVAS_STEP) return min(CANVAS_LEN, -(-need // CANVAS_STEP) * CANVAS_STEP)
def constrained_xargs():
"""Read over the labels only. The engine runs the unembedding, sampler and
self-conditioning over the request's logprob_token_ids instead of the whole
vocabulary. Same argmax, about a quarter less GPU time per read. Logprobs
come back normalized over the labels."""
return {"diffusion_constrained": True} if ARGS.constrained else {}
def pin_xargs(template, slots, steps): def pin_xargs(template, slots, steps):
"""Past one denoise step the template must be held, or accept/renoise """Past one denoise step the template must be held, or accept/renoise
rewrites it: pin every canvas position that is not an answer slot.""" rewrites it: pin every canvas position that is not an answer slot."""
@@ -552,6 +560,7 @@ def one_read(
"diffusion_max_steps": schema["steps"], "diffusion_max_steps": schema["steps"],
"diffusion_read_only": True, "diffusion_read_only": True,
**pin_xargs(template, slots, schema["steps"]), **pin_xargs(template, slots, schema["steps"]),
**constrained_xargs(),
}, },
} }
d = upstream_chat(body) d = upstream_chat(body)
@@ -601,6 +610,7 @@ def one_read_continuation(schema, template, slots, prompt_ids, seed):
"diffusion_max_steps": schema["steps"], "diffusion_max_steps": schema["steps"],
"diffusion_read_only": True, "diffusion_read_only": True,
**pin_xargs(template, slots, schema["steps"]), **pin_xargs(template, slots, schema["steps"]),
**constrained_xargs(),
}, },
} }
d = upstream_completions(body) d = upstream_completions(body)
@@ -1496,6 +1506,12 @@ def main():
default=16, default=16,
help="request widths round up to a multiple of this", help="request widths round up to a multiple of this",
) )
p.add_argument(
"--no-constrained",
dest="constrained",
action="store_false",
help="read over the whole vocabulary instead of the labels",
)
p.add_argument("--host", default="0.0.0.0") p.add_argument("--host", default="0.0.0.0")
p.add_argument("--port", type=int, default=8011) p.add_argument("--port", type=int, default=8011)
p.add_argument( p.add_argument(
+23
View File
@@ -145,6 +145,29 @@ def test_diffusion_rejects_bad_extra_args(extra_args: dict, match: str):
_verify_diffusion(SamplingParams(extra_args=extra_args)) _verify_diffusion(SamplingParams(extra_args=extra_args))
@pytest.mark.parametrize("flag", [True, 1])
def test_diffusion_constrained_needs_logprob_token_ids(flag):
bad = SamplingParams(extra_args={"diffusion_constrained": flag})
with pytest.raises(VLLMValidationError, match="needs logprob_token_ids"):
_verify_diffusion(bad)
ok = SamplingParams(
logprob_token_ids=[3, 5], extra_args={"diffusion_constrained": flag}
)
_verify_diffusion(ok)
# An unset or false flag needs no ids.
_verify_diffusion(SamplingParams(extra_args={"diffusion_constrained": False}))
_verify_diffusion(SamplingParams(extra_args={"diffusion_constrained": 0}))
@pytest.mark.parametrize("value", ["yes", 2, [1]])
def test_diffusion_constrained_must_be_a_bool(value):
params = SamplingParams(extra_args={"diffusion_constrained": value})
with pytest.raises(VLLMValidationError, match="must be a boolean"):
_verify_diffusion(params)
def test_diffusion_seed_canvas_must_fill_the_canvas(): def test_diffusion_seed_canvas_must_fill_the_canvas():
params = SamplingParams(extra_args={"diffusion_seed_canvas": [0] * 7}) params = SamplingParams(extra_args={"diffusion_seed_canvas": [0] * 7})
with pytest.raises(VLLMValidationError, match="exactly 8 ids, got 7"): with pytest.raises(VLLMValidationError, match="exactly 8 ids, got 7"):
@@ -8,9 +8,11 @@ import pytest
import torch import torch
from vllm.model_executor.models.diffusion_gemma import ( from vllm.model_executor.models.diffusion_gemma import (
_MASKED_LOGIT,
DiffusionGemmaRequestStates, DiffusionGemmaRequestStates,
_compiled_sample_step, _compiled_sample_step,
_concat_logprob_stashes, _concat_logprob_stashes,
_mask_rows_to_allowed,
) )
from vllm.platforms import current_platform from vllm.platforms import current_platform
from vllm.v1.outputs import LogprobsTensors from vllm.v1.outputs import LogprobsTensors
@@ -317,3 +319,50 @@ def test_read_emits_at_convergence_while_generation_waits_for_commit(width, step
sampled, counts = _denoise_once(states, [0], width=width) sampled, counts = _denoise_once(states, [0], width=width)
assert counts.tolist() == [width] assert counts.tolist() == [width]
assert torch.equal(sampled[0], states.argmax_canvas[0, :width].int()) assert torch.equal(sampled[0], states.argmax_canvas[0, :width].int())
def test_batch_allowed_needs_one_shared_set():
states = _states()
states.constrained[0] = (1, 2, 3)
states.constrained[1] = (4, 5)
# Mixed sets, or a slot with no set, fall back to per-row masking.
assert states.batch_allowed([0, 1]) is None
assert states.batch_allowed([0, 2]) is None
assert states.batch_allowed([]) is None
shared = states.batch_allowed([0])
assert shared.tolist() == [1, 2, 3]
assert shared.dtype == torch.int64
assert states.batch_allowed([0, 0]) is shared
assert states.allowed_tensor((1, 2, 3)) is shared
def test_mask_rows_to_allowed_masks_only_constrained_rows():
logits = torch.randn(5, VOCAB, device="cuda")
before = logits.clone()
first = torch.tensor([3, 7], device="cuda")
third = torch.tensor([0], device="cuda")
out = _mask_rows_to_allowed(logits, [0, 2, 3], [2, 1, 2], [first, None, third])
assert out is not logits
assert torch.equal(logits, before)
# Masked columns hold the finite sentinel, so entropy stays finite.
kept = out > _MASKED_LOGIT
assert torch.isfinite(out).all()
assert kept[0:2].sum(dim=1).tolist() == [2, 2]
assert kept[0:2][:, first].all()
assert torch.equal(out[0:2][:, first], before[0:2][:, first])
assert torch.equal(out[2], before[2])
assert kept[3:5].sum(dim=1).tolist() == [1, 1]
assert torch.equal(out[3:5][:, third], before[3:5][:, third])
# The masked row's softmax is the distribution renormalized over the set.
torch.testing.assert_close(
out[0].softmax(dim=-1)[first], before[0, first].softmax(dim=-1)
)
def test_mask_rows_to_allowed_is_a_no_op_without_constrained_rows():
logits = torch.randn(3, VOCAB, device="cuda")
assert _mask_rows_to_allowed(logits, [0, 1], [1, 2], [None, None]) is logits
+159 -5
View File
@@ -16,7 +16,7 @@ via Gemma4MultimodalEmbedder.
from __future__ import annotations from __future__ import annotations
from collections.abc import Iterable, Mapping from collections.abc import Iterable, Mapping, Sequence
from types import SimpleNamespace from types import SimpleNamespace
from typing import Any from typing import Any
@@ -339,6 +339,17 @@ class DiffusionGemmaForConditionalGeneration(
) )
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None: def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
states = getattr(self, "diffusion_states", None)
allowed = states.step_allowed if states is not None else None
if allowed is not None:
# One column per allowed id: a [rows, K] GEMM over the K gathered
# rows of the tied embedding.
logits = torch.nn.functional.linear(
hidden_states, self.lm_head.weight[allowed]
).float()
if self.final_logit_softcapping is not None:
logits = _softcap_logits(logits, self.final_logit_softcapping)
return logits
logits = self.logits_processor(self.lm_head, hidden_states) logits = self.logits_processor(self.lm_head, hidden_states)
if logits is not None and self.final_logit_softcapping is not None: if logits is not None and self.final_logit_softcapping is not None:
logits = _softcap_logits(logits, self.final_logit_softcapping) logits = _softcap_logits(logits, self.final_logit_softcapping)
@@ -395,6 +406,39 @@ def _concat_logprob_stashes(
) )
_MASKED_LOGIT = -1e20
def _mask_rows_to_allowed(
logits: torch.Tensor,
row_starts: Sequence[int] | np.ndarray,
row_lens: Sequence[int] | np.ndarray,
allowed_per_row: Sequence[torch.Tensor | None],
) -> torch.Tensor:
"""Mask every column outside a request's allowed ids on that request's
rows. Request i owns rows [row_starts[i], +row_lens[i]); None leaves its
rows alone. Returns a copy when any row is masked, so the caller's tensor
(possibly the runner's) is never written.
Softmax over a masked row equals the K-space distribution the shared
fast path computes, so both paths give the same reads. The mask value is
a large finite negative rather than -inf: the entropy is probs times
log-probs, and 0 * -inf is NaN, while 0 * -1e20 is 0. It stays finite
after the greedy temperature clamp (1e-10) scales it by 1e10.
"""
out: torch.Tensor | None = None
keep = torch.zeros(logits.shape[-1], dtype=torch.bool, device=logits.device)
for start, n, allowed in zip(row_starts, row_lens, allowed_per_row):
if allowed is None:
continue
if out is None:
out = logits.clone()
keep.zero_()
keep[allowed] = True
out[int(start) : int(start) + int(n)].masked_fill_(~keep, _MASKED_LOGIT)
return logits if out is None else out
# Tiles specialize the graph on batch size 1, on a tile as wide as the canvas, # Tiles specialize the graph on batch size 1, on a tile as wide as the canvas,
# on compute_sc and on sizes that coincide with the state buffers. That set # on compute_sc and on sizes that coincide with the state buffers. That set
# is small but passes Dynamo's default of 8, after which every step would run # is small but passes Dynamo's default of 8, after which every step would run
@@ -446,6 +490,10 @@ def _compiled_sample_step(
tp_size: int, tp_size: int,
tp_group_name: str, tp_group_name: str,
compute_sc: bool = True, compute_sc: bool = True,
# Constrained reads: ``logits`` has one column per allowed id and
# ``embed_weight`` holds those ids' rows, so every argmax is an index
# into ``allowed``. ``vocab_size`` stays the real vocabulary for renoise.
allowed: torch.Tensor | None = None,
) -> torch.Tensor: ) -> torch.Tensor:
"""Compiled decode step: temperature → Gumbel sample → probs/confidence → """Compiled decode step: temperature → Gumbel sample → probs/confidence →
accept/renoise → convergence, all as vectorized PyTorch ops. accept/renoise → convergence, all as vectorized PyTorch ops.
@@ -473,6 +521,9 @@ def _compiled_sample_step(
argmax_tokens = ( argmax_tokens = (
scaled.view(-1, scaled.shape[-1]).argmax(dim=-1).view(num_decode, CL) scaled.view(-1, scaled.shape[-1]).argmax(dim=-1).view(num_decode, CL)
) )
if allowed is not None:
new_tokens = allowed[new_tokens]
argmax_tokens = allowed[argmax_tokens]
# ---- Phase 3: Probs, self-conditioning, confidence ---- # ---- Phase 3: Probs, self-conditioning, confidence ----
log_probs = scaled.log_softmax(dim=-1) log_probs = scaled.log_softmax(dim=-1)
@@ -508,6 +559,10 @@ def _compiled_sample_step(
step_tensor[decode_slots] = new_step_val step_tensor[decode_slots] = new_step_val
# Random tokens for renoise / canvas reinit # Random tokens for renoise / canvas reinit
# Renoise draws from the whole vocabulary even for a constrained read.
# The model was trained on random-vocabulary noise. Noise drawn from the
# allowed set looks like garbled text, and later steps denoise it into
# more garbled text.
random_tokens = torch.randint( random_tokens = torch.randint(
0, vocab_size, (num_decode, CL), device=device, dtype=canvas.dtype 0, vocab_size, (num_decode, CL), device=device, dtype=canvas.dtype
) )
@@ -695,6 +750,12 @@ class DiffusionGemmaRequestStates:
# Read-only slots emit on their converging step and skip the commit # Read-only slots emit on their converging step and skip the commit
# forward. # forward.
self.read_only = torch.zeros(max_num_reqs, dtype=torch.bool, device=device) self.read_only = torch.zeros(max_num_reqs, dtype=torch.bool, device=device)
# Constrained reads: slot -> the allowed ids (the request's
# logprob_token_ids). A step whose decode slots all share one set
# runs the unembedding, sampler and self-conditioning over that set.
self.constrained: dict[int, tuple[int, ...]] = {}
self.step_allowed: torch.Tensor | None = None
self._allowed_cache: dict[tuple[int, ...], torch.Tensor] = {}
self.read_only_slots: set[int] = set() self.read_only_slots: set[int] = set()
# Slots capped at one denoise step never consume a soft embed. # Slots capped at one denoise step never consume a soft embed.
self.single_step_slots: set[int] = set() self.single_step_slots: set[int] = set()
@@ -738,6 +799,7 @@ class DiffusionGemmaRequestStates:
self.read_only[slot_idx].fill_(False) self.read_only[slot_idx].fill_(False)
self.read_only_slots.discard(slot_idx) self.read_only_slots.discard(slot_idx)
self.single_step_slots.discard(slot_idx) self.single_step_slots.discard(slot_idx)
self.constrained.pop(slot_idx, None)
self.canvas_width_np[slot_idx] = self.canvas_length self.canvas_width_np[slot_idx] = self.canvas_length
def remove_request(self, slot_idx: int) -> None: def remove_request(self, slot_idx: int) -> None:
@@ -749,6 +811,7 @@ class DiffusionGemmaRequestStates:
self.seeded_slots.discard(slot_idx) self.seeded_slots.discard(slot_idx)
self.read_only_slots.discard(slot_idx) self.read_only_slots.discard(slot_idx)
self.single_step_slots.discard(slot_idx) self.single_step_slots.discard(slot_idx)
self.constrained.pop(slot_idx, None)
def set_seed_canvas(self, slot_idx: int, ids: list[int]) -> None: def set_seed_canvas(self, slot_idx: int, ids: list[int]) -> None:
"""``ids`` covers the slot's canvas width; positions past it are never """``ids`` covers the slot's canvas width; positions past it are never
@@ -767,6 +830,30 @@ class DiffusionGemmaRequestStates:
slot_idx, async_tensor_h2d(positions, dtype=torch.int64, device=self.device) slot_idx, async_tensor_h2d(positions, dtype=torch.int64, device=self.device)
] = True ] = True
def allowed_tensor(self, ids: tuple[int, ...]) -> torch.Tensor:
"""``ids`` as an int64 tensor on the device, built once per tuple."""
t = self._allowed_cache.get(ids)
if t is None:
t = torch.tensor(ids, dtype=torch.int64, device=self.device)
if len(self._allowed_cache) > 64:
self._allowed_cache.clear()
self._allowed_cache[ids] = t
return t
def batch_allowed(self, slots: list[int]) -> torch.Tensor | None:
"""The allowed ids shared by every one of ``slots``, or None.
None means the step runs over the full vocabulary. The sampler then
masks each constrained slot's logit rows to its own set, so a
constrained request reads the same way whoever shares its batch.
"""
if not slots or not self.constrained:
return None
first = self.constrained.get(slots[0])
if first is None or any(self.constrained.get(s) != first for s in slots[1:]):
return None
return self.allowed_tensor(first)
def set_read_only(self, slot_idx: int) -> None: def set_read_only(self, slot_idx: int) -> None:
self.read_only[slot_idx].fill_(True) self.read_only[slot_idx].fill_(True)
self.read_only_slots.add(slot_idx) self.read_only_slots.add(slot_idx)
@@ -828,6 +915,8 @@ class DiffusionGemmaModelState(ModelState):
# the current step, so we add 1 to match the same behavior. # the current step, so we add 1 to match the same behavior.
stability_threshold=self.gen_config["stability_threshold"] + 1, stability_threshold=self.gen_config["stability_threshold"] + 1,
) )
# compute_logits reads the step's shared allowed set from here.
self.model.diffusion_states = self.diffusion_states
self._req_id_to_index: dict[str, int] = {} self._req_id_to_index: dict[str, int] = {}
# Persistent buffer for per-request causal flags, updated in-place # Persistent buffer for per-request causal flags, updated in-place
@@ -979,10 +1068,14 @@ class DiffusionGemmaModelState(ModelState):
inputs_embeds[:num_tokens].copy_(self.model.embed_input_ids(input_ids)) inputs_embeds[:num_tokens].copy_(self.model.embed_input_ids(input_ids))
# Apply self-conditioning ONLY for denoising decode requests. # Apply self-conditioning ONLY for denoising decode requests.
states.step_allowed = None
if input_batch.num_draft_tokens > 0 and self._req_id_to_index: if input_batch.num_draft_tokens > 0 and self._req_id_to_index:
slots_np = input_batch.idx_mapping_np[:num_reqs] slots_np = input_batch.idx_mapping_np[:num_reqs]
num_logits_np = np.diff(input_batch.cu_num_logits_np[: num_reqs + 1]) num_logits_np = np.diff(input_batch.cu_num_logits_np[: num_reqs + 1])
is_decode_indices_np = np.where(num_logits_np > 0)[0] is_decode_indices_np = np.where(num_logits_np > 0)[0]
states.step_allowed = states.batch_allowed(
slots_np[is_decode_indices_np].tolist()
)
self._apply_self_conditioning( self._apply_self_conditioning(
slots_np[is_decode_indices_np], slots_np[is_decode_indices_np],
is_decode_indices_np, is_decode_indices_np,
@@ -1177,6 +1270,16 @@ class DiffusionSampler:
states.set_pins(req_idx, [int(p) for p in pins]) states.set_pins(req_idx, [int(p) for p in pins])
if extra.get("diffusion_read_only"): if extra.get("diffusion_read_only"):
states.set_read_only(req_idx) states.set_read_only(req_idx)
if extra.get("diffusion_constrained"):
ids = list(getattr(sampling_params, "logprob_token_ids", None) or [])
if not ids:
raise ValueError(
"diffusion_constrained needs logprob_token_ids: they are "
"the allowed set."
)
if self.tp_size > 1:
raise ValueError("diffusion_constrained needs tensor parallel 1.")
states.constrained[req_idx] = tuple(int(t) for t in ids)
def apply_staged_writes(self) -> None: def apply_staged_writes(self) -> None:
self.sampling_states.apply_staged_writes() self.sampling_states.apply_staged_writes()
@@ -1361,6 +1464,30 @@ class DiffusionSampler:
# since it mutates is_encoder_phase (commit→False, converge→True). # since it mutates is_encoder_phase (commit→False, converge→True).
is_committing = states.is_encoder_phase[decode_slots].clone() is_committing = states.is_encoder_phase[decode_slots].clone()
# Constrained step: logits are [rows, K] over the shared allowed set,
# so the self-conditioning matmul only needs those K embedding rows.
allowed = states.step_allowed
embed_weight = self.embed_weight
vocab_size = self.vocab_size
sc_vocab_start, sc_vocab_end = self.sc_vocab_start, self.sc_vocab_end
if allowed is not None:
assert logits.shape[-1] == allowed.numel()
embed_weight = self.embed_weight[allowed]
sc_vocab_start, sc_vocab_end = 0, allowed.numel()
elif states.constrained and num_decode > 0:
# The decode slots do not share one set, so this step runs over
# the full vocabulary. Mask each constrained request's rows to
# its own set so it reads the same as on the shared path. The
# helper masks a copy, since the runner owns `logits`: one
# full-vocab copy per step, as for top_k/top_p above.
per_row = [
None if ids is None else states.allowed_tensor(ids)
for ids in (states.constrained.get(s) for s in decode_slots_np.tolist())
]
logits = _mask_rows_to_allowed(
logits, row_starts_np, valid_canvas_len_np, per_row
)
slots_np = input_batch.idx_mapping_np[:num_reqs] slots_np = input_batch.idx_mapping_np[:num_reqs]
max_num_logprobs = self.sampling_states.max_num_logprobs(slots_np) max_num_logprobs = self.sampling_states.max_num_logprobs(slots_np)
# Requests may ask for specific token ids' logprobs instead of, or as # Requests may ask for specific token ids' logprobs instead of, or as
@@ -1431,7 +1558,7 @@ class DiffusionSampler:
states.is_encoder_phase, states.is_encoder_phase,
states.confident, states.confident,
states.self_conditioning_embeds[:, :W], states.self_conditioning_embeds[:, :W],
self.embed_weight, embed_weight,
self.normalizer, self.normalizer,
states.accepted_canvas_history[:, :, :W], states.accepted_canvas_history[:, :, :W],
states.accepted_canvas_history_len, states.accepted_canvas_history_len,
@@ -1448,15 +1575,16 @@ class DiffusionSampler:
t_min=self.t_min, t_min=self.t_min,
t_max=self.t_max, t_max=self.t_max,
confidence_threshold=self.confidence_threshold, confidence_threshold=self.confidence_threshold,
vocab_size=self.vocab_size, vocab_size=vocab_size,
CL=W, CL=W,
ST=states.stability_threshold, ST=states.stability_threshold,
entropy_bound=self.entropy_bound, entropy_bound=self.entropy_bound,
sc_vocab_start=self.sc_vocab_start, sc_vocab_start=sc_vocab_start,
sc_vocab_end=self.sc_vocab_end, sc_vocab_end=sc_vocab_end,
tp_size=self.tp_size, tp_size=self.tp_size,
tp_group_name=self.tp_group_name, tp_group_name=self.tp_group_name,
compute_sc=compute_sc, compute_sc=compute_sc,
allowed=allowed,
) )
# Stash newly converged logprobs, including reads that emit now. # Stash newly converged logprobs, including reads that emit now.
@@ -1483,6 +1611,32 @@ class DiffusionSampler:
if raw_flat is None: if raw_flat is None:
raw_flat = tile_logits.float() raw_flat = tile_logits.float()
src = raw_flat src = raw_flat
if allowed is not None:
# Column j of the K-space logits is allowed[j]:
# report every allowed id, renormalized over
# the set, with the argmax in column 0.
rows = src[pos : pos + k_i]
lp = rows.log_softmax(dim=-1)
am = argmax_tokens[local_idx][:k_i]
sel_lp = lp.gather(1, am.unsqueeze(1))
# Same convention as _ranks_kernel: 1-based,
# ties count, so the rank is the number of
# columns at or above the selected one. Out
# of set ids have zero probability, so the
# rank within the set is the real rank.
ranks = (lp >= sel_lp).sum(dim=1)
self._pending_logprobs[slot] = LogprobsTensors(
logprob_token_ids=torch.cat(
(
allowed[am].unsqueeze(1),
allowed.unsqueeze(0).expand(k_i, -1),
),
dim=1,
),
logprobs=torch.cat((sel_lp, lp), dim=1),
selected_token_ranks=ranks.to(torch.int64),
)
continue
per_req_ids = max_token_ids > 0 per_req_ids = max_token_ids > 0
self._pending_logprobs[slot] = compute_topk_scores( self._pending_logprobs[slot] = compute_topk_scores(
src[pos : pos + k_i], src[pos : pos + k_i],
+12
View File
@@ -94,6 +94,18 @@ def validate_diffusion_sampling_params(
params.max_tokens = min(params.max_tokens or expected_len, expected_len) params.max_tokens = min(params.max_tokens or expected_len, expected_len)
params.ignore_eos = True params.ignore_eos = True
constrained = extra.get("diffusion_constrained")
if constrained is not None:
check(
isinstance(constrained, (bool, int)) and constrained in (0, 1),
"diffusion_constrained must be a boolean (or 0/1).",
)
if constrained:
check(
bool(params.logprob_token_ids),
"diffusion_constrained needs logprob_token_ids: they are the allowed set.",
)
check( check(
bool(async_scheduling) bool(async_scheduling)
or width is None or width is None