[Perf] DiffusionGemma: constrained reads over the request's logprob_token_ids (#58216)

Signed-off-by: Matt Mastracci <matthew@mastracci.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
Matt Mastracci
2026-09-24 13:15:08 -07:00
committed by GitHub
co-authored by Claude Fable 5.1 Lucas Wilkinson mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Misha Goin
parent 92e9c6bf13
commit ab3de6edf2
6 changed files with 262 additions and 6 deletions
+159 -5
View File
@@ -16,7 +16,7 @@ via Gemma4MultimodalEmbedder.
from __future__ import annotations
from collections.abc import Iterable, Mapping
from collections.abc import Iterable, Mapping, Sequence
from types import SimpleNamespace
from typing import Any
@@ -339,6 +339,17 @@ class DiffusionGemmaForConditionalGeneration(
)
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
states = getattr(self, "diffusion_states", None)
allowed = states.step_allowed if states is not None else None
if allowed is not None:
# One column per allowed id: a [rows, K] GEMM over the K gathered
# rows of the tied embedding.
logits = torch.nn.functional.linear(
hidden_states, self.lm_head.weight[allowed]
).float()
if self.final_logit_softcapping is not None:
logits = _softcap_logits(logits, self.final_logit_softcapping)
return logits
logits = self.logits_processor(self.lm_head, hidden_states)
if logits is not None and self.final_logit_softcapping is not None:
logits = _softcap_logits(logits, self.final_logit_softcapping)
@@ -395,6 +406,39 @@ def _concat_logprob_stashes(
)
_MASKED_LOGIT = -1e20
def _mask_rows_to_allowed(
logits: torch.Tensor,
row_starts: Sequence[int] | np.ndarray,
row_lens: Sequence[int] | np.ndarray,
allowed_per_row: Sequence[torch.Tensor | None],
) -> torch.Tensor:
"""Mask every column outside a request's allowed ids on that request's
rows. Request i owns rows [row_starts[i], +row_lens[i]); None leaves its
rows alone. Returns a copy when any row is masked, so the caller's tensor
(possibly the runner's) is never written.
Softmax over a masked row equals the K-space distribution the shared
fast path computes, so both paths give the same reads. The mask value is
a large finite negative rather than -inf: the entropy is probs times
log-probs, and 0 * -inf is NaN, while 0 * -1e20 is 0. It stays finite
after the greedy temperature clamp (1e-10) scales it by 1e10.
"""
out: torch.Tensor | None = None
keep = torch.zeros(logits.shape[-1], dtype=torch.bool, device=logits.device)
for start, n, allowed in zip(row_starts, row_lens, allowed_per_row):
if allowed is None:
continue
if out is None:
out = logits.clone()
keep.zero_()
keep[allowed] = True
out[int(start) : int(start) + int(n)].masked_fill_(~keep, _MASKED_LOGIT)
return logits if out is None else out
# Tiles specialize the graph on batch size 1, on a tile as wide as the canvas,
# on compute_sc and on sizes that coincide with the state buffers. That set
# is small but passes Dynamo's default of 8, after which every step would run
@@ -446,6 +490,10 @@ def _compiled_sample_step(
tp_size: int,
tp_group_name: str,
compute_sc: bool = True,
# Constrained reads: ``logits`` has one column per allowed id and
# ``embed_weight`` holds those ids' rows, so every argmax is an index
# into ``allowed``. ``vocab_size`` stays the real vocabulary for renoise.
allowed: torch.Tensor | None = None,
) -> torch.Tensor:
"""Compiled decode step: temperature → Gumbel sample → probs/confidence →
accept/renoise → convergence, all as vectorized PyTorch ops.
@@ -473,6 +521,9 @@ def _compiled_sample_step(
argmax_tokens = (
scaled.view(-1, scaled.shape[-1]).argmax(dim=-1).view(num_decode, CL)
)
if allowed is not None:
new_tokens = allowed[new_tokens]
argmax_tokens = allowed[argmax_tokens]
# ---- Phase 3: Probs, self-conditioning, confidence ----
log_probs = scaled.log_softmax(dim=-1)
@@ -508,6 +559,10 @@ def _compiled_sample_step(
step_tensor[decode_slots] = new_step_val
# Random tokens for renoise / canvas reinit
# Renoise draws from the whole vocabulary even for a constrained read.
# The model was trained on random-vocabulary noise. Noise drawn from the
# allowed set looks like garbled text, and later steps denoise it into
# more garbled text.
random_tokens = torch.randint(
0, vocab_size, (num_decode, CL), device=device, dtype=canvas.dtype
)
@@ -695,6 +750,12 @@ class DiffusionGemmaRequestStates:
# Read-only slots emit on their converging step and skip the commit
# forward.
self.read_only = torch.zeros(max_num_reqs, dtype=torch.bool, device=device)
# Constrained reads: slot -> the allowed ids (the request's
# logprob_token_ids). A step whose decode slots all share one set
# runs the unembedding, sampler and self-conditioning over that set.
self.constrained: dict[int, tuple[int, ...]] = {}
self.step_allowed: torch.Tensor | None = None
self._allowed_cache: dict[tuple[int, ...], torch.Tensor] = {}
self.read_only_slots: set[int] = set()
# Slots capped at one denoise step never consume a soft embed.
self.single_step_slots: set[int] = set()
@@ -738,6 +799,7 @@ class DiffusionGemmaRequestStates:
self.read_only[slot_idx].fill_(False)
self.read_only_slots.discard(slot_idx)
self.single_step_slots.discard(slot_idx)
self.constrained.pop(slot_idx, None)
self.canvas_width_np[slot_idx] = self.canvas_length
def remove_request(self, slot_idx: int) -> None:
@@ -749,6 +811,7 @@ class DiffusionGemmaRequestStates:
self.seeded_slots.discard(slot_idx)
self.read_only_slots.discard(slot_idx)
self.single_step_slots.discard(slot_idx)
self.constrained.pop(slot_idx, None)
def set_seed_canvas(self, slot_idx: int, ids: list[int]) -> None:
"""``ids`` covers the slot's canvas width; positions past it are never
@@ -767,6 +830,30 @@ class DiffusionGemmaRequestStates:
slot_idx, async_tensor_h2d(positions, dtype=torch.int64, device=self.device)
] = True
def allowed_tensor(self, ids: tuple[int, ...]) -> torch.Tensor:
"""``ids`` as an int64 tensor on the device, built once per tuple."""
t = self._allowed_cache.get(ids)
if t is None:
t = torch.tensor(ids, dtype=torch.int64, device=self.device)
if len(self._allowed_cache) > 64:
self._allowed_cache.clear()
self._allowed_cache[ids] = t
return t
def batch_allowed(self, slots: list[int]) -> torch.Tensor | None:
"""The allowed ids shared by every one of ``slots``, or None.
None means the step runs over the full vocabulary. The sampler then
masks each constrained slot's logit rows to its own set, so a
constrained request reads the same way whoever shares its batch.
"""
if not slots or not self.constrained:
return None
first = self.constrained.get(slots[0])
if first is None or any(self.constrained.get(s) != first for s in slots[1:]):
return None
return self.allowed_tensor(first)
def set_read_only(self, slot_idx: int) -> None:
self.read_only[slot_idx].fill_(True)
self.read_only_slots.add(slot_idx)
@@ -828,6 +915,8 @@ class DiffusionGemmaModelState(ModelState):
# the current step, so we add 1 to match the same behavior.
stability_threshold=self.gen_config["stability_threshold"] + 1,
)
# compute_logits reads the step's shared allowed set from here.
self.model.diffusion_states = self.diffusion_states
self._req_id_to_index: dict[str, int] = {}
# Persistent buffer for per-request causal flags, updated in-place
@@ -979,10 +1068,14 @@ class DiffusionGemmaModelState(ModelState):
inputs_embeds[:num_tokens].copy_(self.model.embed_input_ids(input_ids))
# Apply self-conditioning ONLY for denoising decode requests.
states.step_allowed = None
if input_batch.num_draft_tokens > 0 and self._req_id_to_index:
slots_np = input_batch.idx_mapping_np[:num_reqs]
num_logits_np = np.diff(input_batch.cu_num_logits_np[: num_reqs + 1])
is_decode_indices_np = np.where(num_logits_np > 0)[0]
states.step_allowed = states.batch_allowed(
slots_np[is_decode_indices_np].tolist()
)
self._apply_self_conditioning(
slots_np[is_decode_indices_np],
is_decode_indices_np,
@@ -1177,6 +1270,16 @@ class DiffusionSampler:
states.set_pins(req_idx, [int(p) for p in pins])
if extra.get("diffusion_read_only"):
states.set_read_only(req_idx)
if extra.get("diffusion_constrained"):
ids = list(getattr(sampling_params, "logprob_token_ids", None) or [])
if not ids:
raise ValueError(
"diffusion_constrained needs logprob_token_ids: they are "
"the allowed set."
)
if self.tp_size > 1:
raise ValueError("diffusion_constrained needs tensor parallel 1.")
states.constrained[req_idx] = tuple(int(t) for t in ids)
def apply_staged_writes(self) -> None:
self.sampling_states.apply_staged_writes()
@@ -1361,6 +1464,30 @@ class DiffusionSampler:
# since it mutates is_encoder_phase (commit→False, converge→True).
is_committing = states.is_encoder_phase[decode_slots].clone()
# Constrained step: logits are [rows, K] over the shared allowed set,
# so the self-conditioning matmul only needs those K embedding rows.
allowed = states.step_allowed
embed_weight = self.embed_weight
vocab_size = self.vocab_size
sc_vocab_start, sc_vocab_end = self.sc_vocab_start, self.sc_vocab_end
if allowed is not None:
assert logits.shape[-1] == allowed.numel()
embed_weight = self.embed_weight[allowed]
sc_vocab_start, sc_vocab_end = 0, allowed.numel()
elif states.constrained and num_decode > 0:
# The decode slots do not share one set, so this step runs over
# the full vocabulary. Mask each constrained request's rows to
# its own set so it reads the same as on the shared path. The
# helper masks a copy, since the runner owns `logits`: one
# full-vocab copy per step, as for top_k/top_p above.
per_row = [
None if ids is None else states.allowed_tensor(ids)
for ids in (states.constrained.get(s) for s in decode_slots_np.tolist())
]
logits = _mask_rows_to_allowed(
logits, row_starts_np, valid_canvas_len_np, per_row
)
slots_np = input_batch.idx_mapping_np[:num_reqs]
max_num_logprobs = self.sampling_states.max_num_logprobs(slots_np)
# Requests may ask for specific token ids' logprobs instead of, or as
@@ -1431,7 +1558,7 @@ class DiffusionSampler:
states.is_encoder_phase,
states.confident,
states.self_conditioning_embeds[:, :W],
self.embed_weight,
embed_weight,
self.normalizer,
states.accepted_canvas_history[:, :, :W],
states.accepted_canvas_history_len,
@@ -1448,15 +1575,16 @@ class DiffusionSampler:
t_min=self.t_min,
t_max=self.t_max,
confidence_threshold=self.confidence_threshold,
vocab_size=self.vocab_size,
vocab_size=vocab_size,
CL=W,
ST=states.stability_threshold,
entropy_bound=self.entropy_bound,
sc_vocab_start=self.sc_vocab_start,
sc_vocab_end=self.sc_vocab_end,
sc_vocab_start=sc_vocab_start,
sc_vocab_end=sc_vocab_end,
tp_size=self.tp_size,
tp_group_name=self.tp_group_name,
compute_sc=compute_sc,
allowed=allowed,
)
# Stash newly converged logprobs, including reads that emit now.
@@ -1483,6 +1611,32 @@ class DiffusionSampler:
if raw_flat is None:
raw_flat = tile_logits.float()
src = raw_flat
if allowed is not None:
# Column j of the K-space logits is allowed[j]:
# report every allowed id, renormalized over
# the set, with the argmax in column 0.
rows = src[pos : pos + k_i]
lp = rows.log_softmax(dim=-1)
am = argmax_tokens[local_idx][:k_i]
sel_lp = lp.gather(1, am.unsqueeze(1))
# Same convention as _ranks_kernel: 1-based,
# ties count, so the rank is the number of
# columns at or above the selected one. Out
# of set ids have zero probability, so the
# rank within the set is the real rank.
ranks = (lp >= sel_lp).sum(dim=1)
self._pending_logprobs[slot] = LogprobsTensors(
logprob_token_ids=torch.cat(
(
allowed[am].unsqueeze(1),
allowed.unsqueeze(0).expand(k_i, -1),
),
dim=1,
),
logprobs=torch.cat((sel_lp, lp), dim=1),
selected_token_ranks=ranks.to(torch.int64),
)
continue
per_req_ids = max_token_ids > 0
self._pending_logprobs[slot] = compute_topk_scores(
src[pos : pos + k_i],