[Core] structured generation mode for DiffusionGemma model (Jev-like) (#57250)

Signed-off-by: Matt Mastracci <matthew@mastracci.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Razorback16 <razorback16@protonmail.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Razorback16 <razorback16@protonmail.com>
This commit is contained in:
Matt Mastracci
2026-09-22 20:51:17 +00:00
committed by GitHub
co-authored by Claude Fable 5.1 Lucas Wilkinson Codex Razorback16
parent fae5cbfc8b
commit 1b3b88ec2b
16 changed files with 2795 additions and 155 deletions
+25
View File
@@ -15,8 +15,10 @@ from transformers.models.gemma4.configuration_gemma4 import Gemma4TextConfig
from vllm.config import ModelConfig, ParallelConfig, SpeculativeConfig
from vllm.config.model_arch import ModelArchitectureConfig
from vllm.transformers_utils.configs.diffusion_gemma import DiffusionGemmaConfig
from vllm.transformers_utils.configs.gemma4 import gemma4_layer_config
from vllm.transformers_utils.model_arch_config_convertor import (
MODEL_ARCH_CONFIG_CONVERTORS,
Gemma4ModelArchConfigConvertor,
ModelArchConfigConvertorBase,
)
@@ -496,3 +498,26 @@ def test_deepseek_v4_convertor_splits_vision_architecture():
conv = DeepseekV4ModelArchConfigConvertor(draft_cfg, draft_cfg)
assert conv.get_architectures() == [draft_arch]
assert draft_cfg.architectures == [draft_arch]
def test_diffusion_gemma_head_dims_vary_by_layer_type():
"""Checkpoints carry model_type `diffusion_gemma` at the top level. Under
the base convertor every layer reports the sliding head dim, Gemma4Config
skips its head-dim handling, and the full-attention KV-cache group falls to
FlashInfer, which cannot take a per-request causal tensor."""
text_config = dict(
num_hidden_layers=6,
hidden_size=64,
num_attention_heads=8,
num_key_value_heads=4,
head_dim=16,
global_head_dim=32,
layer_types=["sliding_attention"] * 5 + ["full_attention"],
)
hf_config = DiffusionGemmaConfig(text_config=text_config)
convertor_cls = MODEL_ARCH_CONFIG_CONVERTORS[hf_config.model_type]
assert convertor_cls is Gemma4ModelArchConfigConvertor
arch = convertor_cls(hf_config, hf_config.text_config).convert()
assert [arch[i].head_size for i in range(6)] == [16] * 5 + [32]
+182
View File
@@ -1,11 +1,15 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from types import SimpleNamespace
import pytest
from vllm import SamplingParams
from vllm.config.diffusion import DiffusionConfig
from vllm.exceptions import VLLMValidationError
from vllm.utils.diffusion import validate_diffusion_sampling_params
from vllm.v1.engine.input_processor import InputProcessor
@dataclass
@@ -13,6 +17,7 @@ class MockModelConfig:
is_diffusion: bool = False
max_logprobs: int = 20
logits_processors: list | None = None
return_sampling_mask: bool = False
def get_vocab_size(self) -> int:
return 1024
@@ -59,6 +64,183 @@ def test_verify_leaves_logits_processors_to_admission():
)
def _verify_diffusion(params: SamplingParams, canvas_length: int | None = None):
model_config = MockModelConfig(is_diffusion=True)
params.verify(model_config, None, None, None)
validate_diffusion_sampling_params(
params,
canvas_length=canvas_length,
vocab_size=model_config.get_vocab_size(),
async_scheduling=True,
)
def test_diffusion_extra_args_are_validated_without_a_served_canvas():
# No --diffusion-config: the canvas is unknown, the ids are still checked
# and a read-only request is still normalised.
params = SamplingParams(
max_tokens=64,
extra_args={"diffusion_seed_canvas": [0, 1], "diffusion_read_only": True},
)
_verify_diffusion(params, canvas_length=None)
assert params.ignore_eos is True
bad = SamplingParams(extra_args={"diffusion_seed_canvas": [0, 10**9]})
with pytest.raises(VLLMValidationError, match="ids must be in"):
_verify_diffusion(bad, canvas_length=None)
@pytest.mark.parametrize("async_scheduling", [False, True])
@pytest.mark.parametrize(
"extra_args",
[
{},
{"diffusion_canvas_length": None},
{"diffusion_canvas_length": 4},
{"diffusion_canvas_length": 8},
],
)
def test_narrow_diffusion_canvas_requires_async_scheduling(
async_scheduling, extra_args
):
processor = SimpleNamespace(
model_config=MockModelConfig(is_diffusion=True),
vllm_config=SimpleNamespace(
scheduler_config=SimpleNamespace(async_scheduling=async_scheduling)
),
speculative_config=None,
structured_outputs_config=None,
diffusion_config=DiffusionConfig(canvas_length=8),
tokenizer=None,
validate_logits_processors_params=lambda params: None,
)
params = SamplingParams(extra_args=extra_args)
if not async_scheduling and extra_args.get("diffusion_canvas_length") == 4:
with pytest.raises(VLLMValidationError, match="requires --async-scheduling"):
InputProcessor._validate_params(processor, params, ("generate",))
else:
InputProcessor._validate_params(processor, params, ("generate",))
@pytest.mark.parametrize(
"extra_args, match",
[
({"diffusion_seed_canvas": "abc"}, "list of token ids"),
({"diffusion_seed_canvas": [1, 2.5]}, "list of token ids"),
({"diffusion_seed_canvas": [1, True]}, "list of token ids"),
({"diffusion_seed_canvas": [0, 1024]}, r"in \[0, 1024\)"),
({"diffusion_seed_canvas": [0, -1]}, r"in \[0, 1024\)"),
({"diffusion_max_steps": 0}, "positive integer"),
({"diffusion_max_steps": "1"}, "positive integer"),
({"diffusion_max_steps": True}, "positive integer"),
({"diffusion_read_only": "yes"}, "boolean"),
({"diffusion_read_only": 2}, "boolean"),
({"diffusion_pinned": "0,1"}, "list of canvas positions"),
({"diffusion_pinned": [0, True]}, "list of canvas positions"),
({"diffusion_pinned": [0]}, "needs a diffusion_seed_canvas"),
],
)
def test_diffusion_rejects_bad_extra_args(extra_args: dict, match: str):
with pytest.raises(VLLMValidationError, match=match):
_verify_diffusion(SamplingParams(extra_args=extra_args))
def test_diffusion_seed_canvas_must_fill_the_canvas():
params = SamplingParams(extra_args={"diffusion_seed_canvas": [0] * 7})
with pytest.raises(VLLMValidationError, match="exactly 8 ids, got 7"):
_verify_diffusion(params, canvas_length=8)
# Without the served diffusion config the canvas length is unknown.
_verify_diffusion(params)
@pytest.mark.parametrize("max_tokens, expected", [(100, 8), (5, 5), (None, 8)])
@pytest.mark.parametrize("flag", [True, 1])
def test_diffusion_read_only_ends_after_one_canvas(max_tokens, expected, flag):
params = SamplingParams(
max_tokens=max_tokens, extra_args={"diffusion_read_only": flag}
)
_verify_diffusion(params, canvas_length=8)
assert params.max_tokens == expected
assert params.ignore_eos
@pytest.mark.parametrize(
"width, match",
[
(0, "positive integer"),
("4", "positive integer"),
(True, "positive integer"),
(9, "no larger"),
],
)
def test_diffusion_rejects_bad_canvas_length(width, match):
with pytest.raises(VLLMValidationError, match=match):
_verify_diffusion(
SamplingParams(extra_args={"diffusion_canvas_length": width}),
canvas_length=8,
)
def test_diffusion_canvas_length_sizes_the_seed_and_the_read():
params = SamplingParams(
max_tokens=100,
extra_args={
"diffusion_canvas_length": 4,
"diffusion_seed_canvas": [1, 2, 3, 4],
"diffusion_read_only": True,
},
)
_verify_diffusion(params, canvas_length=8)
assert params.max_tokens == 4
params = SamplingParams(
extra_args={"diffusion_canvas_length": 4, "diffusion_seed_canvas": [0] * 8}
)
with pytest.raises(VLLMValidationError, match="exactly 4 ids, got 8"):
_verify_diffusion(params, canvas_length=8)
def test_diffusion_pinned_positions_stay_inside_the_canvas():
seed = [0] * 8
with pytest.raises(VLLMValidationError, match="inside the canvas"):
_verify_diffusion(
SamplingParams(
extra_args={"diffusion_seed_canvas": seed, "diffusion_pinned": [7, 8]}
),
canvas_length=8,
)
with pytest.raises(VLLMValidationError, match="inside the canvas"):
_verify_diffusion(
SamplingParams(
extra_args={"diffusion_seed_canvas": seed, "diffusion_pinned": [-1]}
)
)
# A narrower request canvas bounds the positions.
with pytest.raises(VLLMValidationError, match="inside the canvas"):
_verify_diffusion(
SamplingParams(
extra_args={
"diffusion_canvas_length": 4,
"diffusion_seed_canvas": seed[:4],
"diffusion_pinned": [4],
}
),
canvas_length=8,
)
def test_diffusion_accepts_extra_args():
params = SamplingParams(
extra_args={
"diffusion_seed_canvas": list(range(8)),
"diffusion_pinned": [0, 1, 7],
"diffusion_max_steps": 4,
"diffusion_read_only": True,
}
)
_verify_diffusion(params, canvas_length=8)
@pytest.mark.parametrize("value", [-(2**63) - 1, 2**64])
def test_extra_args_rejects_nested_integer_overflow(value):
"""Reject extension values before they reach the engine transport."""
+130
View File
@@ -40,6 +40,10 @@ from vllm.utils.hashing import sha256
from vllm.v1.core.encoder_cache_manager import EncoderCacheManager
from vllm.v1.core.kv_cache_coordinator import HybridKVCacheCoordinator
from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash
from vllm.v1.core.sched.diffusion_scheduler import (
DiffusionAsyncScheduler,
diffusion_canvas_width,
)
from vllm.v1.core.sched.interface import PauseState
from vllm.v1.core.sched.output import CachedRequestData, SchedulerOutput
from vllm.v1.core.sched.scheduler import Scheduler
@@ -6872,3 +6876,129 @@ def test_update_draft_token_ids_in_output_strips_padding():
-1,
]
assert scheduler_output.num_invalid_spec_tokens == {request.request_id: 2}
def test_diffusion_canvas_width_defaults_to_the_served_canvas():
def req(extra):
return SimpleNamespace(sampling_params=SimpleNamespace(extra_args=extra))
assert diffusion_canvas_width(req(None), 64) == 64
assert diffusion_canvas_width(req({}), 64) == 64
assert diffusion_canvas_width(req({"diffusion_canvas_length": 16}), 64) == 16
assert diffusion_canvas_width(SimpleNamespace(sampling_params=None), 64) == 64
def _diffusion_request(req_id: str, extra_args: dict) -> Request:
(request,) = create_requests(
num_requests=1, num_tokens=8, max_tokens=64, req_ids=[req_id]
)
request.sampling_params.extra_args = extra_args
return request
def _diffusion_scheduler(**kwargs) -> DiffusionAsyncScheduler:
scheduler = create_scheduler(
async_scheduling=True,
diffusion_canvas_length=8,
scheduler_cls=DiffusionAsyncScheduler,
**kwargs,
)
assert isinstance(scheduler, DiffusionAsyncScheduler)
return scheduler
@pytest.mark.parametrize("async_scheduling", [False, True])
def test_diffusion_scheduler_is_selected_by_default(async_scheduling):
config = create_scheduler(
async_scheduling=async_scheduling, diffusion_canvas_length=8
).vllm_config.scheduler_config
assert config.get_scheduler_cls() is (
DiffusionAsyncScheduler if config.async_scheduling else Scheduler
)
def test_diffusion_scheduler_narrows_the_canvas_per_request():
scheduler = _diffusion_scheduler()
wide = _diffusion_request("wide", {})
narrow = _diffusion_request("narrow", {"diffusion_canvas_length": 4})
scheduler.add_request(wide)
scheduler.add_request(narrow)
# The prefill step lays down the served canvas as placeholders.
scheduler.schedule()
output = scheduler.schedule()
assert output.scheduled_spec_decode_tokens["wide"] == [-1] * 8
assert output.scheduled_spec_decode_tokens["narrow"] == [-1] * 4
assert output.num_scheduled_tokens["narrow"] == 4
assert narrow.num_output_placeholders == 4
@pytest.mark.parametrize("structured", [False, True])
def test_diffusion_scheduler_trims_full_width_worker_drafts(structured):
"""Padded worker drafts must be narrowed before scheduling or grammar validation."""
scheduler = _diffusion_scheduler()
wide = _diffusion_request("wide", {})
narrow = _diffusion_request("narrow", {"diffusion_canvas_length": 4})
for request in (wide, narrow):
scheduler.add_request(request)
prefill = scheduler.schedule()
_model_output(scheduler, prefill, [[], []])
if structured:
for request in (wide, narrow):
request.structured_output_request = SimpleNamespace(
grammar=_RecordingGrammar(), reasoning_ended=True
)
tokens = list(range(8)) if structured else [-1] * 8
drafts = DraftTokenIds(["wide", "narrow"], [tokens.copy(), tokens.copy()])
output = scheduler.schedule()
scheduler.update_draft_token_ids_in_output(drafts, output)
assert output.scheduled_spec_decode_tokens == {
"wide": tokens,
"narrow": tokens[:4],
}
assert output.num_scheduled_tokens == {"wide": 8, "narrow": 4}
if structured:
assert wide.structured_output_request.grammar.seen == [tokens]
assert narrow.structured_output_request.grammar.seen == [tokens[:4]]
def test_diffusion_scheduler_defers_a_read_with_every_step_in_flight():
scheduler = _diffusion_scheduler()
one = _diffusion_request(
"one", {"diffusion_read_only": True, "diffusion_max_steps": 1}
)
two = _diffusion_request(
"two", {"diffusion_read_only": True, "diffusion_max_steps": 2}
)
# Only reads are deferred. A capped generation keeps its extra async step.
gen = _diffusion_request("gen", {"diffusion_max_steps": 1})
for request in (one, two, gen):
scheduler.add_request(request)
scheduler.schedule()
assert set(scheduler.schedule().num_scheduled_tokens) == {"one", "two", "gen"}
assert one.num_output_placeholders == 8
assert set(scheduler.schedule().num_scheduled_tokens) == {"two", "gen"}
assert set(scheduler.schedule().num_scheduled_tokens) == {"gen"}
# The deferral is renewed every call.
assert set(scheduler.schedule().num_scheduled_tokens) == {"gen"}
def test_diffusion_read_deferral_keeps_a_longer_pp_wait():
scheduler = _diffusion_scheduler(pipeline_parallel_size=3, use_v2_model_runner=True)
read = _diffusion_request(
"read", {"diffusion_read_only": True, "diffusion_max_steps": 1}
)
scheduler.add_request(read)
scheduler.schedule()
assert read.next_decode_eligible_step == 4
for _ in range(2):
assert "read" not in scheduler.schedule().num_scheduled_tokens
assert "read" in scheduler.schedule().num_scheduled_tokens
assert read.next_decode_eligible_step == 7
# Deferring this step alone would ask for 6. The PP wait to 7 stands.
assert "read" not in scheduler.schedule().num_scheduled_tokens
assert read.next_decode_eligible_step == 7
+12 -1
View File
@@ -7,6 +7,7 @@ import vllm.envs as envs
from tests.v1.kv_connector.unit.utils import MockKVConfig
from vllm.config import (
CacheConfig,
DiffusionConfig,
ECTransferConfig,
KVTransferConfig,
ModelConfig,
@@ -77,6 +78,8 @@ def create_scheduler(
kv_cache_spec: KVCacheSpec | None = None,
per_request_spec_decode_metrics: str = "none",
scheduling_policy: SchedulerPolicy = "fcfs",
diffusion_canvas_length: int | None = None,
scheduler_cls: type[Scheduler] | None = None,
) -> Scheduler | AsyncScheduler:
"""Create scheduler under test.
@@ -178,6 +181,12 @@ def create_scheduler(
else None
)
diffusion_config: DiffusionConfig | None = None
if diffusion_canvas_length is not None:
# A diffusion checkpoint declares its canvas in the HF config.
model_config.hf_config.canvas_length = diffusion_canvas_length
diffusion_config = DiffusionConfig(canvas_length=diffusion_canvas_length)
vllm_config = VllmConfig(
scheduler_config=scheduler_config,
model_config=model_config,
@@ -188,6 +197,7 @@ def create_scheduler(
),
kv_transfer_config=kv_transfer_config,
speculative_config=speculative_config,
diffusion_config=diffusion_config,
ec_transfer_config=ec_transfer_config,
observability_config=ObservabilityConfig(
per_request_spec_decode_metrics=per_request_spec_decode_metrics,
@@ -207,7 +217,8 @@ def create_scheduler(
)
cache_config.num_gpu_blocks = num_blocks
register_all_kvcache_specs(vllm_config)
scheduler_cls = AsyncScheduler if async_scheduling else Scheduler
if scheduler_cls is None:
scheduler_cls = AsyncScheduler if async_scheduling else Scheduler
scheduler = scheduler_cls(
vllm_config=vllm_config,
kv_cache_config=kv_cache_config,
@@ -80,10 +80,12 @@ def _validate(enable_trace_replay: bool) -> None:
model_config=SimpleNamespace(
return_sampling_mask=False,
enable_trace_replay=enable_trace_replay,
is_diffusion=False,
),
vllm_config=SimpleNamespace(reasoning_config=None),
speculative_config=None,
structured_outputs_config=None,
diffusion_config=None,
tokenizer=None,
validate_logits_processors_params=lambda params: None,
)
@@ -0,0 +1,319 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Per-request DiffusionGemma state behind structured reads: seed canvases,
pinned positions, read-only slots and the per-slot step cap."""
import numpy as np
import pytest
import torch
from vllm.model_executor.models.diffusion_gemma import (
DiffusionGemmaRequestStates,
_compiled_sample_step,
_concat_logprob_stashes,
)
from vllm.platforms import current_platform
from vllm.v1.outputs import LogprobsTensors
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda(), reason="the sampler state lives on the GPU"
)
CL = 8
VOCAB = 64
MAX_REQS = 4
MAX_STEPS = 48
def _states() -> DiffusionGemmaRequestStates:
return DiffusionGemmaRequestStates(
max_num_reqs=MAX_REQS,
canvas_length=CL,
vocab_size=VOCAB,
max_denoising_steps=MAX_STEPS,
device=torch.device("cuda"),
hidden_size=4,
stability_threshold=2,
)
def _slots(*idx: int) -> tuple[np.ndarray, torch.Tensor]:
slots = np.array(idx, dtype=np.int64)
return slots, torch.tensor(slots, device="cuda")
def test_seed_canvas_replaces_only_seeded_slots():
states = _states()
for slot in range(3):
states.add_request(slot)
seed = list(range(CL))
states.set_seed_canvas(1, seed)
slots, slots_gpu = _slots(0, 1, 2)
states.init_canvas(slots_gpu)
before = states.canvas[slots_gpu].clone()
states.apply_seed_canvases(slots, slots_gpu)
after = states.canvas[slots_gpu]
assert after[1].tolist() == seed
assert torch.equal(after[0], before[0])
assert torch.equal(after[2], before[2])
def test_apply_seed_canvases_leaves_unseeded_batches_alone():
states = _states()
states.add_request(0)
slots, slots_gpu = _slots(0)
states.init_canvas(slots_gpu)
before = states.canvas[0].clone()
states.apply_seed_canvases(slots, slots_gpu)
assert torch.equal(states.canvas[0], before)
def test_add_request_clears_seed_and_read_only():
states = _states()
states.add_request(0)
states.set_seed_canvas(0, [1] * CL)
states.set_read_only(0)
assert states.seeded_slots == {0}
assert states.read_only_slots == {0}
states.add_request(0)
assert not states.seeded_slots
assert not states.read_only_slots
assert not bool(states.has_seed[0])
assert not bool(states.read_only[0])
def test_canvas_width_resets_with_the_slot():
states = _states()
states.add_request(0)
states.canvas_width_np[0] = 4
states.set_seed_canvas(0, [7, 7, 7, 7])
assert states.seed_canvas[0, :4].tolist() == [7, 7, 7, 7]
states.add_request(0)
assert states.canvas_width_np[0] == CL
def test_remove_request_forgets_the_slot():
states = _states()
states.add_request(0)
states.set_seed_canvas(0, [1] * CL)
states.set_read_only(0)
states.remove_request(0)
assert not states.seeded_slots
assert not states.read_only_slots
def _denoise_once(
states: DiffusionGemmaRequestStates,
slots: list[int],
compute_sc: bool = True,
width: int = CL,
embed_weight: torch.Tensor | None = None,
embed_dtype: torch.dtype = torch.float32,
) -> tuple[torch.Tensor, torch.Tensor]:
"""One compiled denoise step over ``slots`` with flat logits, so nothing
converges by stability or confidence and only the step cap can end it.
``width`` below CL runs the step on [:, :width] views, as the sampler
does for a narrow tile."""
n = len(slots)
device = states.device
decode_slots = torch.tensor(slots, dtype=torch.int64, device=device)
decode_idx = torch.arange(n, dtype=torch.int64, device=device)
sampled = torch.zeros(n, CL, dtype=torch.int32, device=device)[:, :width]
num_sampled = torch.zeros(n, dtype=torch.int32, device=device)
_compiled_sample_step(
torch.zeros(n * width, VOCAB, device=device),
decode_slots,
decode_idx,
decode_slots,
torch.full((n,), width, dtype=torch.int64, device=device),
states.canvas[:, :width],
states.argmax_canvas[:, :width],
states.step,
states.is_encoder_phase,
states.confident,
states.self_conditioning_embeds[:, :width],
(
torch.zeros(VOCAB, 4, dtype=embed_dtype, device=device)
if embed_weight is None
else embed_weight
),
torch.tensor(1.0, dtype=embed_dtype, device=device),
states.accepted_canvas_history[:, :, :width],
states.accepted_canvas_history_len,
states.max_steps,
states.pin_mask[:, :width],
states.seed_canvas[:, :width],
states.read_only,
sampled,
num_sampled,
torch.zeros(MAX_REQS, CL, dtype=torch.int64, device=device),
max_denoising_steps=float(MAX_STEPS),
t_min=0.5,
t_max=1.0,
confidence_threshold=0.1,
vocab_size=VOCAB,
CL=width,
ST=states.stability_threshold,
entropy_bound=0.1,
sc_vocab_start=0,
sc_vocab_end=VOCAB,
tp_size=1,
tp_group_name="",
compute_sc=compute_sc,
)
return sampled, num_sampled
def test_pinned_positions_hold_their_seed_through_a_step():
states = _states()
states.add_request(0)
states.is_encoder_phase[0] = False
seed = list(range(10, 10 + CL))
states.set_seed_canvas(0, seed)
states.canvas[0] = torch.tensor(seed, device="cuda")
states.set_pins(0, [0, 1, 2, 3])
# Flat logits accept nothing, so every free position is renoised.
_denoise_once(states, [0], embed_weight=torch.ones(VOCAB, 4, device="cuda"))
assert states.canvas[0, :4].tolist() == seed[:4]
# The soft embed is zero at pinned positions and non-zero elsewhere.
sc = states.self_conditioning_embeds[0]
assert not sc[:4].any()
assert sc[4:].any()
def test_add_request_clears_pins():
states = _states()
states.add_request(0)
states.set_seed_canvas(0, [1] * CL)
states.set_pins(0, [2, 3])
assert states.pin_mask[0].tolist() == [False, False, True, True] + [False] * 4
states.add_request(0)
assert not states.pin_mask[0].any()
def test_single_step_tile_skips_self_conditioning():
states = _states()
states.add_request(0)
states.is_encoder_phase[0] = False
states.self_conditioning_embeds[0] = 1.0
_denoise_once(states, [0], compute_sc=False)
assert not states.self_conditioning_embeds[0].any()
@pytest.mark.parametrize("stance", ["default", "force_eager"])
def test_self_conditioning_stores_a_bf16_model_in_the_fp32_buffer(stance):
# The model's embeddings are bf16 while the buffer is fp32. Compiled code
# casts on the store, but eager does not, and the step runs eager once
# torch.compile hits its recompile limit.
states = _states()
states.add_request(0)
states.is_encoder_phase[0] = False
with torch.compiler.set_stance(stance):
_denoise_once(
states,
[0],
embed_weight=torch.ones(VOCAB, 4, dtype=torch.bfloat16, device="cuda"),
embed_dtype=torch.bfloat16,
)
assert states.self_conditioning_embeds.dtype == torch.float32
# uniform probs @ an all-ones embedding
assert torch.allclose(
states.self_conditioning_embeds[0], torch.ones(CL, 4, device="cuda")
)
def test_narrow_tile_leaves_the_rest_of_the_canvas_alone():
states = _states()
states.add_request(0)
states.is_encoder_phase[0] = False
states.canvas[0] = 5
states.argmax_canvas[0] = 5
_denoise_once(states, [0], width=4)
# Columns past the width are untouched; the step counted.
assert states.canvas[0, 4:].tolist() == [5] * (CL - 4)
assert states.argmax_canvas[0, 4:].tolist() == [5] * (CL - 4)
assert states.step[0] == 1
def test_step_cap_is_per_slot():
states = _states()
for slot in (0, 1):
states.add_request(slot)
states.is_encoder_phase[slot] = False
states.max_steps[0] = 1
_denoise_once(states, [0, 1])
# Slot 0 hit its cap and moves to commit. Slot 1 keeps denoising.
assert states.is_encoder_phase[:2].tolist() == [True, False]
assert states.step[:2].tolist() == [1, 1]
def _stash(rows: int, width: int, first_id: int) -> LogprobsTensors:
ids = torch.arange(first_id, first_id + rows * width).reshape(rows, width)
return LogprobsTensors(
logprob_token_ids=ids,
logprobs=-ids.float(),
selected_token_ranks=torch.zeros(rows, dtype=torch.int64),
)
def test_stashes_of_different_widths_join():
# A read that asked for 10 label ids and a generation that asked for 2
# logprobs converged in different steps, so their stashes differ in width.
wide, narrow = _stash(2, 11, 100), _stash(3, 3, 0)
out = _concat_logprob_stashes([narrow, wide], [0, 3])
assert out.logprob_token_ids.shape == (5, 11)
assert out.logprobs.shape == (5, 11)
assert out.cu_num_generated_tokens == [0, 3]
# The narrow rows keep their values and pad with id 0 at -inf.
assert torch.equal(out.logprob_token_ids[:3, :3], narrow.logprob_token_ids)
assert not out.logprob_token_ids[:3, 3:].any()
assert torch.isneginf(out.logprobs[:3, 3:]).all()
assert torch.equal(out.logprobs[3:], wide.logprobs)
@pytest.mark.parametrize("width", [4, CL])
@pytest.mark.parametrize("steps", [1, 2])
def test_read_emits_at_convergence_while_generation_waits_for_commit(width, steps):
states = _states()
for slot in (2, 0):
states.add_request(slot)
states.is_encoder_phase[slot] = False
states.max_steps[slot] = steps
states.set_read_only(2)
for step in range(steps):
sampled, counts = _denoise_once(states, [2, 0], width=width)
assert counts.tolist() == ([width, 0] if step == steps - 1 else [0, 0])
assert torch.equal(sampled[0], states.argmax_canvas[2, :width].int())
assert not states.is_encoder_phase[2]
assert states.is_encoder_phase[0]
assert not states.self_conditioning_embeds[2].any()
sampled, counts = _denoise_once(states, [0], width=width)
assert counts.tolist() == [width]
assert torch.equal(sampled[0], states.argmax_canvas[0, :width].int())