[Core] structured generation mode for DiffusionGemma model (Jev-like) (#57250)
Signed-off-by: Matt Mastracci <matthew@mastracci.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Razorback16 <razorback16@protonmail.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Razorback16 <razorback16@protonmail.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
Lucas Wilkinson
Codex
Razorback16
parent
fae5cbfc8b
commit
1b3b88ec2b
@@ -15,8 +15,10 @@ from transformers.models.gemma4.configuration_gemma4 import Gemma4TextConfig
|
||||
|
||||
from vllm.config import ModelConfig, ParallelConfig, SpeculativeConfig
|
||||
from vllm.config.model_arch import ModelArchitectureConfig
|
||||
from vllm.transformers_utils.configs.diffusion_gemma import DiffusionGemmaConfig
|
||||
from vllm.transformers_utils.configs.gemma4 import gemma4_layer_config
|
||||
from vllm.transformers_utils.model_arch_config_convertor import (
|
||||
MODEL_ARCH_CONFIG_CONVERTORS,
|
||||
Gemma4ModelArchConfigConvertor,
|
||||
ModelArchConfigConvertorBase,
|
||||
)
|
||||
@@ -496,3 +498,26 @@ def test_deepseek_v4_convertor_splits_vision_architecture():
|
||||
conv = DeepseekV4ModelArchConfigConvertor(draft_cfg, draft_cfg)
|
||||
assert conv.get_architectures() == [draft_arch]
|
||||
assert draft_cfg.architectures == [draft_arch]
|
||||
|
||||
|
||||
def test_diffusion_gemma_head_dims_vary_by_layer_type():
|
||||
"""Checkpoints carry model_type `diffusion_gemma` at the top level. Under
|
||||
the base convertor every layer reports the sliding head dim, Gemma4Config
|
||||
skips its head-dim handling, and the full-attention KV-cache group falls to
|
||||
FlashInfer, which cannot take a per-request causal tensor."""
|
||||
text_config = dict(
|
||||
num_hidden_layers=6,
|
||||
hidden_size=64,
|
||||
num_attention_heads=8,
|
||||
num_key_value_heads=4,
|
||||
head_dim=16,
|
||||
global_head_dim=32,
|
||||
layer_types=["sliding_attention"] * 5 + ["full_attention"],
|
||||
)
|
||||
hf_config = DiffusionGemmaConfig(text_config=text_config)
|
||||
|
||||
convertor_cls = MODEL_ARCH_CONFIG_CONVERTORS[hf_config.model_type]
|
||||
assert convertor_cls is Gemma4ModelArchConfigConvertor
|
||||
arch = convertor_cls(hf_config, hf_config.text_config).convert()
|
||||
|
||||
assert [arch[i].head_size for i in range(6)] == [16] * 5 + [32]
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from dataclasses import dataclass
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.config.diffusion import DiffusionConfig
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.utils.diffusion import validate_diffusion_sampling_params
|
||||
from vllm.v1.engine.input_processor import InputProcessor
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -13,6 +17,7 @@ class MockModelConfig:
|
||||
is_diffusion: bool = False
|
||||
max_logprobs: int = 20
|
||||
logits_processors: list | None = None
|
||||
return_sampling_mask: bool = False
|
||||
|
||||
def get_vocab_size(self) -> int:
|
||||
return 1024
|
||||
@@ -59,6 +64,183 @@ def test_verify_leaves_logits_processors_to_admission():
|
||||
)
|
||||
|
||||
|
||||
def _verify_diffusion(params: SamplingParams, canvas_length: int | None = None):
|
||||
model_config = MockModelConfig(is_diffusion=True)
|
||||
params.verify(model_config, None, None, None)
|
||||
validate_diffusion_sampling_params(
|
||||
params,
|
||||
canvas_length=canvas_length,
|
||||
vocab_size=model_config.get_vocab_size(),
|
||||
async_scheduling=True,
|
||||
)
|
||||
|
||||
|
||||
def test_diffusion_extra_args_are_validated_without_a_served_canvas():
|
||||
# No --diffusion-config: the canvas is unknown, the ids are still checked
|
||||
# and a read-only request is still normalised.
|
||||
params = SamplingParams(
|
||||
max_tokens=64,
|
||||
extra_args={"diffusion_seed_canvas": [0, 1], "diffusion_read_only": True},
|
||||
)
|
||||
_verify_diffusion(params, canvas_length=None)
|
||||
assert params.ignore_eos is True
|
||||
|
||||
bad = SamplingParams(extra_args={"diffusion_seed_canvas": [0, 10**9]})
|
||||
with pytest.raises(VLLMValidationError, match="ids must be in"):
|
||||
_verify_diffusion(bad, canvas_length=None)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"extra_args",
|
||||
[
|
||||
{},
|
||||
{"diffusion_canvas_length": None},
|
||||
{"diffusion_canvas_length": 4},
|
||||
{"diffusion_canvas_length": 8},
|
||||
],
|
||||
)
|
||||
def test_narrow_diffusion_canvas_requires_async_scheduling(
|
||||
async_scheduling, extra_args
|
||||
):
|
||||
processor = SimpleNamespace(
|
||||
model_config=MockModelConfig(is_diffusion=True),
|
||||
vllm_config=SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(async_scheduling=async_scheduling)
|
||||
),
|
||||
speculative_config=None,
|
||||
structured_outputs_config=None,
|
||||
diffusion_config=DiffusionConfig(canvas_length=8),
|
||||
tokenizer=None,
|
||||
validate_logits_processors_params=lambda params: None,
|
||||
)
|
||||
params = SamplingParams(extra_args=extra_args)
|
||||
if not async_scheduling and extra_args.get("diffusion_canvas_length") == 4:
|
||||
with pytest.raises(VLLMValidationError, match="requires --async-scheduling"):
|
||||
InputProcessor._validate_params(processor, params, ("generate",))
|
||||
else:
|
||||
InputProcessor._validate_params(processor, params, ("generate",))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"extra_args, match",
|
||||
[
|
||||
({"diffusion_seed_canvas": "abc"}, "list of token ids"),
|
||||
({"diffusion_seed_canvas": [1, 2.5]}, "list of token ids"),
|
||||
({"diffusion_seed_canvas": [1, True]}, "list of token ids"),
|
||||
({"diffusion_seed_canvas": [0, 1024]}, r"in \[0, 1024\)"),
|
||||
({"diffusion_seed_canvas": [0, -1]}, r"in \[0, 1024\)"),
|
||||
({"diffusion_max_steps": 0}, "positive integer"),
|
||||
({"diffusion_max_steps": "1"}, "positive integer"),
|
||||
({"diffusion_max_steps": True}, "positive integer"),
|
||||
({"diffusion_read_only": "yes"}, "boolean"),
|
||||
({"diffusion_read_only": 2}, "boolean"),
|
||||
({"diffusion_pinned": "0,1"}, "list of canvas positions"),
|
||||
({"diffusion_pinned": [0, True]}, "list of canvas positions"),
|
||||
({"diffusion_pinned": [0]}, "needs a diffusion_seed_canvas"),
|
||||
],
|
||||
)
|
||||
def test_diffusion_rejects_bad_extra_args(extra_args: dict, match: str):
|
||||
with pytest.raises(VLLMValidationError, match=match):
|
||||
_verify_diffusion(SamplingParams(extra_args=extra_args))
|
||||
|
||||
|
||||
def test_diffusion_seed_canvas_must_fill_the_canvas():
|
||||
params = SamplingParams(extra_args={"diffusion_seed_canvas": [0] * 7})
|
||||
with pytest.raises(VLLMValidationError, match="exactly 8 ids, got 7"):
|
||||
_verify_diffusion(params, canvas_length=8)
|
||||
# Without the served diffusion config the canvas length is unknown.
|
||||
_verify_diffusion(params)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("max_tokens, expected", [(100, 8), (5, 5), (None, 8)])
|
||||
@pytest.mark.parametrize("flag", [True, 1])
|
||||
def test_diffusion_read_only_ends_after_one_canvas(max_tokens, expected, flag):
|
||||
params = SamplingParams(
|
||||
max_tokens=max_tokens, extra_args={"diffusion_read_only": flag}
|
||||
)
|
||||
_verify_diffusion(params, canvas_length=8)
|
||||
assert params.max_tokens == expected
|
||||
assert params.ignore_eos
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"width, match",
|
||||
[
|
||||
(0, "positive integer"),
|
||||
("4", "positive integer"),
|
||||
(True, "positive integer"),
|
||||
(9, "no larger"),
|
||||
],
|
||||
)
|
||||
def test_diffusion_rejects_bad_canvas_length(width, match):
|
||||
with pytest.raises(VLLMValidationError, match=match):
|
||||
_verify_diffusion(
|
||||
SamplingParams(extra_args={"diffusion_canvas_length": width}),
|
||||
canvas_length=8,
|
||||
)
|
||||
|
||||
|
||||
def test_diffusion_canvas_length_sizes_the_seed_and_the_read():
|
||||
params = SamplingParams(
|
||||
max_tokens=100,
|
||||
extra_args={
|
||||
"diffusion_canvas_length": 4,
|
||||
"diffusion_seed_canvas": [1, 2, 3, 4],
|
||||
"diffusion_read_only": True,
|
||||
},
|
||||
)
|
||||
_verify_diffusion(params, canvas_length=8)
|
||||
assert params.max_tokens == 4
|
||||
|
||||
params = SamplingParams(
|
||||
extra_args={"diffusion_canvas_length": 4, "diffusion_seed_canvas": [0] * 8}
|
||||
)
|
||||
with pytest.raises(VLLMValidationError, match="exactly 4 ids, got 8"):
|
||||
_verify_diffusion(params, canvas_length=8)
|
||||
|
||||
|
||||
def test_diffusion_pinned_positions_stay_inside_the_canvas():
|
||||
seed = [0] * 8
|
||||
with pytest.raises(VLLMValidationError, match="inside the canvas"):
|
||||
_verify_diffusion(
|
||||
SamplingParams(
|
||||
extra_args={"diffusion_seed_canvas": seed, "diffusion_pinned": [7, 8]}
|
||||
),
|
||||
canvas_length=8,
|
||||
)
|
||||
with pytest.raises(VLLMValidationError, match="inside the canvas"):
|
||||
_verify_diffusion(
|
||||
SamplingParams(
|
||||
extra_args={"diffusion_seed_canvas": seed, "diffusion_pinned": [-1]}
|
||||
)
|
||||
)
|
||||
# A narrower request canvas bounds the positions.
|
||||
with pytest.raises(VLLMValidationError, match="inside the canvas"):
|
||||
_verify_diffusion(
|
||||
SamplingParams(
|
||||
extra_args={
|
||||
"diffusion_canvas_length": 4,
|
||||
"diffusion_seed_canvas": seed[:4],
|
||||
"diffusion_pinned": [4],
|
||||
}
|
||||
),
|
||||
canvas_length=8,
|
||||
)
|
||||
|
||||
|
||||
def test_diffusion_accepts_extra_args():
|
||||
params = SamplingParams(
|
||||
extra_args={
|
||||
"diffusion_seed_canvas": list(range(8)),
|
||||
"diffusion_pinned": [0, 1, 7],
|
||||
"diffusion_max_steps": 4,
|
||||
"diffusion_read_only": True,
|
||||
}
|
||||
)
|
||||
_verify_diffusion(params, canvas_length=8)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [-(2**63) - 1, 2**64])
|
||||
def test_extra_args_rejects_nested_integer_overflow(value):
|
||||
"""Reject extension values before they reach the engine transport."""
|
||||
|
||||
@@ -40,6 +40,10 @@ from vllm.utils.hashing import sha256
|
||||
from vllm.v1.core.encoder_cache_manager import EncoderCacheManager
|
||||
from vllm.v1.core.kv_cache_coordinator import HybridKVCacheCoordinator
|
||||
from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash
|
||||
from vllm.v1.core.sched.diffusion_scheduler import (
|
||||
DiffusionAsyncScheduler,
|
||||
diffusion_canvas_width,
|
||||
)
|
||||
from vllm.v1.core.sched.interface import PauseState
|
||||
from vllm.v1.core.sched.output import CachedRequestData, SchedulerOutput
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
@@ -6872,3 +6876,129 @@ def test_update_draft_token_ids_in_output_strips_padding():
|
||||
-1,
|
||||
]
|
||||
assert scheduler_output.num_invalid_spec_tokens == {request.request_id: 2}
|
||||
|
||||
|
||||
def test_diffusion_canvas_width_defaults_to_the_served_canvas():
|
||||
def req(extra):
|
||||
return SimpleNamespace(sampling_params=SimpleNamespace(extra_args=extra))
|
||||
|
||||
assert diffusion_canvas_width(req(None), 64) == 64
|
||||
assert diffusion_canvas_width(req({}), 64) == 64
|
||||
assert diffusion_canvas_width(req({"diffusion_canvas_length": 16}), 64) == 16
|
||||
assert diffusion_canvas_width(SimpleNamespace(sampling_params=None), 64) == 64
|
||||
|
||||
|
||||
def _diffusion_request(req_id: str, extra_args: dict) -> Request:
|
||||
(request,) = create_requests(
|
||||
num_requests=1, num_tokens=8, max_tokens=64, req_ids=[req_id]
|
||||
)
|
||||
request.sampling_params.extra_args = extra_args
|
||||
return request
|
||||
|
||||
|
||||
def _diffusion_scheduler(**kwargs) -> DiffusionAsyncScheduler:
|
||||
scheduler = create_scheduler(
|
||||
async_scheduling=True,
|
||||
diffusion_canvas_length=8,
|
||||
scheduler_cls=DiffusionAsyncScheduler,
|
||||
**kwargs,
|
||||
)
|
||||
assert isinstance(scheduler, DiffusionAsyncScheduler)
|
||||
return scheduler
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [False, True])
|
||||
def test_diffusion_scheduler_is_selected_by_default(async_scheduling):
|
||||
config = create_scheduler(
|
||||
async_scheduling=async_scheduling, diffusion_canvas_length=8
|
||||
).vllm_config.scheduler_config
|
||||
assert config.get_scheduler_cls() is (
|
||||
DiffusionAsyncScheduler if config.async_scheduling else Scheduler
|
||||
)
|
||||
|
||||
|
||||
def test_diffusion_scheduler_narrows_the_canvas_per_request():
|
||||
scheduler = _diffusion_scheduler()
|
||||
wide = _diffusion_request("wide", {})
|
||||
narrow = _diffusion_request("narrow", {"diffusion_canvas_length": 4})
|
||||
scheduler.add_request(wide)
|
||||
scheduler.add_request(narrow)
|
||||
|
||||
# The prefill step lays down the served canvas as placeholders.
|
||||
scheduler.schedule()
|
||||
output = scheduler.schedule()
|
||||
assert output.scheduled_spec_decode_tokens["wide"] == [-1] * 8
|
||||
assert output.scheduled_spec_decode_tokens["narrow"] == [-1] * 4
|
||||
assert output.num_scheduled_tokens["narrow"] == 4
|
||||
assert narrow.num_output_placeholders == 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize("structured", [False, True])
|
||||
def test_diffusion_scheduler_trims_full_width_worker_drafts(structured):
|
||||
"""Padded worker drafts must be narrowed before scheduling or grammar validation."""
|
||||
scheduler = _diffusion_scheduler()
|
||||
wide = _diffusion_request("wide", {})
|
||||
narrow = _diffusion_request("narrow", {"diffusion_canvas_length": 4})
|
||||
for request in (wide, narrow):
|
||||
scheduler.add_request(request)
|
||||
prefill = scheduler.schedule()
|
||||
_model_output(scheduler, prefill, [[], []])
|
||||
|
||||
if structured:
|
||||
for request in (wide, narrow):
|
||||
request.structured_output_request = SimpleNamespace(
|
||||
grammar=_RecordingGrammar(), reasoning_ended=True
|
||||
)
|
||||
tokens = list(range(8)) if structured else [-1] * 8
|
||||
drafts = DraftTokenIds(["wide", "narrow"], [tokens.copy(), tokens.copy()])
|
||||
output = scheduler.schedule()
|
||||
scheduler.update_draft_token_ids_in_output(drafts, output)
|
||||
|
||||
assert output.scheduled_spec_decode_tokens == {
|
||||
"wide": tokens,
|
||||
"narrow": tokens[:4],
|
||||
}
|
||||
assert output.num_scheduled_tokens == {"wide": 8, "narrow": 4}
|
||||
if structured:
|
||||
assert wide.structured_output_request.grammar.seen == [tokens]
|
||||
assert narrow.structured_output_request.grammar.seen == [tokens[:4]]
|
||||
|
||||
|
||||
def test_diffusion_scheduler_defers_a_read_with_every_step_in_flight():
|
||||
scheduler = _diffusion_scheduler()
|
||||
one = _diffusion_request(
|
||||
"one", {"diffusion_read_only": True, "diffusion_max_steps": 1}
|
||||
)
|
||||
two = _diffusion_request(
|
||||
"two", {"diffusion_read_only": True, "diffusion_max_steps": 2}
|
||||
)
|
||||
# Only reads are deferred. A capped generation keeps its extra async step.
|
||||
gen = _diffusion_request("gen", {"diffusion_max_steps": 1})
|
||||
for request in (one, two, gen):
|
||||
scheduler.add_request(request)
|
||||
|
||||
scheduler.schedule()
|
||||
assert set(scheduler.schedule().num_scheduled_tokens) == {"one", "two", "gen"}
|
||||
assert one.num_output_placeholders == 8
|
||||
assert set(scheduler.schedule().num_scheduled_tokens) == {"two", "gen"}
|
||||
assert set(scheduler.schedule().num_scheduled_tokens) == {"gen"}
|
||||
# The deferral is renewed every call.
|
||||
assert set(scheduler.schedule().num_scheduled_tokens) == {"gen"}
|
||||
|
||||
|
||||
def test_diffusion_read_deferral_keeps_a_longer_pp_wait():
|
||||
scheduler = _diffusion_scheduler(pipeline_parallel_size=3, use_v2_model_runner=True)
|
||||
read = _diffusion_request(
|
||||
"read", {"diffusion_read_only": True, "diffusion_max_steps": 1}
|
||||
)
|
||||
scheduler.add_request(read)
|
||||
|
||||
scheduler.schedule()
|
||||
assert read.next_decode_eligible_step == 4
|
||||
for _ in range(2):
|
||||
assert "read" not in scheduler.schedule().num_scheduled_tokens
|
||||
assert "read" in scheduler.schedule().num_scheduled_tokens
|
||||
assert read.next_decode_eligible_step == 7
|
||||
# Deferring this step alone would ask for 6. The PP wait to 7 stands.
|
||||
assert "read" not in scheduler.schedule().num_scheduled_tokens
|
||||
assert read.next_decode_eligible_step == 7
|
||||
|
||||
+12
-1
@@ -7,6 +7,7 @@ import vllm.envs as envs
|
||||
from tests.v1.kv_connector.unit.utils import MockKVConfig
|
||||
from vllm.config import (
|
||||
CacheConfig,
|
||||
DiffusionConfig,
|
||||
ECTransferConfig,
|
||||
KVTransferConfig,
|
||||
ModelConfig,
|
||||
@@ -77,6 +78,8 @@ def create_scheduler(
|
||||
kv_cache_spec: KVCacheSpec | None = None,
|
||||
per_request_spec_decode_metrics: str = "none",
|
||||
scheduling_policy: SchedulerPolicy = "fcfs",
|
||||
diffusion_canvas_length: int | None = None,
|
||||
scheduler_cls: type[Scheduler] | None = None,
|
||||
) -> Scheduler | AsyncScheduler:
|
||||
"""Create scheduler under test.
|
||||
|
||||
@@ -178,6 +181,12 @@ def create_scheduler(
|
||||
else None
|
||||
)
|
||||
|
||||
diffusion_config: DiffusionConfig | None = None
|
||||
if diffusion_canvas_length is not None:
|
||||
# A diffusion checkpoint declares its canvas in the HF config.
|
||||
model_config.hf_config.canvas_length = diffusion_canvas_length
|
||||
diffusion_config = DiffusionConfig(canvas_length=diffusion_canvas_length)
|
||||
|
||||
vllm_config = VllmConfig(
|
||||
scheduler_config=scheduler_config,
|
||||
model_config=model_config,
|
||||
@@ -188,6 +197,7 @@ def create_scheduler(
|
||||
),
|
||||
kv_transfer_config=kv_transfer_config,
|
||||
speculative_config=speculative_config,
|
||||
diffusion_config=diffusion_config,
|
||||
ec_transfer_config=ec_transfer_config,
|
||||
observability_config=ObservabilityConfig(
|
||||
per_request_spec_decode_metrics=per_request_spec_decode_metrics,
|
||||
@@ -207,7 +217,8 @@ def create_scheduler(
|
||||
)
|
||||
cache_config.num_gpu_blocks = num_blocks
|
||||
register_all_kvcache_specs(vllm_config)
|
||||
scheduler_cls = AsyncScheduler if async_scheduling else Scheduler
|
||||
if scheduler_cls is None:
|
||||
scheduler_cls = AsyncScheduler if async_scheduling else Scheduler
|
||||
scheduler = scheduler_cls(
|
||||
vllm_config=vllm_config,
|
||||
kv_cache_config=kv_cache_config,
|
||||
|
||||
@@ -80,10 +80,12 @@ def _validate(enable_trace_replay: bool) -> None:
|
||||
model_config=SimpleNamespace(
|
||||
return_sampling_mask=False,
|
||||
enable_trace_replay=enable_trace_replay,
|
||||
is_diffusion=False,
|
||||
),
|
||||
vllm_config=SimpleNamespace(reasoning_config=None),
|
||||
speculative_config=None,
|
||||
structured_outputs_config=None,
|
||||
diffusion_config=None,
|
||||
tokenizer=None,
|
||||
validate_logits_processors_params=lambda params: None,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,319 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Per-request DiffusionGemma state behind structured reads: seed canvases,
|
||||
pinned positions, read-only slots and the per-slot step cap."""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.diffusion_gemma import (
|
||||
DiffusionGemmaRequestStates,
|
||||
_compiled_sample_step,
|
||||
_concat_logprob_stashes,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.outputs import LogprobsTensors
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="the sampler state lives on the GPU"
|
||||
)
|
||||
|
||||
CL = 8
|
||||
VOCAB = 64
|
||||
MAX_REQS = 4
|
||||
MAX_STEPS = 48
|
||||
|
||||
|
||||
def _states() -> DiffusionGemmaRequestStates:
|
||||
return DiffusionGemmaRequestStates(
|
||||
max_num_reqs=MAX_REQS,
|
||||
canvas_length=CL,
|
||||
vocab_size=VOCAB,
|
||||
max_denoising_steps=MAX_STEPS,
|
||||
device=torch.device("cuda"),
|
||||
hidden_size=4,
|
||||
stability_threshold=2,
|
||||
)
|
||||
|
||||
|
||||
def _slots(*idx: int) -> tuple[np.ndarray, torch.Tensor]:
|
||||
slots = np.array(idx, dtype=np.int64)
|
||||
return slots, torch.tensor(slots, device="cuda")
|
||||
|
||||
|
||||
def test_seed_canvas_replaces_only_seeded_slots():
|
||||
states = _states()
|
||||
for slot in range(3):
|
||||
states.add_request(slot)
|
||||
seed = list(range(CL))
|
||||
states.set_seed_canvas(1, seed)
|
||||
slots, slots_gpu = _slots(0, 1, 2)
|
||||
states.init_canvas(slots_gpu)
|
||||
before = states.canvas[slots_gpu].clone()
|
||||
|
||||
states.apply_seed_canvases(slots, slots_gpu)
|
||||
|
||||
after = states.canvas[slots_gpu]
|
||||
assert after[1].tolist() == seed
|
||||
assert torch.equal(after[0], before[0])
|
||||
assert torch.equal(after[2], before[2])
|
||||
|
||||
|
||||
def test_apply_seed_canvases_leaves_unseeded_batches_alone():
|
||||
states = _states()
|
||||
states.add_request(0)
|
||||
slots, slots_gpu = _slots(0)
|
||||
states.init_canvas(slots_gpu)
|
||||
before = states.canvas[0].clone()
|
||||
|
||||
states.apply_seed_canvases(slots, slots_gpu)
|
||||
|
||||
assert torch.equal(states.canvas[0], before)
|
||||
|
||||
|
||||
def test_add_request_clears_seed_and_read_only():
|
||||
states = _states()
|
||||
states.add_request(0)
|
||||
states.set_seed_canvas(0, [1] * CL)
|
||||
states.set_read_only(0)
|
||||
assert states.seeded_slots == {0}
|
||||
assert states.read_only_slots == {0}
|
||||
|
||||
states.add_request(0)
|
||||
|
||||
assert not states.seeded_slots
|
||||
assert not states.read_only_slots
|
||||
assert not bool(states.has_seed[0])
|
||||
assert not bool(states.read_only[0])
|
||||
|
||||
|
||||
def test_canvas_width_resets_with_the_slot():
|
||||
states = _states()
|
||||
states.add_request(0)
|
||||
states.canvas_width_np[0] = 4
|
||||
states.set_seed_canvas(0, [7, 7, 7, 7])
|
||||
assert states.seed_canvas[0, :4].tolist() == [7, 7, 7, 7]
|
||||
|
||||
states.add_request(0)
|
||||
|
||||
assert states.canvas_width_np[0] == CL
|
||||
|
||||
|
||||
def test_remove_request_forgets_the_slot():
|
||||
states = _states()
|
||||
states.add_request(0)
|
||||
states.set_seed_canvas(0, [1] * CL)
|
||||
states.set_read_only(0)
|
||||
|
||||
states.remove_request(0)
|
||||
|
||||
assert not states.seeded_slots
|
||||
assert not states.read_only_slots
|
||||
|
||||
|
||||
def _denoise_once(
|
||||
states: DiffusionGemmaRequestStates,
|
||||
slots: list[int],
|
||||
compute_sc: bool = True,
|
||||
width: int = CL,
|
||||
embed_weight: torch.Tensor | None = None,
|
||||
embed_dtype: torch.dtype = torch.float32,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""One compiled denoise step over ``slots`` with flat logits, so nothing
|
||||
converges by stability or confidence and only the step cap can end it.
|
||||
``width`` below CL runs the step on [:, :width] views, as the sampler
|
||||
does for a narrow tile."""
|
||||
n = len(slots)
|
||||
device = states.device
|
||||
decode_slots = torch.tensor(slots, dtype=torch.int64, device=device)
|
||||
decode_idx = torch.arange(n, dtype=torch.int64, device=device)
|
||||
sampled = torch.zeros(n, CL, dtype=torch.int32, device=device)[:, :width]
|
||||
num_sampled = torch.zeros(n, dtype=torch.int32, device=device)
|
||||
_compiled_sample_step(
|
||||
torch.zeros(n * width, VOCAB, device=device),
|
||||
decode_slots,
|
||||
decode_idx,
|
||||
decode_slots,
|
||||
torch.full((n,), width, dtype=torch.int64, device=device),
|
||||
states.canvas[:, :width],
|
||||
states.argmax_canvas[:, :width],
|
||||
states.step,
|
||||
states.is_encoder_phase,
|
||||
states.confident,
|
||||
states.self_conditioning_embeds[:, :width],
|
||||
(
|
||||
torch.zeros(VOCAB, 4, dtype=embed_dtype, device=device)
|
||||
if embed_weight is None
|
||||
else embed_weight
|
||||
),
|
||||
torch.tensor(1.0, dtype=embed_dtype, device=device),
|
||||
states.accepted_canvas_history[:, :, :width],
|
||||
states.accepted_canvas_history_len,
|
||||
states.max_steps,
|
||||
states.pin_mask[:, :width],
|
||||
states.seed_canvas[:, :width],
|
||||
states.read_only,
|
||||
sampled,
|
||||
num_sampled,
|
||||
torch.zeros(MAX_REQS, CL, dtype=torch.int64, device=device),
|
||||
max_denoising_steps=float(MAX_STEPS),
|
||||
t_min=0.5,
|
||||
t_max=1.0,
|
||||
confidence_threshold=0.1,
|
||||
vocab_size=VOCAB,
|
||||
CL=width,
|
||||
ST=states.stability_threshold,
|
||||
entropy_bound=0.1,
|
||||
sc_vocab_start=0,
|
||||
sc_vocab_end=VOCAB,
|
||||
tp_size=1,
|
||||
tp_group_name="",
|
||||
compute_sc=compute_sc,
|
||||
)
|
||||
return sampled, num_sampled
|
||||
|
||||
|
||||
def test_pinned_positions_hold_their_seed_through_a_step():
|
||||
states = _states()
|
||||
states.add_request(0)
|
||||
states.is_encoder_phase[0] = False
|
||||
seed = list(range(10, 10 + CL))
|
||||
states.set_seed_canvas(0, seed)
|
||||
states.canvas[0] = torch.tensor(seed, device="cuda")
|
||||
states.set_pins(0, [0, 1, 2, 3])
|
||||
|
||||
# Flat logits accept nothing, so every free position is renoised.
|
||||
_denoise_once(states, [0], embed_weight=torch.ones(VOCAB, 4, device="cuda"))
|
||||
|
||||
assert states.canvas[0, :4].tolist() == seed[:4]
|
||||
# The soft embed is zero at pinned positions and non-zero elsewhere.
|
||||
sc = states.self_conditioning_embeds[0]
|
||||
assert not sc[:4].any()
|
||||
assert sc[4:].any()
|
||||
|
||||
|
||||
def test_add_request_clears_pins():
|
||||
states = _states()
|
||||
states.add_request(0)
|
||||
states.set_seed_canvas(0, [1] * CL)
|
||||
states.set_pins(0, [2, 3])
|
||||
assert states.pin_mask[0].tolist() == [False, False, True, True] + [False] * 4
|
||||
|
||||
states.add_request(0)
|
||||
|
||||
assert not states.pin_mask[0].any()
|
||||
|
||||
|
||||
def test_single_step_tile_skips_self_conditioning():
|
||||
states = _states()
|
||||
states.add_request(0)
|
||||
states.is_encoder_phase[0] = False
|
||||
states.self_conditioning_embeds[0] = 1.0
|
||||
|
||||
_denoise_once(states, [0], compute_sc=False)
|
||||
|
||||
assert not states.self_conditioning_embeds[0].any()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stance", ["default", "force_eager"])
|
||||
def test_self_conditioning_stores_a_bf16_model_in_the_fp32_buffer(stance):
|
||||
# The model's embeddings are bf16 while the buffer is fp32. Compiled code
|
||||
# casts on the store, but eager does not, and the step runs eager once
|
||||
# torch.compile hits its recompile limit.
|
||||
states = _states()
|
||||
states.add_request(0)
|
||||
states.is_encoder_phase[0] = False
|
||||
|
||||
with torch.compiler.set_stance(stance):
|
||||
_denoise_once(
|
||||
states,
|
||||
[0],
|
||||
embed_weight=torch.ones(VOCAB, 4, dtype=torch.bfloat16, device="cuda"),
|
||||
embed_dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
assert states.self_conditioning_embeds.dtype == torch.float32
|
||||
# uniform probs @ an all-ones embedding
|
||||
assert torch.allclose(
|
||||
states.self_conditioning_embeds[0], torch.ones(CL, 4, device="cuda")
|
||||
)
|
||||
|
||||
|
||||
def test_narrow_tile_leaves_the_rest_of_the_canvas_alone():
|
||||
states = _states()
|
||||
states.add_request(0)
|
||||
states.is_encoder_phase[0] = False
|
||||
states.canvas[0] = 5
|
||||
states.argmax_canvas[0] = 5
|
||||
|
||||
_denoise_once(states, [0], width=4)
|
||||
|
||||
# Columns past the width are untouched; the step counted.
|
||||
assert states.canvas[0, 4:].tolist() == [5] * (CL - 4)
|
||||
assert states.argmax_canvas[0, 4:].tolist() == [5] * (CL - 4)
|
||||
assert states.step[0] == 1
|
||||
|
||||
|
||||
def test_step_cap_is_per_slot():
|
||||
states = _states()
|
||||
for slot in (0, 1):
|
||||
states.add_request(slot)
|
||||
states.is_encoder_phase[slot] = False
|
||||
states.max_steps[0] = 1
|
||||
|
||||
_denoise_once(states, [0, 1])
|
||||
|
||||
# Slot 0 hit its cap and moves to commit. Slot 1 keeps denoising.
|
||||
assert states.is_encoder_phase[:2].tolist() == [True, False]
|
||||
assert states.step[:2].tolist() == [1, 1]
|
||||
|
||||
|
||||
def _stash(rows: int, width: int, first_id: int) -> LogprobsTensors:
|
||||
ids = torch.arange(first_id, first_id + rows * width).reshape(rows, width)
|
||||
return LogprobsTensors(
|
||||
logprob_token_ids=ids,
|
||||
logprobs=-ids.float(),
|
||||
selected_token_ranks=torch.zeros(rows, dtype=torch.int64),
|
||||
)
|
||||
|
||||
|
||||
def test_stashes_of_different_widths_join():
|
||||
# A read that asked for 10 label ids and a generation that asked for 2
|
||||
# logprobs converged in different steps, so their stashes differ in width.
|
||||
wide, narrow = _stash(2, 11, 100), _stash(3, 3, 0)
|
||||
|
||||
out = _concat_logprob_stashes([narrow, wide], [0, 3])
|
||||
|
||||
assert out.logprob_token_ids.shape == (5, 11)
|
||||
assert out.logprobs.shape == (5, 11)
|
||||
assert out.cu_num_generated_tokens == [0, 3]
|
||||
# The narrow rows keep their values and pad with id 0 at -inf.
|
||||
assert torch.equal(out.logprob_token_ids[:3, :3], narrow.logprob_token_ids)
|
||||
assert not out.logprob_token_ids[:3, 3:].any()
|
||||
assert torch.isneginf(out.logprobs[:3, 3:]).all()
|
||||
assert torch.equal(out.logprobs[3:], wide.logprobs)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("width", [4, CL])
|
||||
@pytest.mark.parametrize("steps", [1, 2])
|
||||
def test_read_emits_at_convergence_while_generation_waits_for_commit(width, steps):
|
||||
states = _states()
|
||||
for slot in (2, 0):
|
||||
states.add_request(slot)
|
||||
states.is_encoder_phase[slot] = False
|
||||
states.max_steps[slot] = steps
|
||||
states.set_read_only(2)
|
||||
|
||||
for step in range(steps):
|
||||
sampled, counts = _denoise_once(states, [2, 0], width=width)
|
||||
assert counts.tolist() == ([width, 0] if step == steps - 1 else [0, 0])
|
||||
|
||||
assert torch.equal(sampled[0], states.argmax_canvas[2, :width].int())
|
||||
assert not states.is_encoder_phase[2]
|
||||
assert states.is_encoder_phase[0]
|
||||
assert not states.self_conditioning_embeds[2].any()
|
||||
|
||||
sampled, counts = _denoise_once(states, [0], width=width)
|
||||
assert counts.tolist() == [width]
|
||||
assert torch.equal(sampled[0], states.argmax_canvas[0, :width].int())
|
||||
Reference in New Issue
Block a user