[Core] structured generation mode for DiffusionGemma model (Jev-like) (#57250)

Signed-off-by: Matt Mastracci <matthew@mastracci.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Razorback16 <razorback16@protonmail.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Razorback16 <razorback16@protonmail.com>
This commit is contained in:
Matt Mastracci
2026-09-22 20:51:17 +00:00
committed by GitHub
co-authored by Claude Fable 5.1 Lucas Wilkinson Codex Razorback16
parent fae5cbfc8b
commit 1b3b88ec2b
16 changed files with 2795 additions and 155 deletions
+130
View File
@@ -40,6 +40,10 @@ from vllm.utils.hashing import sha256
from vllm.v1.core.encoder_cache_manager import EncoderCacheManager
from vllm.v1.core.kv_cache_coordinator import HybridKVCacheCoordinator
from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash
from vllm.v1.core.sched.diffusion_scheduler import (
DiffusionAsyncScheduler,
diffusion_canvas_width,
)
from vllm.v1.core.sched.interface import PauseState
from vllm.v1.core.sched.output import CachedRequestData, SchedulerOutput
from vllm.v1.core.sched.scheduler import Scheduler
@@ -6872,3 +6876,129 @@ def test_update_draft_token_ids_in_output_strips_padding():
-1,
]
assert scheduler_output.num_invalid_spec_tokens == {request.request_id: 2}
def test_diffusion_canvas_width_defaults_to_the_served_canvas():
def req(extra):
return SimpleNamespace(sampling_params=SimpleNamespace(extra_args=extra))
assert diffusion_canvas_width(req(None), 64) == 64
assert diffusion_canvas_width(req({}), 64) == 64
assert diffusion_canvas_width(req({"diffusion_canvas_length": 16}), 64) == 16
assert diffusion_canvas_width(SimpleNamespace(sampling_params=None), 64) == 64
def _diffusion_request(req_id: str, extra_args: dict) -> Request:
(request,) = create_requests(
num_requests=1, num_tokens=8, max_tokens=64, req_ids=[req_id]
)
request.sampling_params.extra_args = extra_args
return request
def _diffusion_scheduler(**kwargs) -> DiffusionAsyncScheduler:
scheduler = create_scheduler(
async_scheduling=True,
diffusion_canvas_length=8,
scheduler_cls=DiffusionAsyncScheduler,
**kwargs,
)
assert isinstance(scheduler, DiffusionAsyncScheduler)
return scheduler
@pytest.mark.parametrize("async_scheduling", [False, True])
def test_diffusion_scheduler_is_selected_by_default(async_scheduling):
config = create_scheduler(
async_scheduling=async_scheduling, diffusion_canvas_length=8
).vllm_config.scheduler_config
assert config.get_scheduler_cls() is (
DiffusionAsyncScheduler if config.async_scheduling else Scheduler
)
def test_diffusion_scheduler_narrows_the_canvas_per_request():
scheduler = _diffusion_scheduler()
wide = _diffusion_request("wide", {})
narrow = _diffusion_request("narrow", {"diffusion_canvas_length": 4})
scheduler.add_request(wide)
scheduler.add_request(narrow)
# The prefill step lays down the served canvas as placeholders.
scheduler.schedule()
output = scheduler.schedule()
assert output.scheduled_spec_decode_tokens["wide"] == [-1] * 8
assert output.scheduled_spec_decode_tokens["narrow"] == [-1] * 4
assert output.num_scheduled_tokens["narrow"] == 4
assert narrow.num_output_placeholders == 4
@pytest.mark.parametrize("structured", [False, True])
def test_diffusion_scheduler_trims_full_width_worker_drafts(structured):
"""Padded worker drafts must be narrowed before scheduling or grammar validation."""
scheduler = _diffusion_scheduler()
wide = _diffusion_request("wide", {})
narrow = _diffusion_request("narrow", {"diffusion_canvas_length": 4})
for request in (wide, narrow):
scheduler.add_request(request)
prefill = scheduler.schedule()
_model_output(scheduler, prefill, [[], []])
if structured:
for request in (wide, narrow):
request.structured_output_request = SimpleNamespace(
grammar=_RecordingGrammar(), reasoning_ended=True
)
tokens = list(range(8)) if structured else [-1] * 8
drafts = DraftTokenIds(["wide", "narrow"], [tokens.copy(), tokens.copy()])
output = scheduler.schedule()
scheduler.update_draft_token_ids_in_output(drafts, output)
assert output.scheduled_spec_decode_tokens == {
"wide": tokens,
"narrow": tokens[:4],
}
assert output.num_scheduled_tokens == {"wide": 8, "narrow": 4}
if structured:
assert wide.structured_output_request.grammar.seen == [tokens]
assert narrow.structured_output_request.grammar.seen == [tokens[:4]]
def test_diffusion_scheduler_defers_a_read_with_every_step_in_flight():
scheduler = _diffusion_scheduler()
one = _diffusion_request(
"one", {"diffusion_read_only": True, "diffusion_max_steps": 1}
)
two = _diffusion_request(
"two", {"diffusion_read_only": True, "diffusion_max_steps": 2}
)
# Only reads are deferred. A capped generation keeps its extra async step.
gen = _diffusion_request("gen", {"diffusion_max_steps": 1})
for request in (one, two, gen):
scheduler.add_request(request)
scheduler.schedule()
assert set(scheduler.schedule().num_scheduled_tokens) == {"one", "two", "gen"}
assert one.num_output_placeholders == 8
assert set(scheduler.schedule().num_scheduled_tokens) == {"two", "gen"}
assert set(scheduler.schedule().num_scheduled_tokens) == {"gen"}
# The deferral is renewed every call.
assert set(scheduler.schedule().num_scheduled_tokens) == {"gen"}
def test_diffusion_read_deferral_keeps_a_longer_pp_wait():
scheduler = _diffusion_scheduler(pipeline_parallel_size=3, use_v2_model_runner=True)
read = _diffusion_request(
"read", {"diffusion_read_only": True, "diffusion_max_steps": 1}
)
scheduler.add_request(read)
scheduler.schedule()
assert read.next_decode_eligible_step == 4
for _ in range(2):
assert "read" not in scheduler.schedule().num_scheduled_tokens
assert "read" in scheduler.schedule().num_scheduled_tokens
assert read.next_decode_eligible_step == 7
# Deferring this step alone would ask for 6. The PP wait to 7 stands.
assert "read" not in scheduler.schedule().num_scheduled_tokens
assert read.next_decode_eligible_step == 7
+12 -1
View File
@@ -7,6 +7,7 @@ import vllm.envs as envs
from tests.v1.kv_connector.unit.utils import MockKVConfig
from vllm.config import (
CacheConfig,
DiffusionConfig,
ECTransferConfig,
KVTransferConfig,
ModelConfig,
@@ -77,6 +78,8 @@ def create_scheduler(
kv_cache_spec: KVCacheSpec | None = None,
per_request_spec_decode_metrics: str = "none",
scheduling_policy: SchedulerPolicy = "fcfs",
diffusion_canvas_length: int | None = None,
scheduler_cls: type[Scheduler] | None = None,
) -> Scheduler | AsyncScheduler:
"""Create scheduler under test.
@@ -178,6 +181,12 @@ def create_scheduler(
else None
)
diffusion_config: DiffusionConfig | None = None
if diffusion_canvas_length is not None:
# A diffusion checkpoint declares its canvas in the HF config.
model_config.hf_config.canvas_length = diffusion_canvas_length
diffusion_config = DiffusionConfig(canvas_length=diffusion_canvas_length)
vllm_config = VllmConfig(
scheduler_config=scheduler_config,
model_config=model_config,
@@ -188,6 +197,7 @@ def create_scheduler(
),
kv_transfer_config=kv_transfer_config,
speculative_config=speculative_config,
diffusion_config=diffusion_config,
ec_transfer_config=ec_transfer_config,
observability_config=ObservabilityConfig(
per_request_spec_decode_metrics=per_request_spec_decode_metrics,
@@ -207,7 +217,8 @@ def create_scheduler(
)
cache_config.num_gpu_blocks = num_blocks
register_all_kvcache_specs(vllm_config)
scheduler_cls = AsyncScheduler if async_scheduling else Scheduler
if scheduler_cls is None:
scheduler_cls = AsyncScheduler if async_scheduling else Scheduler
scheduler = scheduler_cls(
vllm_config=vllm_config,
kv_cache_config=kv_cache_config,