[Core] structured generation mode for DiffusionGemma model (Jev-like) (#57250)
Signed-off-by: Matt Mastracci <matthew@mastracci.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Razorback16 <razorback16@protonmail.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Razorback16 <razorback16@protonmail.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
Lucas Wilkinson
Codex
Razorback16
parent
fae5cbfc8b
commit
1b3b88ec2b
@@ -40,6 +40,10 @@ from vllm.utils.hashing import sha256
|
||||
from vllm.v1.core.encoder_cache_manager import EncoderCacheManager
|
||||
from vllm.v1.core.kv_cache_coordinator import HybridKVCacheCoordinator
|
||||
from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash
|
||||
from vllm.v1.core.sched.diffusion_scheduler import (
|
||||
DiffusionAsyncScheduler,
|
||||
diffusion_canvas_width,
|
||||
)
|
||||
from vllm.v1.core.sched.interface import PauseState
|
||||
from vllm.v1.core.sched.output import CachedRequestData, SchedulerOutput
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
@@ -6872,3 +6876,129 @@ def test_update_draft_token_ids_in_output_strips_padding():
|
||||
-1,
|
||||
]
|
||||
assert scheduler_output.num_invalid_spec_tokens == {request.request_id: 2}
|
||||
|
||||
|
||||
def test_diffusion_canvas_width_defaults_to_the_served_canvas():
|
||||
def req(extra):
|
||||
return SimpleNamespace(sampling_params=SimpleNamespace(extra_args=extra))
|
||||
|
||||
assert diffusion_canvas_width(req(None), 64) == 64
|
||||
assert diffusion_canvas_width(req({}), 64) == 64
|
||||
assert diffusion_canvas_width(req({"diffusion_canvas_length": 16}), 64) == 16
|
||||
assert diffusion_canvas_width(SimpleNamespace(sampling_params=None), 64) == 64
|
||||
|
||||
|
||||
def _diffusion_request(req_id: str, extra_args: dict) -> Request:
|
||||
(request,) = create_requests(
|
||||
num_requests=1, num_tokens=8, max_tokens=64, req_ids=[req_id]
|
||||
)
|
||||
request.sampling_params.extra_args = extra_args
|
||||
return request
|
||||
|
||||
|
||||
def _diffusion_scheduler(**kwargs) -> DiffusionAsyncScheduler:
|
||||
scheduler = create_scheduler(
|
||||
async_scheduling=True,
|
||||
diffusion_canvas_length=8,
|
||||
scheduler_cls=DiffusionAsyncScheduler,
|
||||
**kwargs,
|
||||
)
|
||||
assert isinstance(scheduler, DiffusionAsyncScheduler)
|
||||
return scheduler
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [False, True])
|
||||
def test_diffusion_scheduler_is_selected_by_default(async_scheduling):
|
||||
config = create_scheduler(
|
||||
async_scheduling=async_scheduling, diffusion_canvas_length=8
|
||||
).vllm_config.scheduler_config
|
||||
assert config.get_scheduler_cls() is (
|
||||
DiffusionAsyncScheduler if config.async_scheduling else Scheduler
|
||||
)
|
||||
|
||||
|
||||
def test_diffusion_scheduler_narrows_the_canvas_per_request():
|
||||
scheduler = _diffusion_scheduler()
|
||||
wide = _diffusion_request("wide", {})
|
||||
narrow = _diffusion_request("narrow", {"diffusion_canvas_length": 4})
|
||||
scheduler.add_request(wide)
|
||||
scheduler.add_request(narrow)
|
||||
|
||||
# The prefill step lays down the served canvas as placeholders.
|
||||
scheduler.schedule()
|
||||
output = scheduler.schedule()
|
||||
assert output.scheduled_spec_decode_tokens["wide"] == [-1] * 8
|
||||
assert output.scheduled_spec_decode_tokens["narrow"] == [-1] * 4
|
||||
assert output.num_scheduled_tokens["narrow"] == 4
|
||||
assert narrow.num_output_placeholders == 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize("structured", [False, True])
|
||||
def test_diffusion_scheduler_trims_full_width_worker_drafts(structured):
|
||||
"""Padded worker drafts must be narrowed before scheduling or grammar validation."""
|
||||
scheduler = _diffusion_scheduler()
|
||||
wide = _diffusion_request("wide", {})
|
||||
narrow = _diffusion_request("narrow", {"diffusion_canvas_length": 4})
|
||||
for request in (wide, narrow):
|
||||
scheduler.add_request(request)
|
||||
prefill = scheduler.schedule()
|
||||
_model_output(scheduler, prefill, [[], []])
|
||||
|
||||
if structured:
|
||||
for request in (wide, narrow):
|
||||
request.structured_output_request = SimpleNamespace(
|
||||
grammar=_RecordingGrammar(), reasoning_ended=True
|
||||
)
|
||||
tokens = list(range(8)) if structured else [-1] * 8
|
||||
drafts = DraftTokenIds(["wide", "narrow"], [tokens.copy(), tokens.copy()])
|
||||
output = scheduler.schedule()
|
||||
scheduler.update_draft_token_ids_in_output(drafts, output)
|
||||
|
||||
assert output.scheduled_spec_decode_tokens == {
|
||||
"wide": tokens,
|
||||
"narrow": tokens[:4],
|
||||
}
|
||||
assert output.num_scheduled_tokens == {"wide": 8, "narrow": 4}
|
||||
if structured:
|
||||
assert wide.structured_output_request.grammar.seen == [tokens]
|
||||
assert narrow.structured_output_request.grammar.seen == [tokens[:4]]
|
||||
|
||||
|
||||
def test_diffusion_scheduler_defers_a_read_with_every_step_in_flight():
|
||||
scheduler = _diffusion_scheduler()
|
||||
one = _diffusion_request(
|
||||
"one", {"diffusion_read_only": True, "diffusion_max_steps": 1}
|
||||
)
|
||||
two = _diffusion_request(
|
||||
"two", {"diffusion_read_only": True, "diffusion_max_steps": 2}
|
||||
)
|
||||
# Only reads are deferred. A capped generation keeps its extra async step.
|
||||
gen = _diffusion_request("gen", {"diffusion_max_steps": 1})
|
||||
for request in (one, two, gen):
|
||||
scheduler.add_request(request)
|
||||
|
||||
scheduler.schedule()
|
||||
assert set(scheduler.schedule().num_scheduled_tokens) == {"one", "two", "gen"}
|
||||
assert one.num_output_placeholders == 8
|
||||
assert set(scheduler.schedule().num_scheduled_tokens) == {"two", "gen"}
|
||||
assert set(scheduler.schedule().num_scheduled_tokens) == {"gen"}
|
||||
# The deferral is renewed every call.
|
||||
assert set(scheduler.schedule().num_scheduled_tokens) == {"gen"}
|
||||
|
||||
|
||||
def test_diffusion_read_deferral_keeps_a_longer_pp_wait():
|
||||
scheduler = _diffusion_scheduler(pipeline_parallel_size=3, use_v2_model_runner=True)
|
||||
read = _diffusion_request(
|
||||
"read", {"diffusion_read_only": True, "diffusion_max_steps": 1}
|
||||
)
|
||||
scheduler.add_request(read)
|
||||
|
||||
scheduler.schedule()
|
||||
assert read.next_decode_eligible_step == 4
|
||||
for _ in range(2):
|
||||
assert "read" not in scheduler.schedule().num_scheduled_tokens
|
||||
assert "read" in scheduler.schedule().num_scheduled_tokens
|
||||
assert read.next_decode_eligible_step == 7
|
||||
# Deferring this step alone would ask for 6. The PP wait to 7 stands.
|
||||
assert "read" not in scheduler.schedule().num_scheduled_tokens
|
||||
assert read.next_decode_eligible_step == 7
|
||||
|
||||
+12
-1
@@ -7,6 +7,7 @@ import vllm.envs as envs
|
||||
from tests.v1.kv_connector.unit.utils import MockKVConfig
|
||||
from vllm.config import (
|
||||
CacheConfig,
|
||||
DiffusionConfig,
|
||||
ECTransferConfig,
|
||||
KVTransferConfig,
|
||||
ModelConfig,
|
||||
@@ -77,6 +78,8 @@ def create_scheduler(
|
||||
kv_cache_spec: KVCacheSpec | None = None,
|
||||
per_request_spec_decode_metrics: str = "none",
|
||||
scheduling_policy: SchedulerPolicy = "fcfs",
|
||||
diffusion_canvas_length: int | None = None,
|
||||
scheduler_cls: type[Scheduler] | None = None,
|
||||
) -> Scheduler | AsyncScheduler:
|
||||
"""Create scheduler under test.
|
||||
|
||||
@@ -178,6 +181,12 @@ def create_scheduler(
|
||||
else None
|
||||
)
|
||||
|
||||
diffusion_config: DiffusionConfig | None = None
|
||||
if diffusion_canvas_length is not None:
|
||||
# A diffusion checkpoint declares its canvas in the HF config.
|
||||
model_config.hf_config.canvas_length = diffusion_canvas_length
|
||||
diffusion_config = DiffusionConfig(canvas_length=diffusion_canvas_length)
|
||||
|
||||
vllm_config = VllmConfig(
|
||||
scheduler_config=scheduler_config,
|
||||
model_config=model_config,
|
||||
@@ -188,6 +197,7 @@ def create_scheduler(
|
||||
),
|
||||
kv_transfer_config=kv_transfer_config,
|
||||
speculative_config=speculative_config,
|
||||
diffusion_config=diffusion_config,
|
||||
ec_transfer_config=ec_transfer_config,
|
||||
observability_config=ObservabilityConfig(
|
||||
per_request_spec_decode_metrics=per_request_spec_decode_metrics,
|
||||
@@ -207,7 +217,8 @@ def create_scheduler(
|
||||
)
|
||||
cache_config.num_gpu_blocks = num_blocks
|
||||
register_all_kvcache_specs(vllm_config)
|
||||
scheduler_cls = AsyncScheduler if async_scheduling else Scheduler
|
||||
if scheduler_cls is None:
|
||||
scheduler_cls = AsyncScheduler if async_scheduling else Scheduler
|
||||
scheduler = scheduler_cls(
|
||||
vllm_config=vllm_config,
|
||||
kv_cache_config=kv_cache_config,
|
||||
|
||||
Reference in New Issue
Block a user