[Spec Decode] Enable Gemma4 DSpark adaptive verification with FlashInfer (#57263)

Signed-off-by: zixi-qi <zixi@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
qizixi
2026-09-26 17:01:17 +08:00
committed by GitHub
co-authored by OpenAI Codex Claude Opus 5.5 Nick Hill
parent 3b4566c5cf
commit a4eb3f25d6
24 changed files with 894 additions and 135 deletions
@@ -294,6 +294,7 @@ def _make_spec_decode_manager(
dynamic_spec_schedule: list[tuple[int, int, int]] | None = None,
max_num_seqs: int = 8,
use_kda_recoverssm: bool = False,
varlen_decode: bool = False,
) -> gpu_cudagraph_utils.CudaGraphManager:
monkeypatch.setattr(
gpu_cudagraph_utils,
@@ -316,6 +317,7 @@ def _make_spec_decode_manager(
device=torch.device("cpu"),
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
decode_query_len=decode_query_len,
varlen_decode=varlen_decode,
)
manager._graphs_captured = True
return manager
@@ -373,6 +375,28 @@ def test_mixed_batch_never_selects_a_uniform_decode_graph(monkeypatch):
assert desc.num_tokens == 16
def test_varlen_decode_graph_takes_bounded_decode_batches_only(monkeypatch):
"""A varlen decode graph replays any decode batch within its query-length
bound. The runner passes no bound for a batch with a prefill, which must not
replay one however short the prefill is."""
manager = _make_spec_decode_manager(monkeypatch, varlen_decode=True)
def dispatch(max_query_len):
return manager.dispatch(
num_reqs=3,
num_tokens=6,
uniform_token_count=None,
num_active_loras=0,
max_query_len=max_query_len,
)
desc = dispatch(_DECODE_QUERY_LEN)
assert desc.cg_mode == CUDAGraphMode.FULL
assert desc.max_query_len == _DECODE_QUERY_LEN
assert dispatch(None).cg_mode == CUDAGraphMode.PIECEWISE
assert dispatch(_DECODE_QUERY_LEN + 1).cg_mode == CUDAGraphMode.PIECEWISE
def test_mixed_batch_at_decode_only_token_count_still_gets_a_graph(monkeypatch):
"""A mixed batch must not fall to eager where only decode graphs are staged.