[Bugfix][Spec Decode] Capture the widest uniform decode batch by default (#50488)

Signed-off-by: Rahul Chalamala <22563365+rchalamala@users.noreply.github.com>
Signed-off-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Signed-off-by: Rahul Chalamala <rahul@modal.com>
Co-authored-by: Janelle Cai <janelle.cai@modal.com>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Co-authored-by: Devin <devin@cognition.ai>
This commit is contained in:
Rahul Chalamala
2026-08-28 22:38:13 -07:00
committed by GitHub
co-authored by Janelle Cai Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Devin
parent a758a9f671
commit d3d79ffc1e
6 changed files with 621 additions and 30 deletions
+3 -1
View File
@@ -703,7 +703,9 @@ class CompilationConfig:
If not specified, max_cudagraph_capture_size is capped at 512 by default,
or 1024 on data center Blackwell GPUs. This avoids OOM in tight memory
scenarios with small max_num_seqs, and limits capture of large graphs that
increase startup time and memory usage.
increase startup time and memory usage. When the uniform decode query length
exceeds one, uniform decode sizes are appended and can raise the final value
above this default.
"""
dynamic_shapes_config: DynamicShapesConfig = field(
+122 -6
View File
@@ -618,6 +618,27 @@ class VllmConfig:
return self.num_speculative_tokens
return 0
@property
def uniform_decode_query_len(self) -> int:
"""Query length of every request in a uniform decode batch.
A decode step submits one query for the newly sampled token plus one
for each draft token, so the widest uniform decode batch the scheduler
can build is `max_num_seqs * uniform_decode_query_len` tokens. Anything
that has to cover a decode batch reads this, so the sizing rule cannot
drift between the places that apply it.
This deliberately does not derive from the KV slots a drafter reserves
past the target's query range, which is a *reservation* contract rather
than a query-length one. The two do not differ by a constant: DFlash
reserves `num_speculative_tokens + 1` slots yet still verifies `1 +
num_speculative_tokens` queries, while EAGLE reserves
`num_speculative_tokens` and verifies the same `1 + n`. Deriving one
from the other would under-size EAGLE by a full request width, which is
the failure this property exists to prevent.
"""
return 1 + self.num_speculative_tokens
@property
def use_v2_model_runner(self) -> bool:
use_v2_model_runner = envs.VLLM_USE_V2_MODEL_RUNNER
@@ -1897,8 +1918,10 @@ class VllmConfig:
```python
default_max_graph_size = 1024 if is_data_center_blackwell else 512
max_graph_size = min(max_num_seqs * decode_query_len * 2,
default_max_graph_size)
decode_query_len = self.uniform_decode_query_len
max_graph_size = min(
max_num_seqs * decode_query_len * 2, default_max_graph_size
)
# 1, 2, 4, then multiples of 8 up to 256 and then multiples of 16
# up to max_graph_size
cudagraph_capture_sizes = [1, 2, 4] + list(range(8, 256, 8)) + list(
@@ -1906,7 +1929,10 @@ class VllmConfig:
`max_num_batched_tokens` is also appended to the list if it fits
within `max_cudagraph_capture_size`, so the max batch size is captured
even when off-stride.
even when off-stride. Likewise, when the uniform decode query length
exceeds one, the widest uniform decode batch (`max_num_seqs *
decode_query_len`) is appended when it fits, since it need not land on
an 8- or 16-token stride.
In the end, `vllm_config.compilation_config.cudagraph_capture_sizes`
will be the final sizes to capture cudagraph (in ascending order).
@@ -1945,17 +1971,98 @@ class VllmConfig:
max_cudagraph_capture_size = (
self.compilation_config.max_cudagraph_capture_size
)
# Decode sizes to cover, in tokens. Populated only when a request
# is more than one token wide and only when the default is computed
# here, so an explicit capture range is left exactly as configured.
uniform_decode_sizes: list[int] = []
if max_cudagraph_capture_size is None:
from vllm.platforms import current_platform
decode_query_len = 1 + self.num_speculative_tokens
default_max_graph_size = (
1024 if current_platform.is_device_capability_family(100) else 512
)
decode_query_len = self.uniform_decode_query_len
max_num_seqs = self.scheduler_config.max_num_seqs
max_cudagraph_capture_size = min(
self.scheduler_config.max_num_seqs * decode_query_len * 2,
default_max_graph_size,
max_num_seqs * decode_query_len * 2, default_max_graph_size
)
if decode_query_len > 1:
# A uniform decode batch is decode_query_len tokens per
# request, so the widest one is far outside this ceiling.
# Coverage comes from appending the decode sizes rather than
# extending the token-strided grid. Extending that grid to
# the widest decode size produces 581 sizes at
# max_num_seqs=512 and 16 draft tokens, versus 100 with the
# request-count grid.
#
# The grid would not buy decode coverage anyway. Dispatch
# requires an exact multiple of decode_query_len, so a
# token-strided entry is only usable when it happens to be
# one; at query length 17 a captured 560 rounds to 561 and
# is rejected. Scaling a request-count grid keeps every
# entry usable and the count comparable to the non-
# speculative case.
def request_counts(max_reqs: int) -> list[int]:
# At most the platform default number of requests,
# mirroring the one-token-per-request decode ceiling.
max_reqs = min(max_reqs, default_max_graph_size)
counts = [n for n in (1, 2, 4) if n <= max_reqs]
counts += list(range(8, min(max_reqs + 1, 256), 8))
counts += list(range(256, max_reqs + 1, 16))
return sorted(set(counts + [max_reqs]))
# Dynamic speculative decoding picks the draft width from
# the batch size, so a decode step is only uniform within a
# tier and each tier needs its own sizes. Scaling by the
# widest one alone leaves the narrower tiers short: the
# manager rounds a capture size up to a multiple of the
# tier's query length and drops it once the implied request
# count exceeds max_num_seqs, so at query length 3 sizes
# built from 17 stop covering at 227 of 256 requests.
decode_tiers = [(decode_query_len, max_num_seqs)]
speculative_config = self.speculative_config
if (
speculative_config is not None
and speculative_config.uses_dynamic_speculative_decoding()
):
from vllm.v1.spec_decode.dynamic.utils import (
build_dynamic_sd_schedule_lookup,
)
schedule = (
speculative_config.num_speculative_tokens_per_batch_size
)
assert schedule is not None
# Read the tiers off the dense lookup the scheduler
# runs on, so the clamp against num_speculative_tokens
# and the carry-forward through gaps and the tail
# cannot drift from it. Validation lives elsewhere; an
# invalid schedule keeps the single-tier default.
try:
dense_schedule = build_dynamic_sd_schedule_lookup(
schedule,
vllm_max_batch_size=max_num_seqs,
vllm_num_speculative_tokens=self.num_speculative_tokens,
)
except ValueError:
pass
else:
# Ascending batch size, so the last write per
# query length is the widest batch running at it.
widest_batch: dict[int, int] = {}
for batch_size, num_spec in enumerate(
dense_schedule[1:], start=1
):
widest_batch[num_spec + 1] = batch_size
decode_tiers = list(widest_batch.items())
uniform_decode_sizes = sorted(
{
n * query_len
for query_len, tier_max_reqs in decode_tiers
for n in request_counts(tier_max_reqs)
}
)
max_num_tokens = self.scheduler_config.max_num_batched_tokens
max_cudagraph_capture_size = min(max_num_tokens, max_cudagraph_capture_size)
@@ -2003,6 +2110,13 @@ class VllmConfig:
and max_num_tokens not in cudagraph_capture_sizes
):
cudagraph_capture_sizes.append(max_num_tokens)
# These extend past the token-strided ceiling, which counts one
# token per request. valid_max_size below raises the final
# max_cudagraph_capture_size to the widest of them. They remain
# filtered by max_num_tokens.
cudagraph_capture_sizes += [
size for size in uniform_decode_sizes if size <= max_num_tokens
]
# de-duplicate and sort the sizes
cudagraph_capture_sizes = sorted(set(cudagraph_capture_sizes))
@@ -2010,6 +2124,8 @@ class VllmConfig:
self.parallel_config.tensor_parallel_size > 1
and self.compilation_config.pass_config.enable_sp
):
# Sequence parallelism only captures TP-divisible sizes, so a
# wider non-divisible decode batch cannot be captured under SP.
cudagraph_capture_sizes = self.update_sizes_for_sequence_parallelism(
cudagraph_capture_sizes
)
+12 -9
View File
@@ -34,6 +34,7 @@ from vllm.sequence import IntermediateTensors
from vllm.utils.math_utils import round_up
from vllm.utils.torch_utils import current_stream
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.spec_decode.dynamic.utils import build_dynamic_sd_schedule_lookup
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens
@@ -209,21 +210,23 @@ class CudaGraphManager:
speculative_config
and speculative_config.uses_dynamic_speculative_decoding()
):
num_spec_per_batch_size = (
speculative_config.num_speculative_tokens_per_batch_size
)
# uses_dynamic_speculative_decoding() guarantees this is set.
assert num_spec_per_batch_size is not None
# decode_query_len = num_speculative_steps + num_new_sampled_tokens
# _per_step. Recover num_new_sampled_tokens_per_step
# from the values the manager already has.
num_new_sampled_tokens_per_step = (
self.decode_query_len - self.vllm_config.num_speculative_tokens
)
# Each entry is (range_start, range_end, num_speculative_tokens).
decode_query_lens = [
x[2] + num_new_sampled_tokens_per_step for x in num_spec_per_batch_size
]
dense_schedule = build_dynamic_sd_schedule_lookup(
speculative_config.num_speculative_tokens_per_batch_size,
vllm_max_batch_size=self.max_num_reqs,
vllm_num_speculative_tokens=self.vllm_config.num_speculative_tokens,
)
decode_query_lens = sorted(
{
num_spec + num_new_sampled_tokens_per_step
for num_spec in dense_schedule[1:]
}
)
else:
decode_query_lens = [self.decode_query_len]