[Bugfix][Spec Decode] Capture the widest uniform decode batch by default (#50488)
Signed-off-by: Rahul Chalamala <22563365+rchalamala@users.noreply.github.com> Signed-off-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Signed-off-by: Rahul Chalamala <rahul@modal.com> Co-authored-by: Janelle Cai <janelle.cai@modal.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Devin <devin@cognition.ai>
This commit is contained in:
co-authored by
Janelle Cai
Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Devin
parent
a758a9f671
commit
d3d79ffc1e
@@ -703,7 +703,9 @@ class CompilationConfig:
|
||||
If not specified, max_cudagraph_capture_size is capped at 512 by default,
|
||||
or 1024 on data center Blackwell GPUs. This avoids OOM in tight memory
|
||||
scenarios with small max_num_seqs, and limits capture of large graphs that
|
||||
increase startup time and memory usage.
|
||||
increase startup time and memory usage. When the uniform decode query length
|
||||
exceeds one, uniform decode sizes are appended and can raise the final value
|
||||
above this default.
|
||||
"""
|
||||
|
||||
dynamic_shapes_config: DynamicShapesConfig = field(
|
||||
|
||||
+122
-6
@@ -618,6 +618,27 @@ class VllmConfig:
|
||||
return self.num_speculative_tokens
|
||||
return 0
|
||||
|
||||
@property
|
||||
def uniform_decode_query_len(self) -> int:
|
||||
"""Query length of every request in a uniform decode batch.
|
||||
|
||||
A decode step submits one query for the newly sampled token plus one
|
||||
for each draft token, so the widest uniform decode batch the scheduler
|
||||
can build is `max_num_seqs * uniform_decode_query_len` tokens. Anything
|
||||
that has to cover a decode batch reads this, so the sizing rule cannot
|
||||
drift between the places that apply it.
|
||||
|
||||
This deliberately does not derive from the KV slots a drafter reserves
|
||||
past the target's query range, which is a *reservation* contract rather
|
||||
than a query-length one. The two do not differ by a constant: DFlash
|
||||
reserves `num_speculative_tokens + 1` slots yet still verifies `1 +
|
||||
num_speculative_tokens` queries, while EAGLE reserves
|
||||
`num_speculative_tokens` and verifies the same `1 + n`. Deriving one
|
||||
from the other would under-size EAGLE by a full request width, which is
|
||||
the failure this property exists to prevent.
|
||||
"""
|
||||
return 1 + self.num_speculative_tokens
|
||||
|
||||
@property
|
||||
def use_v2_model_runner(self) -> bool:
|
||||
use_v2_model_runner = envs.VLLM_USE_V2_MODEL_RUNNER
|
||||
@@ -1897,8 +1918,10 @@ class VllmConfig:
|
||||
|
||||
```python
|
||||
default_max_graph_size = 1024 if is_data_center_blackwell else 512
|
||||
max_graph_size = min(max_num_seqs * decode_query_len * 2,
|
||||
default_max_graph_size)
|
||||
decode_query_len = self.uniform_decode_query_len
|
||||
max_graph_size = min(
|
||||
max_num_seqs * decode_query_len * 2, default_max_graph_size
|
||||
)
|
||||
# 1, 2, 4, then multiples of 8 up to 256 and then multiples of 16
|
||||
# up to max_graph_size
|
||||
cudagraph_capture_sizes = [1, 2, 4] + list(range(8, 256, 8)) + list(
|
||||
@@ -1906,7 +1929,10 @@ class VllmConfig:
|
||||
|
||||
`max_num_batched_tokens` is also appended to the list if it fits
|
||||
within `max_cudagraph_capture_size`, so the max batch size is captured
|
||||
even when off-stride.
|
||||
even when off-stride. Likewise, when the uniform decode query length
|
||||
exceeds one, the widest uniform decode batch (`max_num_seqs *
|
||||
decode_query_len`) is appended when it fits, since it need not land on
|
||||
an 8- or 16-token stride.
|
||||
|
||||
In the end, `vllm_config.compilation_config.cudagraph_capture_sizes`
|
||||
will be the final sizes to capture cudagraph (in ascending order).
|
||||
@@ -1945,17 +1971,98 @@ class VllmConfig:
|
||||
max_cudagraph_capture_size = (
|
||||
self.compilation_config.max_cudagraph_capture_size
|
||||
)
|
||||
# Decode sizes to cover, in tokens. Populated only when a request
|
||||
# is more than one token wide and only when the default is computed
|
||||
# here, so an explicit capture range is left exactly as configured.
|
||||
uniform_decode_sizes: list[int] = []
|
||||
if max_cudagraph_capture_size is None:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
decode_query_len = 1 + self.num_speculative_tokens
|
||||
default_max_graph_size = (
|
||||
1024 if current_platform.is_device_capability_family(100) else 512
|
||||
)
|
||||
decode_query_len = self.uniform_decode_query_len
|
||||
max_num_seqs = self.scheduler_config.max_num_seqs
|
||||
max_cudagraph_capture_size = min(
|
||||
self.scheduler_config.max_num_seqs * decode_query_len * 2,
|
||||
default_max_graph_size,
|
||||
max_num_seqs * decode_query_len * 2, default_max_graph_size
|
||||
)
|
||||
if decode_query_len > 1:
|
||||
# A uniform decode batch is decode_query_len tokens per
|
||||
# request, so the widest one is far outside this ceiling.
|
||||
# Coverage comes from appending the decode sizes rather than
|
||||
# extending the token-strided grid. Extending that grid to
|
||||
# the widest decode size produces 581 sizes at
|
||||
# max_num_seqs=512 and 16 draft tokens, versus 100 with the
|
||||
# request-count grid.
|
||||
#
|
||||
# The grid would not buy decode coverage anyway. Dispatch
|
||||
# requires an exact multiple of decode_query_len, so a
|
||||
# token-strided entry is only usable when it happens to be
|
||||
# one; at query length 17 a captured 560 rounds to 561 and
|
||||
# is rejected. Scaling a request-count grid keeps every
|
||||
# entry usable and the count comparable to the non-
|
||||
# speculative case.
|
||||
def request_counts(max_reqs: int) -> list[int]:
|
||||
# At most the platform default number of requests,
|
||||
# mirroring the one-token-per-request decode ceiling.
|
||||
max_reqs = min(max_reqs, default_max_graph_size)
|
||||
counts = [n for n in (1, 2, 4) if n <= max_reqs]
|
||||
counts += list(range(8, min(max_reqs + 1, 256), 8))
|
||||
counts += list(range(256, max_reqs + 1, 16))
|
||||
return sorted(set(counts + [max_reqs]))
|
||||
|
||||
# Dynamic speculative decoding picks the draft width from
|
||||
# the batch size, so a decode step is only uniform within a
|
||||
# tier and each tier needs its own sizes. Scaling by the
|
||||
# widest one alone leaves the narrower tiers short: the
|
||||
# manager rounds a capture size up to a multiple of the
|
||||
# tier's query length and drops it once the implied request
|
||||
# count exceeds max_num_seqs, so at query length 3 sizes
|
||||
# built from 17 stop covering at 227 of 256 requests.
|
||||
decode_tiers = [(decode_query_len, max_num_seqs)]
|
||||
speculative_config = self.speculative_config
|
||||
if (
|
||||
speculative_config is not None
|
||||
and speculative_config.uses_dynamic_speculative_decoding()
|
||||
):
|
||||
from vllm.v1.spec_decode.dynamic.utils import (
|
||||
build_dynamic_sd_schedule_lookup,
|
||||
)
|
||||
|
||||
schedule = (
|
||||
speculative_config.num_speculative_tokens_per_batch_size
|
||||
)
|
||||
assert schedule is not None
|
||||
# Read the tiers off the dense lookup the scheduler
|
||||
# runs on, so the clamp against num_speculative_tokens
|
||||
# and the carry-forward through gaps and the tail
|
||||
# cannot drift from it. Validation lives elsewhere; an
|
||||
# invalid schedule keeps the single-tier default.
|
||||
try:
|
||||
dense_schedule = build_dynamic_sd_schedule_lookup(
|
||||
schedule,
|
||||
vllm_max_batch_size=max_num_seqs,
|
||||
vllm_num_speculative_tokens=self.num_speculative_tokens,
|
||||
)
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
# Ascending batch size, so the last write per
|
||||
# query length is the widest batch running at it.
|
||||
widest_batch: dict[int, int] = {}
|
||||
for batch_size, num_spec in enumerate(
|
||||
dense_schedule[1:], start=1
|
||||
):
|
||||
widest_batch[num_spec + 1] = batch_size
|
||||
decode_tiers = list(widest_batch.items())
|
||||
|
||||
uniform_decode_sizes = sorted(
|
||||
{
|
||||
n * query_len
|
||||
for query_len, tier_max_reqs in decode_tiers
|
||||
for n in request_counts(tier_max_reqs)
|
||||
}
|
||||
)
|
||||
max_num_tokens = self.scheduler_config.max_num_batched_tokens
|
||||
max_cudagraph_capture_size = min(max_num_tokens, max_cudagraph_capture_size)
|
||||
|
||||
@@ -2003,6 +2110,13 @@ class VllmConfig:
|
||||
and max_num_tokens not in cudagraph_capture_sizes
|
||||
):
|
||||
cudagraph_capture_sizes.append(max_num_tokens)
|
||||
# These extend past the token-strided ceiling, which counts one
|
||||
# token per request. valid_max_size below raises the final
|
||||
# max_cudagraph_capture_size to the widest of them. They remain
|
||||
# filtered by max_num_tokens.
|
||||
cudagraph_capture_sizes += [
|
||||
size for size in uniform_decode_sizes if size <= max_num_tokens
|
||||
]
|
||||
# de-duplicate and sort the sizes
|
||||
cudagraph_capture_sizes = sorted(set(cudagraph_capture_sizes))
|
||||
|
||||
@@ -2010,6 +2124,8 @@ class VllmConfig:
|
||||
self.parallel_config.tensor_parallel_size > 1
|
||||
and self.compilation_config.pass_config.enable_sp
|
||||
):
|
||||
# Sequence parallelism only captures TP-divisible sizes, so a
|
||||
# wider non-divisible decode batch cannot be captured under SP.
|
||||
cudagraph_capture_sizes = self.update_sizes_for_sequence_parallelism(
|
||||
cudagraph_capture_sizes
|
||||
)
|
||||
|
||||
@@ -34,6 +34,7 @@ from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.utils.torch_utils import current_stream
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.spec_decode.dynamic.utils import build_dynamic_sd_schedule_lookup
|
||||
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens
|
||||
@@ -209,21 +210,23 @@ class CudaGraphManager:
|
||||
speculative_config
|
||||
and speculative_config.uses_dynamic_speculative_decoding()
|
||||
):
|
||||
num_spec_per_batch_size = (
|
||||
speculative_config.num_speculative_tokens_per_batch_size
|
||||
)
|
||||
# uses_dynamic_speculative_decoding() guarantees this is set.
|
||||
assert num_spec_per_batch_size is not None
|
||||
# decode_query_len = num_speculative_steps + num_new_sampled_tokens
|
||||
# _per_step. Recover num_new_sampled_tokens_per_step
|
||||
# from the values the manager already has.
|
||||
num_new_sampled_tokens_per_step = (
|
||||
self.decode_query_len - self.vllm_config.num_speculative_tokens
|
||||
)
|
||||
# Each entry is (range_start, range_end, num_speculative_tokens).
|
||||
decode_query_lens = [
|
||||
x[2] + num_new_sampled_tokens_per_step for x in num_spec_per_batch_size
|
||||
]
|
||||
dense_schedule = build_dynamic_sd_schedule_lookup(
|
||||
speculative_config.num_speculative_tokens_per_batch_size,
|
||||
vllm_max_batch_size=self.max_num_reqs,
|
||||
vllm_num_speculative_tokens=self.vllm_config.num_speculative_tokens,
|
||||
)
|
||||
decode_query_lens = sorted(
|
||||
{
|
||||
num_spec + num_new_sampled_tokens_per_step
|
||||
for num_spec in dense_schedule[1:]
|
||||
}
|
||||
)
|
||||
else:
|
||||
decode_query_lens = [self.decode_query_len]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user