diff --git a/docs/features/speculative_decoding/dynamic_speculative_decoding.md b/docs/features/speculative_decoding/dynamic_speculative_decoding.md index eecf789d6d..aa55c5f08d 100644 --- a/docs/features/speculative_decoding/dynamic_speculative_decoding.md +++ b/docs/features/speculative_decoding/dynamic_speculative_decoding.md @@ -71,8 +71,5 @@ VLLM_USE_V2_MODEL_RUNNER=0 vllm serve meta-llama/Llama-3.1-8B-Instruct \ ## Limitations -* only tested with Eagle and Eagle-3. Other SD methods may or may not work out of the box -* only usable with Model Runner V1 -* not compatible with full cuda graph so we force piece-wise cuda graph with this feature - -We are working on enabling it on MRv2 with full cuda graph support. +* Tested with Eagle, Eagle-3, and DFlash. Other SD methods may or may not work out of the box +* Full Cudagraph only works with Model Runner V2. MRv1 only supports piece-wise cuda graph with this feature diff --git a/tests/test_config.py b/tests/test_config.py index 4e76030fd7..3837057658 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -34,6 +34,7 @@ from vllm.config.vllm import ( OptimizationLevel, ) from vllm.platforms import current_platform +from vllm.v1.attention.backend import AttentionCGSupport DEVICE_TYPE = current_platform.device_type @@ -67,6 +68,36 @@ def test_v2_model_runner_env_tri_state(monkeypatch, env_value, expected): assert envs.VLLM_USE_V2_MODEL_RUNNER is expected +@pytest.mark.parametrize( + ("use_v2_model_runner", "expected_capture_sizes"), + [ + (False, [4, 8, 12, 16]), + (True, list(range(1, 17))), + ], +) +def test_resolve_cudagraph_mode_adjusts_spec_decode_sizes_only_for_v1( + use_v2_model_runner, + expected_capture_sizes, +): + compilation_config = CompilationConfig( + cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE, + cudagraph_capture_sizes=list(range(1, 17)), + ) + compilation_config.max_cudagraph_capture_size = 16 + compilation_config.post_init_cudagraph_sizes() + + cudagraph_mode = compilation_config.resolve_cudagraph_mode_and_sizes( + AttentionCGSupport.ALWAYS, + "FakeAttentionBackend", + uniform_decode_query_len=4, + use_v2_model_runner=use_v2_model_runner, + tensor_parallel_size=1, + ) + + assert cudagraph_mode == CUDAGraphMode.FULL_AND_PIECEWISE + assert compilation_config.cudagraph_capture_sizes == expected_capture_sizes + + @pytest.mark.parametrize( ("model_config", "expected"), [ diff --git a/tests/v1/spec_decode/test_dynamic_sd_cug.py b/tests/v1/spec_decode/test_dynamic_sd_cug.py new file mode 100644 index 0000000000..d75495ea60 --- /dev/null +++ b/tests/v1/spec_decode/test_dynamic_sd_cug.py @@ -0,0 +1,328 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest +import torch + +from vllm.config import ( + CompilationConfig, + CUDAGraphMode, + ParallelConfig, + SchedulerConfig, + VllmConfig, +) +from vllm.v1.worker.gpu import cudagraph_utils as gpu_cudagraph_utils + +pytestmark = pytest.mark.cpu_test + + +def _create_vllm_config_for_dsd( + max_num_seqs: int, + max_spec_tokens: int, + *, + cudagraph_mode: str = "FULL_AND_PIECEWISE", + use_dynamic_sd: bool = True, + num_spec_per_batch_size: list[tuple[int, int, int]] | None = None, +) -> MagicMock: + """Create a minimal config that exercises DSD cudagraph dispatch. + + The test uses an exact capture-size grid so that every valid uniform decode + shape has a directly matching FULL graph candidate. + + ``num_spec_per_batch_size`` lets a test supply an explicit DSD schedule of + ``(range_start, range_end, num_speculative_tokens)`` tuples. When omitted, + a schedule covering every query length in ``[1, max_decode_query_len]`` is + generated. + """ + + max_decode_query_len = max_spec_tokens + 1 + max_capture_tokens = max_num_seqs * max_decode_query_len + + compilation_config = CompilationConfig( + cudagraph_mode=cudagraph_mode, + cudagraph_capture_sizes=list(range(1, max_capture_tokens + 1)), + ) + compilation_config.max_cudagraph_capture_size = max_capture_tokens + compilation_config.post_init_cudagraph_sizes() + + vllm_config = MagicMock(spec=VllmConfig) + vllm_config.compilation_config = compilation_config + vllm_config.scheduler_config = SchedulerConfig.default_factory( + max_num_seqs=max_num_seqs, + ) + vllm_config.parallel_config = ParallelConfig() + # num_speculative_tokens is the max K (num_speculative_steps). The manager + # recovers num_new_sampled_tokens_per_step as + # decode_query_len - num_speculative_tokens; with decode_query_len = + # max_spec_tokens + 1 this yields the normal per-step bonus of 1. + vllm_config.num_speculative_tokens = max_spec_tokens + + speculative_config = MagicMock() + speculative_config.uses_dynamic_speculative_decoding.return_value = use_dynamic_sd + if use_dynamic_sd: + # DSD reads the per-batch-size schedule; a schedule entry with K + # speculative tokens maps to decode query length K + 1. By default + # provide every query length in [1, max_decode_query_len] (i.e. K in + # [0, max_spec_tokens]) so the manager captures a FULL decode graph for + # each uniform shape. + if num_spec_per_batch_size is None: + num_spec_per_batch_size = [ + (qlen, qlen, qlen - 1) for qlen in range(1, max_decode_query_len + 1) + ] + speculative_config.num_speculative_tokens_per_batch_size = ( + num_spec_per_batch_size + ) + else: + speculative_config.num_speculative_tokens_per_batch_size = None + vllm_config.speculative_config = speculative_config + + return vllm_config + + +def test_dynamic_sd_full_cudagraph_covers_all_uniform_decode_shapes(monkeypatch): + """Dynamic SD should create FULL decode candidates for every k in [1, K+1]. + + This validates the MRv2 CudaGraphManager path directly: once candidate + shapes have been built, dispatch() should pick a FULL graph for every + uniform decode batch shape produced by DSD up to max_num_seqs. + """ + + max_num_seqs = 512 + max_spec_tokens = 7 + max_decode_query_len = max_spec_tokens + 1 + + # CudaGraphManager consults PP rank helpers during initialization even + # though this test only exercises CPU-side candidate generation. + monkeypatch.setattr( + gpu_cudagraph_utils, + "get_pp_group", + lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True), + ) + + vllm_config = _create_vllm_config_for_dsd( + max_num_seqs=max_num_seqs, + max_spec_tokens=max_spec_tokens, + ) + manager = gpu_cudagraph_utils.CudaGraphManager( + vllm_config=vllm_config, + device=torch.device("cpu"), + cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE, + decode_query_len=max_decode_query_len, + ) + + # dispatch() only uses the precomputed candidate table after graphs are + # considered captured. The actual graph objects are irrelevant here. + manager._graphs_captured = True + + for num_reqs in range(1, max_num_seqs + 1): + for max_query_len in range(1, max_decode_query_len + 1): + # Uniform decode means every request contributes the same number of + # tokens, so the total token count is exactly num_reqs * query_len. + num_tokens = num_reqs * max_query_len + uniform_tok_count = gpu_cudagraph_utils.get_uniform_token_count( + num_reqs, + num_tokens, + max_query_len, + ) + + # The scheduler should mark every one of these shapes as a uniform + # decode batch, which is what enables FULL decode graph selection. + assert uniform_tok_count == max_query_len + + desc = manager.dispatch( + num_reqs=num_reqs, + num_tokens=num_tokens, + uniform_token_count=uniform_tok_count, + num_active_loras=0, + ) + + # With DSD enabled, MRv2 should have captured a FULL candidate for + # every k in [1, K+1], so dispatch should stay on the FULL path. + assert desc.cg_mode == CUDAGraphMode.FULL + assert desc.uniform_token_count == max_query_len + assert desc.num_tokens == num_tokens + assert desc.num_reqs == num_reqs + assert desc.num_active_loras == 0 + + +def test_dynamic_sd_non_uniform_batch_falls_back_to_piecewise(monkeypatch): + """DSD should use PIECEWISE when the batch is not a uniform decode batch. + + FULL DSD graphs are captured separately for each decode query length k. + When runtime tokens are not uniform, uniform_token_count is None and those + FULL candidates should be skipped in favor of the mixed-batch PIECEWISE + graph under FULL_AND_PIECEWISE mode. + """ + + max_spec_tokens = 4 + + monkeypatch.setattr( + gpu_cudagraph_utils, + "get_pp_group", + lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True), + ) + + vllm_config = _create_vllm_config_for_dsd( + max_num_seqs=512, + max_spec_tokens=max_spec_tokens, + cudagraph_mode="FULL_AND_PIECEWISE", + use_dynamic_sd=True, + ) + manager = gpu_cudagraph_utils.CudaGraphManager( + vllm_config=vllm_config, + device=torch.device("cpu"), + cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE, + decode_query_len=max_spec_tokens + 1, + ) + manager._graphs_captured = True + + # This shape is intentionally non-uniform: 3 tokens across 2 requests + # cannot correspond to a single per-request query length. + desc = manager.dispatch( + num_reqs=2, + num_tokens=3, + uniform_token_count=None, + num_active_loras=0, + ) + + assert desc.cg_mode == CUDAGraphMode.PIECEWISE + assert desc.uniform_token_count is None + assert desc.num_reqs is None + assert desc.num_tokens == 3 + assert desc.num_active_loras == 0 + + +def test_basic_sd_does_not_capture_shorter_full_decode_shapes(monkeypatch): + """Without DSD, only the max decode query length should get FULL graphs. + + Basic SD captures FULL decode graphs only for decode_query_len = K + 1. + Uniform batches with smaller query lengths should therefore miss the FULL + path entirely when using FULL_AND_PIECEWISE. + """ + + max_num_seqs = 512 + max_spec_tokens = 7 + max_decode_query_len = max_spec_tokens + 1 + + monkeypatch.setattr( + gpu_cudagraph_utils, + "get_pp_group", + lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True), + ) + + vllm_config = _create_vllm_config_for_dsd( + max_num_seqs=max_num_seqs, + max_spec_tokens=max_spec_tokens, + cudagraph_mode="FULL_AND_PIECEWISE", + use_dynamic_sd=False, + ) + manager = gpu_cudagraph_utils.CudaGraphManager( + vllm_config=vllm_config, + device=torch.device("cpu"), + cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE, + decode_query_len=max_decode_query_len, + ) + manager._graphs_captured = True + + for num_reqs in range(1, max_num_seqs + 1): + for max_query_len in range(1, max_decode_query_len): + # These are still uniform decode batches, but basic SD should only + # have FULL graphs for query_len == max_decode_query_len. + num_tokens = num_reqs * max_query_len + uniform_tok_count = gpu_cudagraph_utils.get_uniform_token_count( + num_reqs, + num_tokens, + max_query_len, + ) + assert uniform_tok_count == max_query_len + + desc = manager.dispatch( + num_reqs=num_reqs, + num_tokens=num_tokens, + uniform_token_count=uniform_tok_count, + num_active_loras=0, + ) + + assert desc.cg_mode == CUDAGraphMode.PIECEWISE + assert desc.uniform_token_count is None + assert desc.num_tokens == num_tokens + assert desc.num_reqs is None + assert desc.num_active_loras == 0 + + +def test_dynamic_sd_only_captures_scheduled_query_lengths(monkeypatch): + """DSD should only capture FULL graphs for query lengths in the schedule. + + With a partial schedule of ``(1, 32, 4)`` and ``(32, 128, 3)``, only the + scheduled speculative-token counts (K = 4 and K = 3) become decode query + lengths (K + 1 = 5 and 4). Uniform batches at those query lengths should get + FULL graphs, while every other query length (e.g. the lower values 1, 2, 3) + must fall back to the mixed-batch PIECEWISE graph. + """ + + max_num_seqs = 128 + max_spec_tokens = 7 + max_decode_query_len = max_spec_tokens + 1 + + # (range_start, range_end, num_speculative_tokens): K = 4 and K = 3 are + # scheduled, so FULL decode graphs should exist for query lengths K + 1, + # i.e. exactly {5, 4}. + num_spec_per_batch_size = [(1, 32, 4), (32, 128, 3)] + scheduled_query_lens = {entry[2] + 1 for entry in num_spec_per_batch_size} + + monkeypatch.setattr( + gpu_cudagraph_utils, + "get_pp_group", + lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True), + ) + + vllm_config = _create_vllm_config_for_dsd( + max_num_seqs=max_num_seqs, + max_spec_tokens=max_spec_tokens, + cudagraph_mode="FULL_AND_PIECEWISE", + use_dynamic_sd=True, + num_spec_per_batch_size=num_spec_per_batch_size, + ) + manager = gpu_cudagraph_utils.CudaGraphManager( + vllm_config=vllm_config, + device=torch.device("cpu"), + cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE, + decode_query_len=max_decode_query_len, + ) + manager._graphs_captured = True + + for num_reqs in range(1, max_num_seqs + 1): + for max_query_len in range(1, max_decode_query_len + 1): + num_tokens = num_reqs * max_query_len + uniform_tok_count = gpu_cudagraph_utils.get_uniform_token_count( + num_reqs, + num_tokens, + max_query_len, + ) + assert uniform_tok_count == max_query_len + + desc = manager.dispatch( + num_reqs=num_reqs, + num_tokens=num_tokens, + uniform_token_count=uniform_tok_count, + num_active_loras=0, + ) + + if max_query_len in scheduled_query_lens: + # Scheduled query lengths get a dedicated FULL decode graph. + assert desc.cg_mode == CUDAGraphMode.FULL + assert desc.uniform_token_count == max_query_len + assert desc.num_tokens == num_tokens + assert desc.num_reqs == num_reqs + else: + # Unscheduled query lengths (including the lower values 1 and 2) + # have no FULL candidate and must fall back to PIECEWISE. + assert desc.cg_mode == CUDAGraphMode.PIECEWISE + assert desc.uniform_token_count is None + assert desc.num_tokens == num_tokens + assert desc.num_reqs is None + assert desc.num_active_loras == 0 diff --git a/vllm/config/compilation.py b/vllm/config/compilation.py index c7244d40d6..810c40131f 100644 --- a/vllm/config/compilation.py +++ b/vllm/config/compilation.py @@ -1321,6 +1321,7 @@ class CompilationConfig: min_cg_support: "AttentionCGSupport", min_cg_attn_backend: str | None, uniform_decode_query_len: int = 1, + use_v2_model_runner: bool = False, tensor_parallel_size: int = 1, kv_cache_config: "KVCacheConfig | None" = None, max_num_reqs: int | None = None, @@ -1421,13 +1422,16 @@ class CompilationConfig: "and make sure compilation mode is VLLM_COMPILE" ) - # Adjust cudagraph sizes to be a multiple of uniform_decode_query_len + # MRV1 adjusts cudagraph sizes to be a multiple of uniform_decode_query_len # to avoid: https://github.com/vllm-project/vllm/issues/28207 and temp-fix: # https://github.com/vllm-project/vllm/issues/28207#issuecomment-3504004536 # Will be removed in the near future when we have separate cudagraph capture # sizes for decode and mixed prefill-decode. + # MRV2 handles cudagraph capture sizing in cudagraph_utils.py + # and doesn't need below: https://github.com/vllm-project/vllm/pull/45953 if ( - cudagraph_mode.decode_mode() == CUDAGraphMode.FULL + not use_v2_model_runner + and cudagraph_mode.decode_mode() == CUDAGraphMode.FULL and uniform_decode_query_len > 1 ): self.adjust_cudagraph_sizes_for_spec_decode( diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index de52e5228e..c3f689c0a9 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -774,13 +774,15 @@ class VllmConfig: speculative_config is None or not speculative_config.uses_dynamic_speculative_decoding() or not self.compilation_config.cudagraph_mode.has_full_cudagraphs() + or self.use_v2_model_runner ): return logger.warning_once( "Dynamic speculative decoding changes the target verification " "length at runtime. Overriding cudagraph_mode from %s to " - "PIECEWISE for reliability.", + "PIECEWISE for reliability. Use VLLM_USE_V2_MODEL_RUNNER=1 " + "if you want to use full CUDA graphs.", self.compilation_config.cudagraph_mode.name, ) self.compilation_config.cudagraph_mode = CUDAGraphMode.PIECEWISE @@ -2062,9 +2064,6 @@ class VllmConfig: ): unsupported.append(f"speculative method '{speculative_config.method}'") - if speculative_config.uses_dynamic_speculative_decoding(): - unsupported.append("dynamic speculative decoding") - # V2 EagleSpeculator does not support parallel_drafting (for P-Eagle). # DFlash and DSpark use parallel drafting natively in V2 via their # own speculators. diff --git a/vllm/v1/worker/gpu/cudagraph_utils.py b/vllm/v1/worker/gpu/cudagraph_utils.py index aa022f6d99..9b93d17035 100644 --- a/vllm/v1/worker/gpu/cudagraph_utils.py +++ b/vllm/v1/worker/gpu/cudagraph_utils.py @@ -27,6 +27,7 @@ from vllm.logger import init_logger from vllm.model_executor.offloader.base import get_offloader from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors +from vllm.utils.math_utils import round_up from vllm.v1.kv_cache_interface import KVCacheConfig from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer from vllm.v1.worker.gpu.block_table import BlockTables @@ -141,10 +142,7 @@ class CudaGraphManager: self._candidates: dict[tuple[int, int], list[BatchExecutionDescriptor]] = {} self._capture_descs: dict[CUDAGraphMode, list[BatchExecutionDescriptor]] = {} - # adjust the cudagraph sizes to be a multiple of the uniform decode query length - self.compilation_config.adjust_cudagraph_sizes_for_spec_decode( - self.decode_query_len, self.tp_size - ) + self._init_candidates() # Breakable CUDA graph (PW CUDA graph without torch.compile) @@ -191,31 +189,72 @@ class CudaGraphManager: decode_mode = self.cudagraph_mode.decode_mode() mixed_mode = self.cudagraph_mode.mixed_mode() separate_decode_routine = self.cudagraph_mode.separate_routine() + max_cg_capture_size = self.compilation_config.max_cudagraph_capture_size descs_by_token_lora: dict[tuple[int, int], list[BatchExecutionDescriptor]] = ( defaultdict(list) ) - descs_by_mode = defaultdict(list) + descs_by_mode: defaultdict[CUDAGraphMode, list[BatchExecutionDescriptor]] = ( + defaultdict(list) + ) + + # When using Dynamic SD, num_speculative_tokens is the max number of + # draft tokens. The scheduler might use a smaller number so we need + # to capture graphs for all possible values during decode. + speculative_config = self.vllm_config.speculative_config + if ( + speculative_config + and speculative_config.uses_dynamic_speculative_decoding() + ): + num_spec_per_batch_size = ( + speculative_config.num_speculative_tokens_per_batch_size + ) + # uses_dynamic_speculative_decoding() guarantees this is set. + assert num_spec_per_batch_size is not None + # decode_query_len = num_speculative_steps + num_new_sampled_tokens + # _per_step. Recover num_new_sampled_tokens_per_step + # from the values the manager already has. + num_new_sampled_tokens_per_step = ( + self.decode_query_len - self.vllm_config.num_speculative_tokens + ) + # Each entry is (range_start, range_end, num_speculative_tokens). + decode_query_lens = [ + x[2] + num_new_sampled_tokens_per_step for x in num_spec_per_batch_size + ] + else: + decode_query_lens = [self.decode_query_len] for num_tokens, num_active_loras in product( capture_sizes, self.lora_capture_cases ): # Capture uniform decode specfifc graphs if required # (i.e. separate decode routine) - if ( - separate_decode_routine - and decode_mode - and self.decode_query_len <= num_tokens <= max_decode_tokens - ): - desc = BatchExecutionDescriptor( - cg_mode=decode_mode, - num_tokens=num_tokens, - num_reqs=num_tokens // self.decode_query_len, - uniform_token_count=self.decode_query_len, - num_active_loras=num_active_loras, - ) - descs_by_mode[decode_mode].append(desc) - descs_by_token_lora[(num_tokens, num_active_loras)].append(desc) + if separate_decode_routine and decode_mode: + for decode_query_len in decode_query_lens: + rounded_num_tokens = round_up(num_tokens, decode_query_len) + rounded_num_reqs = rounded_num_tokens // decode_query_len + + if ( + rounded_num_tokens > max_decode_tokens + or rounded_num_tokens > max_cg_capture_size + or rounded_num_reqs > self.max_num_reqs + ): + continue + + desc = BatchExecutionDescriptor( + cg_mode=decode_mode, + num_tokens=rounded_num_tokens, + num_reqs=rounded_num_reqs, + uniform_token_count=decode_query_len, + num_active_loras=num_active_loras, + ) + + # avoid duplicate graphs + if desc not in descs_by_mode[decode_mode]: + descs_by_mode[decode_mode].append(desc) + descs_by_token_lora[ + (rounded_num_tokens, num_active_loras) + ].append(desc) if mixed_mode: # for PIECEWISE graphs there is no limit on requests when replaying diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 7a82032313..c74307d0b7 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -454,9 +454,10 @@ class GPUModelRunner(LoRAModelRunnerMixin): attn_cg_support.min_cg_support, attn_cg_support.min_cg_attn_backend, self.decode_query_len, - self.parallel_config.tensor_parallel_size, - self.kv_cache_config, - self.max_num_reqs, + use_v2_model_runner=True, + tensor_parallel_size=self.parallel_config.tensor_parallel_size, + kv_cache_config=self.kv_cache_config, + max_num_reqs=self.max_num_reqs, ) self.cudagraph_manager = ModelCudaGraphManager( self.vllm_config, diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 7394d5de9f..3930a07b24 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -6930,9 +6930,10 @@ class GPUModelRunner( min_cg_support, min_cg_attn_backend, self.uniform_decode_query_len, - self.parallel_config.tensor_parallel_size, - self.kv_cache_config, - self.max_num_reqs, + use_v2_model_runner=False, + tensor_parallel_size=self.parallel_config.tensor_parallel_size, + kv_cache_config=self.kv_cache_config, + max_num_reqs=self.max_num_reqs, is_profiling=is_profiling, ) # Trigger cudagraph dispatching keys initialization after