[MRV2][SD] Make Dynamic SD comatible with Full Cuda Graphs (#45953)
Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com> Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Benjamin Chislett
Nick Hill
parent
67ff0ae30f
commit
07516fda67
@@ -71,8 +71,5 @@ VLLM_USE_V2_MODEL_RUNNER=0 vllm serve meta-llama/Llama-3.1-8B-Instruct \
|
||||
|
||||
## Limitations
|
||||
|
||||
* only tested with Eagle and Eagle-3. Other SD methods may or may not work out of the box
|
||||
* only usable with Model Runner V1
|
||||
* not compatible with full cuda graph so we force piece-wise cuda graph with this feature
|
||||
|
||||
We are working on enabling it on MRv2 with full cuda graph support.
|
||||
* Tested with Eagle, Eagle-3, and DFlash. Other SD methods may or may not work out of the box
|
||||
* Full Cudagraph only works with Model Runner V2. MRv1 only supports piece-wise cuda graph with this feature
|
||||
|
||||
@@ -34,6 +34,7 @@ from vllm.config.vllm import (
|
||||
OptimizationLevel,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backend import AttentionCGSupport
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
@@ -67,6 +68,36 @@ def test_v2_model_runner_env_tri_state(monkeypatch, env_value, expected):
|
||||
assert envs.VLLM_USE_V2_MODEL_RUNNER is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("use_v2_model_runner", "expected_capture_sizes"),
|
||||
[
|
||||
(False, [4, 8, 12, 16]),
|
||||
(True, list(range(1, 17))),
|
||||
],
|
||||
)
|
||||
def test_resolve_cudagraph_mode_adjusts_spec_decode_sizes_only_for_v1(
|
||||
use_v2_model_runner,
|
||||
expected_capture_sizes,
|
||||
):
|
||||
compilation_config = CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
cudagraph_capture_sizes=list(range(1, 17)),
|
||||
)
|
||||
compilation_config.max_cudagraph_capture_size = 16
|
||||
compilation_config.post_init_cudagraph_sizes()
|
||||
|
||||
cudagraph_mode = compilation_config.resolve_cudagraph_mode_and_sizes(
|
||||
AttentionCGSupport.ALWAYS,
|
||||
"FakeAttentionBackend",
|
||||
uniform_decode_query_len=4,
|
||||
use_v2_model_runner=use_v2_model_runner,
|
||||
tensor_parallel_size=1,
|
||||
)
|
||||
|
||||
assert cudagraph_mode == CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
assert compilation_config.cudagraph_capture_sizes == expected_capture_sizes
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model_config", "expected"),
|
||||
[
|
||||
|
||||
@@ -0,0 +1,328 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import (
|
||||
CompilationConfig,
|
||||
CUDAGraphMode,
|
||||
ParallelConfig,
|
||||
SchedulerConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.v1.worker.gpu import cudagraph_utils as gpu_cudagraph_utils
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
def _create_vllm_config_for_dsd(
|
||||
max_num_seqs: int,
|
||||
max_spec_tokens: int,
|
||||
*,
|
||||
cudagraph_mode: str = "FULL_AND_PIECEWISE",
|
||||
use_dynamic_sd: bool = True,
|
||||
num_spec_per_batch_size: list[tuple[int, int, int]] | None = None,
|
||||
) -> MagicMock:
|
||||
"""Create a minimal config that exercises DSD cudagraph dispatch.
|
||||
|
||||
The test uses an exact capture-size grid so that every valid uniform decode
|
||||
shape has a directly matching FULL graph candidate.
|
||||
|
||||
``num_spec_per_batch_size`` lets a test supply an explicit DSD schedule of
|
||||
``(range_start, range_end, num_speculative_tokens)`` tuples. When omitted,
|
||||
a schedule covering every query length in ``[1, max_decode_query_len]`` is
|
||||
generated.
|
||||
"""
|
||||
|
||||
max_decode_query_len = max_spec_tokens + 1
|
||||
max_capture_tokens = max_num_seqs * max_decode_query_len
|
||||
|
||||
compilation_config = CompilationConfig(
|
||||
cudagraph_mode=cudagraph_mode,
|
||||
cudagraph_capture_sizes=list(range(1, max_capture_tokens + 1)),
|
||||
)
|
||||
compilation_config.max_cudagraph_capture_size = max_capture_tokens
|
||||
compilation_config.post_init_cudagraph_sizes()
|
||||
|
||||
vllm_config = MagicMock(spec=VllmConfig)
|
||||
vllm_config.compilation_config = compilation_config
|
||||
vllm_config.scheduler_config = SchedulerConfig.default_factory(
|
||||
max_num_seqs=max_num_seqs,
|
||||
)
|
||||
vllm_config.parallel_config = ParallelConfig()
|
||||
# num_speculative_tokens is the max K (num_speculative_steps). The manager
|
||||
# recovers num_new_sampled_tokens_per_step as
|
||||
# decode_query_len - num_speculative_tokens; with decode_query_len =
|
||||
# max_spec_tokens + 1 this yields the normal per-step bonus of 1.
|
||||
vllm_config.num_speculative_tokens = max_spec_tokens
|
||||
|
||||
speculative_config = MagicMock()
|
||||
speculative_config.uses_dynamic_speculative_decoding.return_value = use_dynamic_sd
|
||||
if use_dynamic_sd:
|
||||
# DSD reads the per-batch-size schedule; a schedule entry with K
|
||||
# speculative tokens maps to decode query length K + 1. By default
|
||||
# provide every query length in [1, max_decode_query_len] (i.e. K in
|
||||
# [0, max_spec_tokens]) so the manager captures a FULL decode graph for
|
||||
# each uniform shape.
|
||||
if num_spec_per_batch_size is None:
|
||||
num_spec_per_batch_size = [
|
||||
(qlen, qlen, qlen - 1) for qlen in range(1, max_decode_query_len + 1)
|
||||
]
|
||||
speculative_config.num_speculative_tokens_per_batch_size = (
|
||||
num_spec_per_batch_size
|
||||
)
|
||||
else:
|
||||
speculative_config.num_speculative_tokens_per_batch_size = None
|
||||
vllm_config.speculative_config = speculative_config
|
||||
|
||||
return vllm_config
|
||||
|
||||
|
||||
def test_dynamic_sd_full_cudagraph_covers_all_uniform_decode_shapes(monkeypatch):
|
||||
"""Dynamic SD should create FULL decode candidates for every k in [1, K+1].
|
||||
|
||||
This validates the MRv2 CudaGraphManager path directly: once candidate
|
||||
shapes have been built, dispatch() should pick a FULL graph for every
|
||||
uniform decode batch shape produced by DSD up to max_num_seqs.
|
||||
"""
|
||||
|
||||
max_num_seqs = 512
|
||||
max_spec_tokens = 7
|
||||
max_decode_query_len = max_spec_tokens + 1
|
||||
|
||||
# CudaGraphManager consults PP rank helpers during initialization even
|
||||
# though this test only exercises CPU-side candidate generation.
|
||||
monkeypatch.setattr(
|
||||
gpu_cudagraph_utils,
|
||||
"get_pp_group",
|
||||
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
|
||||
)
|
||||
|
||||
vllm_config = _create_vllm_config_for_dsd(
|
||||
max_num_seqs=max_num_seqs,
|
||||
max_spec_tokens=max_spec_tokens,
|
||||
)
|
||||
manager = gpu_cudagraph_utils.CudaGraphManager(
|
||||
vllm_config=vllm_config,
|
||||
device=torch.device("cpu"),
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
decode_query_len=max_decode_query_len,
|
||||
)
|
||||
|
||||
# dispatch() only uses the precomputed candidate table after graphs are
|
||||
# considered captured. The actual graph objects are irrelevant here.
|
||||
manager._graphs_captured = True
|
||||
|
||||
for num_reqs in range(1, max_num_seqs + 1):
|
||||
for max_query_len in range(1, max_decode_query_len + 1):
|
||||
# Uniform decode means every request contributes the same number of
|
||||
# tokens, so the total token count is exactly num_reqs * query_len.
|
||||
num_tokens = num_reqs * max_query_len
|
||||
uniform_tok_count = gpu_cudagraph_utils.get_uniform_token_count(
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
max_query_len,
|
||||
)
|
||||
|
||||
# The scheduler should mark every one of these shapes as a uniform
|
||||
# decode batch, which is what enables FULL decode graph selection.
|
||||
assert uniform_tok_count == max_query_len
|
||||
|
||||
desc = manager.dispatch(
|
||||
num_reqs=num_reqs,
|
||||
num_tokens=num_tokens,
|
||||
uniform_token_count=uniform_tok_count,
|
||||
num_active_loras=0,
|
||||
)
|
||||
|
||||
# With DSD enabled, MRv2 should have captured a FULL candidate for
|
||||
# every k in [1, K+1], so dispatch should stay on the FULL path.
|
||||
assert desc.cg_mode == CUDAGraphMode.FULL
|
||||
assert desc.uniform_token_count == max_query_len
|
||||
assert desc.num_tokens == num_tokens
|
||||
assert desc.num_reqs == num_reqs
|
||||
assert desc.num_active_loras == 0
|
||||
|
||||
|
||||
def test_dynamic_sd_non_uniform_batch_falls_back_to_piecewise(monkeypatch):
|
||||
"""DSD should use PIECEWISE when the batch is not a uniform decode batch.
|
||||
|
||||
FULL DSD graphs are captured separately for each decode query length k.
|
||||
When runtime tokens are not uniform, uniform_token_count is None and those
|
||||
FULL candidates should be skipped in favor of the mixed-batch PIECEWISE
|
||||
graph under FULL_AND_PIECEWISE mode.
|
||||
"""
|
||||
|
||||
max_spec_tokens = 4
|
||||
|
||||
monkeypatch.setattr(
|
||||
gpu_cudagraph_utils,
|
||||
"get_pp_group",
|
||||
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
|
||||
)
|
||||
|
||||
vllm_config = _create_vllm_config_for_dsd(
|
||||
max_num_seqs=512,
|
||||
max_spec_tokens=max_spec_tokens,
|
||||
cudagraph_mode="FULL_AND_PIECEWISE",
|
||||
use_dynamic_sd=True,
|
||||
)
|
||||
manager = gpu_cudagraph_utils.CudaGraphManager(
|
||||
vllm_config=vllm_config,
|
||||
device=torch.device("cpu"),
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
decode_query_len=max_spec_tokens + 1,
|
||||
)
|
||||
manager._graphs_captured = True
|
||||
|
||||
# This shape is intentionally non-uniform: 3 tokens across 2 requests
|
||||
# cannot correspond to a single per-request query length.
|
||||
desc = manager.dispatch(
|
||||
num_reqs=2,
|
||||
num_tokens=3,
|
||||
uniform_token_count=None,
|
||||
num_active_loras=0,
|
||||
)
|
||||
|
||||
assert desc.cg_mode == CUDAGraphMode.PIECEWISE
|
||||
assert desc.uniform_token_count is None
|
||||
assert desc.num_reqs is None
|
||||
assert desc.num_tokens == 3
|
||||
assert desc.num_active_loras == 0
|
||||
|
||||
|
||||
def test_basic_sd_does_not_capture_shorter_full_decode_shapes(monkeypatch):
|
||||
"""Without DSD, only the max decode query length should get FULL graphs.
|
||||
|
||||
Basic SD captures FULL decode graphs only for decode_query_len = K + 1.
|
||||
Uniform batches with smaller query lengths should therefore miss the FULL
|
||||
path entirely when using FULL_AND_PIECEWISE.
|
||||
"""
|
||||
|
||||
max_num_seqs = 512
|
||||
max_spec_tokens = 7
|
||||
max_decode_query_len = max_spec_tokens + 1
|
||||
|
||||
monkeypatch.setattr(
|
||||
gpu_cudagraph_utils,
|
||||
"get_pp_group",
|
||||
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
|
||||
)
|
||||
|
||||
vllm_config = _create_vllm_config_for_dsd(
|
||||
max_num_seqs=max_num_seqs,
|
||||
max_spec_tokens=max_spec_tokens,
|
||||
cudagraph_mode="FULL_AND_PIECEWISE",
|
||||
use_dynamic_sd=False,
|
||||
)
|
||||
manager = gpu_cudagraph_utils.CudaGraphManager(
|
||||
vllm_config=vllm_config,
|
||||
device=torch.device("cpu"),
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
decode_query_len=max_decode_query_len,
|
||||
)
|
||||
manager._graphs_captured = True
|
||||
|
||||
for num_reqs in range(1, max_num_seqs + 1):
|
||||
for max_query_len in range(1, max_decode_query_len):
|
||||
# These are still uniform decode batches, but basic SD should only
|
||||
# have FULL graphs for query_len == max_decode_query_len.
|
||||
num_tokens = num_reqs * max_query_len
|
||||
uniform_tok_count = gpu_cudagraph_utils.get_uniform_token_count(
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
max_query_len,
|
||||
)
|
||||
assert uniform_tok_count == max_query_len
|
||||
|
||||
desc = manager.dispatch(
|
||||
num_reqs=num_reqs,
|
||||
num_tokens=num_tokens,
|
||||
uniform_token_count=uniform_tok_count,
|
||||
num_active_loras=0,
|
||||
)
|
||||
|
||||
assert desc.cg_mode == CUDAGraphMode.PIECEWISE
|
||||
assert desc.uniform_token_count is None
|
||||
assert desc.num_tokens == num_tokens
|
||||
assert desc.num_reqs is None
|
||||
assert desc.num_active_loras == 0
|
||||
|
||||
|
||||
def test_dynamic_sd_only_captures_scheduled_query_lengths(monkeypatch):
|
||||
"""DSD should only capture FULL graphs for query lengths in the schedule.
|
||||
|
||||
With a partial schedule of ``(1, 32, 4)`` and ``(32, 128, 3)``, only the
|
||||
scheduled speculative-token counts (K = 4 and K = 3) become decode query
|
||||
lengths (K + 1 = 5 and 4). Uniform batches at those query lengths should get
|
||||
FULL graphs, while every other query length (e.g. the lower values 1, 2, 3)
|
||||
must fall back to the mixed-batch PIECEWISE graph.
|
||||
"""
|
||||
|
||||
max_num_seqs = 128
|
||||
max_spec_tokens = 7
|
||||
max_decode_query_len = max_spec_tokens + 1
|
||||
|
||||
# (range_start, range_end, num_speculative_tokens): K = 4 and K = 3 are
|
||||
# scheduled, so FULL decode graphs should exist for query lengths K + 1,
|
||||
# i.e. exactly {5, 4}.
|
||||
num_spec_per_batch_size = [(1, 32, 4), (32, 128, 3)]
|
||||
scheduled_query_lens = {entry[2] + 1 for entry in num_spec_per_batch_size}
|
||||
|
||||
monkeypatch.setattr(
|
||||
gpu_cudagraph_utils,
|
||||
"get_pp_group",
|
||||
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
|
||||
)
|
||||
|
||||
vllm_config = _create_vllm_config_for_dsd(
|
||||
max_num_seqs=max_num_seqs,
|
||||
max_spec_tokens=max_spec_tokens,
|
||||
cudagraph_mode="FULL_AND_PIECEWISE",
|
||||
use_dynamic_sd=True,
|
||||
num_spec_per_batch_size=num_spec_per_batch_size,
|
||||
)
|
||||
manager = gpu_cudagraph_utils.CudaGraphManager(
|
||||
vllm_config=vllm_config,
|
||||
device=torch.device("cpu"),
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
decode_query_len=max_decode_query_len,
|
||||
)
|
||||
manager._graphs_captured = True
|
||||
|
||||
for num_reqs in range(1, max_num_seqs + 1):
|
||||
for max_query_len in range(1, max_decode_query_len + 1):
|
||||
num_tokens = num_reqs * max_query_len
|
||||
uniform_tok_count = gpu_cudagraph_utils.get_uniform_token_count(
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
max_query_len,
|
||||
)
|
||||
assert uniform_tok_count == max_query_len
|
||||
|
||||
desc = manager.dispatch(
|
||||
num_reqs=num_reqs,
|
||||
num_tokens=num_tokens,
|
||||
uniform_token_count=uniform_tok_count,
|
||||
num_active_loras=0,
|
||||
)
|
||||
|
||||
if max_query_len in scheduled_query_lens:
|
||||
# Scheduled query lengths get a dedicated FULL decode graph.
|
||||
assert desc.cg_mode == CUDAGraphMode.FULL
|
||||
assert desc.uniform_token_count == max_query_len
|
||||
assert desc.num_tokens == num_tokens
|
||||
assert desc.num_reqs == num_reqs
|
||||
else:
|
||||
# Unscheduled query lengths (including the lower values 1 and 2)
|
||||
# have no FULL candidate and must fall back to PIECEWISE.
|
||||
assert desc.cg_mode == CUDAGraphMode.PIECEWISE
|
||||
assert desc.uniform_token_count is None
|
||||
assert desc.num_tokens == num_tokens
|
||||
assert desc.num_reqs is None
|
||||
assert desc.num_active_loras == 0
|
||||
@@ -1321,6 +1321,7 @@ class CompilationConfig:
|
||||
min_cg_support: "AttentionCGSupport",
|
||||
min_cg_attn_backend: str | None,
|
||||
uniform_decode_query_len: int = 1,
|
||||
use_v2_model_runner: bool = False,
|
||||
tensor_parallel_size: int = 1,
|
||||
kv_cache_config: "KVCacheConfig | None" = None,
|
||||
max_num_reqs: int | None = None,
|
||||
@@ -1421,13 +1422,16 @@ class CompilationConfig:
|
||||
"and make sure compilation mode is VLLM_COMPILE"
|
||||
)
|
||||
|
||||
# Adjust cudagraph sizes to be a multiple of uniform_decode_query_len
|
||||
# MRV1 adjusts cudagraph sizes to be a multiple of uniform_decode_query_len
|
||||
# to avoid: https://github.com/vllm-project/vllm/issues/28207 and temp-fix:
|
||||
# https://github.com/vllm-project/vllm/issues/28207#issuecomment-3504004536
|
||||
# Will be removed in the near future when we have separate cudagraph capture
|
||||
# sizes for decode and mixed prefill-decode.
|
||||
# MRV2 handles cudagraph capture sizing in cudagraph_utils.py
|
||||
# and doesn't need below: https://github.com/vllm-project/vllm/pull/45953
|
||||
if (
|
||||
cudagraph_mode.decode_mode() == CUDAGraphMode.FULL
|
||||
not use_v2_model_runner
|
||||
and cudagraph_mode.decode_mode() == CUDAGraphMode.FULL
|
||||
and uniform_decode_query_len > 1
|
||||
):
|
||||
self.adjust_cudagraph_sizes_for_spec_decode(
|
||||
|
||||
+3
-4
@@ -774,13 +774,15 @@ class VllmConfig:
|
||||
speculative_config is None
|
||||
or not speculative_config.uses_dynamic_speculative_decoding()
|
||||
or not self.compilation_config.cudagraph_mode.has_full_cudagraphs()
|
||||
or self.use_v2_model_runner
|
||||
):
|
||||
return
|
||||
|
||||
logger.warning_once(
|
||||
"Dynamic speculative decoding changes the target verification "
|
||||
"length at runtime. Overriding cudagraph_mode from %s to "
|
||||
"PIECEWISE for reliability.",
|
||||
"PIECEWISE for reliability. Use VLLM_USE_V2_MODEL_RUNNER=1 "
|
||||
"if you want to use full CUDA graphs.",
|
||||
self.compilation_config.cudagraph_mode.name,
|
||||
)
|
||||
self.compilation_config.cudagraph_mode = CUDAGraphMode.PIECEWISE
|
||||
@@ -2062,9 +2064,6 @@ class VllmConfig:
|
||||
):
|
||||
unsupported.append(f"speculative method '{speculative_config.method}'")
|
||||
|
||||
if speculative_config.uses_dynamic_speculative_decoding():
|
||||
unsupported.append("dynamic speculative decoding")
|
||||
|
||||
# V2 EagleSpeculator does not support parallel_drafting (for P-Eagle).
|
||||
# DFlash and DSpark use parallel drafting natively in V2 via their
|
||||
# own speculators.
|
||||
|
||||
@@ -27,6 +27,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.offloader.base import get_offloader
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
@@ -141,10 +142,7 @@ class CudaGraphManager:
|
||||
|
||||
self._candidates: dict[tuple[int, int], list[BatchExecutionDescriptor]] = {}
|
||||
self._capture_descs: dict[CUDAGraphMode, list[BatchExecutionDescriptor]] = {}
|
||||
# adjust the cudagraph sizes to be a multiple of the uniform decode query length
|
||||
self.compilation_config.adjust_cudagraph_sizes_for_spec_decode(
|
||||
self.decode_query_len, self.tp_size
|
||||
)
|
||||
|
||||
self._init_candidates()
|
||||
|
||||
# Breakable CUDA graph (PW CUDA graph without torch.compile)
|
||||
@@ -191,31 +189,72 @@ class CudaGraphManager:
|
||||
decode_mode = self.cudagraph_mode.decode_mode()
|
||||
mixed_mode = self.cudagraph_mode.mixed_mode()
|
||||
separate_decode_routine = self.cudagraph_mode.separate_routine()
|
||||
max_cg_capture_size = self.compilation_config.max_cudagraph_capture_size
|
||||
|
||||
descs_by_token_lora: dict[tuple[int, int], list[BatchExecutionDescriptor]] = (
|
||||
defaultdict(list)
|
||||
)
|
||||
descs_by_mode = defaultdict(list)
|
||||
descs_by_mode: defaultdict[CUDAGraphMode, list[BatchExecutionDescriptor]] = (
|
||||
defaultdict(list)
|
||||
)
|
||||
|
||||
# When using Dynamic SD, num_speculative_tokens is the max number of
|
||||
# draft tokens. The scheduler might use a smaller number so we need
|
||||
# to capture graphs for all possible values during decode.
|
||||
speculative_config = self.vllm_config.speculative_config
|
||||
if (
|
||||
speculative_config
|
||||
and speculative_config.uses_dynamic_speculative_decoding()
|
||||
):
|
||||
num_spec_per_batch_size = (
|
||||
speculative_config.num_speculative_tokens_per_batch_size
|
||||
)
|
||||
# uses_dynamic_speculative_decoding() guarantees this is set.
|
||||
assert num_spec_per_batch_size is not None
|
||||
# decode_query_len = num_speculative_steps + num_new_sampled_tokens
|
||||
# _per_step. Recover num_new_sampled_tokens_per_step
|
||||
# from the values the manager already has.
|
||||
num_new_sampled_tokens_per_step = (
|
||||
self.decode_query_len - self.vllm_config.num_speculative_tokens
|
||||
)
|
||||
# Each entry is (range_start, range_end, num_speculative_tokens).
|
||||
decode_query_lens = [
|
||||
x[2] + num_new_sampled_tokens_per_step for x in num_spec_per_batch_size
|
||||
]
|
||||
else:
|
||||
decode_query_lens = [self.decode_query_len]
|
||||
|
||||
for num_tokens, num_active_loras in product(
|
||||
capture_sizes, self.lora_capture_cases
|
||||
):
|
||||
# Capture uniform decode specfifc graphs if required
|
||||
# (i.e. separate decode routine)
|
||||
if (
|
||||
separate_decode_routine
|
||||
and decode_mode
|
||||
and self.decode_query_len <= num_tokens <= max_decode_tokens
|
||||
):
|
||||
desc = BatchExecutionDescriptor(
|
||||
cg_mode=decode_mode,
|
||||
num_tokens=num_tokens,
|
||||
num_reqs=num_tokens // self.decode_query_len,
|
||||
uniform_token_count=self.decode_query_len,
|
||||
num_active_loras=num_active_loras,
|
||||
)
|
||||
descs_by_mode[decode_mode].append(desc)
|
||||
descs_by_token_lora[(num_tokens, num_active_loras)].append(desc)
|
||||
if separate_decode_routine and decode_mode:
|
||||
for decode_query_len in decode_query_lens:
|
||||
rounded_num_tokens = round_up(num_tokens, decode_query_len)
|
||||
rounded_num_reqs = rounded_num_tokens // decode_query_len
|
||||
|
||||
if (
|
||||
rounded_num_tokens > max_decode_tokens
|
||||
or rounded_num_tokens > max_cg_capture_size
|
||||
or rounded_num_reqs > self.max_num_reqs
|
||||
):
|
||||
continue
|
||||
|
||||
desc = BatchExecutionDescriptor(
|
||||
cg_mode=decode_mode,
|
||||
num_tokens=rounded_num_tokens,
|
||||
num_reqs=rounded_num_reqs,
|
||||
uniform_token_count=decode_query_len,
|
||||
num_active_loras=num_active_loras,
|
||||
)
|
||||
|
||||
# avoid duplicate graphs
|
||||
if desc not in descs_by_mode[decode_mode]:
|
||||
descs_by_mode[decode_mode].append(desc)
|
||||
descs_by_token_lora[
|
||||
(rounded_num_tokens, num_active_loras)
|
||||
].append(desc)
|
||||
|
||||
if mixed_mode:
|
||||
# for PIECEWISE graphs there is no limit on requests when replaying
|
||||
|
||||
@@ -454,9 +454,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
attn_cg_support.min_cg_support,
|
||||
attn_cg_support.min_cg_attn_backend,
|
||||
self.decode_query_len,
|
||||
self.parallel_config.tensor_parallel_size,
|
||||
self.kv_cache_config,
|
||||
self.max_num_reqs,
|
||||
use_v2_model_runner=True,
|
||||
tensor_parallel_size=self.parallel_config.tensor_parallel_size,
|
||||
kv_cache_config=self.kv_cache_config,
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
)
|
||||
self.cudagraph_manager = ModelCudaGraphManager(
|
||||
self.vllm_config,
|
||||
|
||||
@@ -6930,9 +6930,10 @@ class GPUModelRunner(
|
||||
min_cg_support,
|
||||
min_cg_attn_backend,
|
||||
self.uniform_decode_query_len,
|
||||
self.parallel_config.tensor_parallel_size,
|
||||
self.kv_cache_config,
|
||||
self.max_num_reqs,
|
||||
use_v2_model_runner=False,
|
||||
tensor_parallel_size=self.parallel_config.tensor_parallel_size,
|
||||
kv_cache_config=self.kv_cache_config,
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
is_profiling=is_profiling,
|
||||
)
|
||||
# Trigger cudagraph dispatching keys initialization after
|
||||
|
||||
Reference in New Issue
Block a user