[Bugfix][MRV2][Spec Decode] Honor dynamic K in autoregressive speculators, 1.2~1.3x kernel perf improvement (#57053)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -20,8 +20,12 @@ from vllm.distributed.device_communicators import pynccl_allocator
|
|||||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||||
from vllm.v1.worker.gpu import cudagraph_utils as gpu_cudagraph_utils
|
from vllm.v1.worker.gpu import cudagraph_utils as gpu_cudagraph_utils
|
||||||
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
|
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
|
||||||
|
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
|
||||||
from vllm.v1.worker.gpu.input_batch import InputBuffers
|
from vllm.v1.worker.gpu.input_batch import InputBuffers
|
||||||
from vllm.v1.worker.gpu.pcp_manager import PCPManager
|
from vllm.v1.worker.gpu.pcp_manager import PCPManager
|
||||||
|
from vllm.v1.worker.gpu.spec_decode.autoregressive import (
|
||||||
|
cudagraph_utils as spec_cudagraph_utils,
|
||||||
|
)
|
||||||
from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import (
|
from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import (
|
||||||
SpeculatorCudaGraphManager,
|
SpeculatorCudaGraphManager,
|
||||||
)
|
)
|
||||||
@@ -226,8 +230,10 @@ def test_speculator_capture_preserves_decode_query_bounds(
|
|||||||
)
|
)
|
||||||
manager = SpeculatorCudaGraphManager.__new__(SpeculatorCudaGraphManager)
|
manager = SpeculatorCudaGraphManager.__new__(SpeculatorCudaGraphManager)
|
||||||
manager.max_num_reqs = num_reqs
|
manager.max_num_reqs = num_reqs
|
||||||
|
manager.decode_query_len = uniform_token_count or max_query_len or num_tokens
|
||||||
manager.vllm_config = SimpleNamespace(
|
manager.vllm_config = SimpleNamespace(
|
||||||
parallel_config=SimpleNamespace(data_parallel_size=1)
|
parallel_config=SimpleNamespace(data_parallel_size=1),
|
||||||
|
num_speculative_tokens=3,
|
||||||
)
|
)
|
||||||
buffers = InputBuffers(num_reqs, num_tokens, torch.device("cpu"))
|
buffers = InputBuffers(num_reqs, num_tokens, torch.device("cpu"))
|
||||||
block_tables = MagicMock()
|
block_tables = MagicMock()
|
||||||
@@ -488,6 +494,78 @@ def test_dynamic_spec_decode_shared_token_count_stays_reachable(monkeypatch):
|
|||||||
), desc
|
), desc
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("specialize", [False, True])
|
||||||
|
def test_speculator_replays_graph_for_runtime_width(monkeypatch, specialize):
|
||||||
|
"""Equal batch shapes must replay the graph captured for the selected K."""
|
||||||
|
monkeypatch.setattr(
|
||||||
|
gpu_cudagraph_utils,
|
||||||
|
"get_pp_group",
|
||||||
|
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
gpu_cudagraph_utils.current_platform, "get_global_graph_pool", lambda: object()
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(gpu_cudagraph_utils, "get_offloader", MagicMock)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
spec_cudagraph_utils, "prepare_inputs_to_capture", lambda *a, **kw: ({}, {})
|
||||||
|
)
|
||||||
|
forward = MagicMock()
|
||||||
|
replayed_widths: list[int] = []
|
||||||
|
|
||||||
|
def capture(manager, create_forward_fn, progress_bar_desc):
|
||||||
|
for desc in manager._capture_descs[CUDAGraphMode.FULL]:
|
||||||
|
create_forward_fn(desc, warmup=False)(CUDAGraphMode.NONE)
|
||||||
|
k = forward.call_args.kwargs["num_speculative_steps"]
|
||||||
|
graph = MagicMock()
|
||||||
|
graph.replay.side_effect = lambda k=k: replayed_widths.append(k)
|
||||||
|
manager.graphs[desc] = graph
|
||||||
|
manager._graphs_captured = True
|
||||||
|
|
||||||
|
monkeypatch.setattr(gpu_cudagraph_utils.CudaGraphManager, "capture", capture)
|
||||||
|
manager = spec_cudagraph_utils.SpeculatorCudaGraphManager(
|
||||||
|
_create_decode_vllm_config(
|
||||||
|
capture_sizes=[4],
|
||||||
|
num_speculative_tokens=4,
|
||||||
|
dynamic_spec_schedule=[(1, 2, 4), (3, 4, 2), (5, 6, 1), (7, 8, 0)],
|
||||||
|
),
|
||||||
|
device=torch.device("cpu"),
|
||||||
|
cudagraph_mode=CUDAGraphMode.FULL_DECODE_ONLY,
|
||||||
|
decode_query_len=1,
|
||||||
|
)
|
||||||
|
manager.capture(
|
||||||
|
forward,
|
||||||
|
MagicMock(),
|
||||||
|
MagicMock(),
|
||||||
|
MagicMock(),
|
||||||
|
[],
|
||||||
|
MagicMock(),
|
||||||
|
specialize_spec_tokens=specialize,
|
||||||
|
)
|
||||||
|
captured_widths = [4, 2] if specialize else [4]
|
||||||
|
assert [c.kwargs["num_speculative_steps"] for c in forward.call_args_list] == (
|
||||||
|
captured_widths
|
||||||
|
)
|
||||||
|
|
||||||
|
for k in (2, 4, 2, 3):
|
||||||
|
desc, _ = dispatch_cg_and_sync_dp(
|
||||||
|
manager,
|
||||||
|
num_reqs=3,
|
||||||
|
num_tokens=3,
|
||||||
|
uniform_token_count=1,
|
||||||
|
dp_size=1,
|
||||||
|
dp_rank=0,
|
||||||
|
)
|
||||||
|
desc = manager.specialize_spec_tokens(desc, k)
|
||||||
|
captured_k = k if specialize else 4
|
||||||
|
if captured_k in captured_widths:
|
||||||
|
assert desc.cg_mode == CUDAGraphMode.FULL
|
||||||
|
assert desc.num_tokens == 4
|
||||||
|
manager.run_fullgraph(desc)
|
||||||
|
assert replayed_widths[-1] == captured_k
|
||||||
|
else:
|
||||||
|
assert desc.cg_mode == CUDAGraphMode.NONE
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"decode_query_len,capture_sizes",
|
"decode_query_len,capture_sizes",
|
||||||
[(1, [1, 2, 4, 8]), (2, [2, 4, 8, 16]), (8, [8, 16, 32, 64])],
|
[(1, [1, 2, 4, 8]), (2, [2, 4, 8, 16]), (8, [8, 16, 32, 64])],
|
||||||
|
|||||||
@@ -374,19 +374,22 @@ def test_run_model_reuses_tensor_return_for_mtp(monkeypatch):
|
|||||||
(
|
(
|
||||||
"method_name",
|
"method_name",
|
||||||
"cg_mode",
|
"cg_mode",
|
||||||
|
"num_speculative_steps",
|
||||||
"expected_eager_calls",
|
"expected_eager_calls",
|
||||||
"expected_graph_replays",
|
"expected_graph_replays",
|
||||||
),
|
),
|
||||||
[
|
[
|
||||||
("_multi_step_decode", CUDAGraphMode.NONE, 3, 0),
|
("_multi_step_decode", CUDAGraphMode.NONE, 4, 3, 0),
|
||||||
("_multi_step_decode", CUDAGraphMode.FULL, 0, 3),
|
("_multi_step_decode", CUDAGraphMode.FULL, 4, 0, 3),
|
||||||
("_fused_multi_step_decode", CUDAGraphMode.NONE, 3, 0),
|
("_fused_multi_step_decode", CUDAGraphMode.NONE, 4, 3, 0),
|
||||||
("_fused_multi_step_decode", CUDAGraphMode.FULL, 0, 1),
|
("_fused_multi_step_decode", CUDAGraphMode.FULL, 4, 0, 1),
|
||||||
|
("_multi_step_decode", CUDAGraphMode.FULL, 3, 0, 2),
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
def test_multi_step_decode_replays_captured_graph_as_expected(
|
def test_multi_step_decode_replays_captured_graph_as_expected(
|
||||||
method_name,
|
method_name,
|
||||||
cg_mode,
|
cg_mode,
|
||||||
|
num_speculative_steps,
|
||||||
expected_eager_calls,
|
expected_eager_calls,
|
||||||
expected_graph_replays,
|
expected_graph_replays,
|
||||||
):
|
):
|
||||||
@@ -416,6 +419,7 @@ def test_multi_step_decode_replays_captured_graph_as_expected(
|
|||||||
batch_desc=batch_desc,
|
batch_desc=batch_desc,
|
||||||
seq_lens_cpu_upper_bound=None,
|
seq_lens_cpu_upper_bound=None,
|
||||||
num_tokens_across_dp=None,
|
num_tokens_across_dp=None,
|
||||||
|
num_speculative_steps=num_speculative_steps,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert generate_draft.call_count == expected_eager_calls
|
assert generate_draft.call_count == expected_eager_calls
|
||||||
|
|||||||
@@ -46,6 +46,7 @@ def test_non_last_pp_rank_uses_global_batch_for_sample_feedback():
|
|||||||
finished_req_ids=set(),
|
finished_req_ids=set(),
|
||||||
ec_connector_output=None,
|
ec_connector_output=None,
|
||||||
cudagraph_stats=None,
|
cudagraph_stats=None,
|
||||||
|
num_spec_tokens_to_schedule=0,
|
||||||
)
|
)
|
||||||
|
|
||||||
runner.sample_tokens(None)
|
runner.sample_tokens(None)
|
||||||
|
|||||||
@@ -190,6 +190,7 @@ def test_v2_sample_tokens_runs_eplb_on_non_last_pp_rank(monkeypatch):
|
|||||||
ec_connector_output=None,
|
ec_connector_output=None,
|
||||||
routed_experts=None,
|
routed_experts=None,
|
||||||
cudagraph_stats=None,
|
cudagraph_stats=None,
|
||||||
|
num_spec_tokens_to_schedule=0,
|
||||||
)
|
)
|
||||||
runner.req_states = SimpleNamespace()
|
runner.req_states = SimpleNamespace()
|
||||||
|
|
||||||
|
|||||||
@@ -116,9 +116,14 @@ class _StepRecorder:
|
|||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
# (blocks held per group, num_computed_tokens, num_scheduled_tokens)
|
# (blocks held per group, num_computed_tokens, num_scheduled_tokens)
|
||||||
self.steps: list[tuple[list[int], int, int]] = []
|
self.steps: list[tuple[list[int], int, int]] = []
|
||||||
|
self.num_spec_tokens_to_schedule: list[int] = []
|
||||||
self._held: dict[str, list[int]] = {}
|
self._held: dict[str, list[int]] = {}
|
||||||
|
|
||||||
def execute_model(self, scheduler_output) -> None:
|
def execute_model(self, scheduler_output) -> None:
|
||||||
|
if scheduler_output.total_num_scheduled_tokens:
|
||||||
|
self.num_spec_tokens_to_schedule.append(
|
||||||
|
scheduler_output.num_spec_tokens_to_schedule
|
||||||
|
)
|
||||||
for new_req in scheduler_output.scheduled_new_reqs:
|
for new_req in scheduler_output.scheduled_new_reqs:
|
||||||
self._held[new_req.req_id] = [len(ids) for ids in new_req.block_ids]
|
self._held[new_req.req_id] = [len(ids) for ids in new_req.block_ids]
|
||||||
self._record(new_req.req_id, new_req.num_computed_tokens, scheduler_output)
|
self._record(new_req.req_id, new_req.num_computed_tokens, scheduler_output)
|
||||||
@@ -161,7 +166,7 @@ def _assert_covers_lookahead(
|
|||||||
|
|
||||||
# 0 covers eagle / MTP / draft models, 1 covers DFlash's extra in-fill query.
|
# 0 covers eagle / MTP / draft models, 1 covers DFlash's extra in-fill query.
|
||||||
@pytest.mark.parametrize("extra_lookahead", [0, 1])
|
@pytest.mark.parametrize("extra_lookahead", [0, 1])
|
||||||
@pytest.mark.parametrize("num_spec_steps", [2, 3, 5, 7])
|
@pytest.mark.parametrize("num_spec_steps", [1, 2, 3, 5, 7])
|
||||||
def test_warmup_kernels_reserves_lookahead_blocks(num_spec_steps, extra_lookahead):
|
def test_warmup_kernels_reserves_lookahead_blocks(num_spec_steps, extra_lookahead):
|
||||||
num_lookahead_tokens = num_spec_steps + extra_lookahead
|
num_lookahead_tokens = num_spec_steps + extra_lookahead
|
||||||
recorder = _StepRecorder()
|
recorder = _StepRecorder()
|
||||||
@@ -173,6 +178,7 @@ def test_warmup_kernels_reserves_lookahead_blocks(num_spec_steps, extra_lookahea
|
|||||||
)
|
)
|
||||||
|
|
||||||
_assert_covers_lookahead(recorder.steps, num_lookahead_tokens)
|
_assert_covers_lookahead(recorder.steps, num_lookahead_tokens)
|
||||||
|
assert set(recorder.num_spec_tokens_to_schedule) == {num_spec_steps}
|
||||||
|
|
||||||
|
|
||||||
def test_mixed_warmup_reserves_lookahead_blocks():
|
def test_mixed_warmup_reserves_lookahead_blocks():
|
||||||
@@ -187,6 +193,7 @@ def test_mixed_warmup_reserves_lookahead_blocks():
|
|||||||
)
|
)
|
||||||
|
|
||||||
_assert_covers_lookahead(recorder.steps, num_lookahead_tokens)
|
_assert_covers_lookahead(recorder.steps, num_lookahead_tokens)
|
||||||
|
assert set(recorder.num_spec_tokens_to_schedule) == {NUM_SPEC_STEPS}
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("mamba_cache_mode", ["none", "align"])
|
@pytest.mark.parametrize("mamba_cache_mode", ["none", "align"])
|
||||||
|
|||||||
@@ -272,6 +272,7 @@ class CudaGraphManager:
|
|||||||
if (
|
if (
|
||||||
speculative_config
|
speculative_config
|
||||||
and speculative_config.uses_dynamic_speculative_decoding()
|
and speculative_config.uses_dynamic_speculative_decoding()
|
||||||
|
and self.decode_query_len > self.vllm_config.num_speculative_tokens
|
||||||
):
|
):
|
||||||
# decode_query_len = num_speculative_steps + num_new_sampled_tokens
|
# decode_query_len = num_speculative_steps + num_new_sampled_tokens
|
||||||
# _per_step. Recover num_new_sampled_tokens_per_step
|
# _per_step. Recover num_new_sampled_tokens_per_step
|
||||||
|
|||||||
@@ -2057,6 +2057,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
finished_req_ids=finished_req_ids,
|
finished_req_ids=finished_req_ids,
|
||||||
ec_connector_output=ec_connector_output,
|
ec_connector_output=ec_connector_output,
|
||||||
cudagraph_stats=cudagraph_stats,
|
cudagraph_stats=cudagraph_stats,
|
||||||
|
num_spec_tokens_to_schedule=scheduler_output.num_spec_tokens_to_schedule,
|
||||||
)
|
)
|
||||||
|
|
||||||
if not self.is_last_pp_rank:
|
if not self.is_last_pp_rank:
|
||||||
@@ -2086,6 +2087,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
finished_req_ids = self.execute_model_state.finished_req_ids
|
finished_req_ids = self.execute_model_state.finished_req_ids
|
||||||
ec_connector_output = self.execute_model_state.ec_connector_output
|
ec_connector_output = self.execute_model_state.ec_connector_output
|
||||||
cudagraph_stats = self.execute_model_state.cudagraph_stats
|
cudagraph_stats = self.execute_model_state.cudagraph_stats
|
||||||
|
num_spec_tokens = self.execute_model_state.num_spec_tokens_to_schedule
|
||||||
self.execute_model_state = None
|
self.execute_model_state = None
|
||||||
|
|
||||||
if not self.is_last_pp_rank:
|
if not self.is_last_pp_rank:
|
||||||
@@ -2237,9 +2239,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
self.req_states.next_prefill_tokens,
|
self.req_states.next_prefill_tokens,
|
||||||
self.sampler.sampling_states.temperature.gpu,
|
self.sampler.sampling_states.temperature.gpu,
|
||||||
self.sampler.sampling_states.seeds.gpu,
|
self.sampler.sampling_states.seeds.gpu,
|
||||||
|
num_speculative_tokens=num_spec_tokens,
|
||||||
dp_sync=dp_sync,
|
dp_sync=dp_sync,
|
||||||
mm_inputs=mm_inputs,
|
mm_inputs=mm_inputs,
|
||||||
)
|
)
|
||||||
|
if num_spec_tokens < self.num_speculative_steps:
|
||||||
|
draft_tokens[:, num_spec_tokens:] = -1
|
||||||
self.req_states.draft_tokens[input_batch.idx_mapping] = draft_tokens
|
self.req_states.draft_tokens[input_batch.idx_mapping] = draft_tokens
|
||||||
if self.adaptive_verification is not None:
|
if self.adaptive_verification is not None:
|
||||||
self.adaptive_verification.record_confidences(
|
self.adaptive_verification.record_confidences(
|
||||||
@@ -2251,7 +2256,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
# not have a speculator (i.e. self.speculator is None)
|
# not have a speculator (i.e. self.speculator is None)
|
||||||
self.draft_tokens_handler.set_draft_tokens(
|
self.draft_tokens_handler.set_draft_tokens(
|
||||||
input_batch,
|
input_batch,
|
||||||
self.req_states.draft_tokens[input_batch.idx_mapping],
|
self.req_states.draft_tokens[input_batch.idx_mapping, :num_spec_tokens],
|
||||||
)
|
)
|
||||||
if self.pp_handler is not None:
|
if self.pp_handler is not None:
|
||||||
self.pp_handler.broadcast_drafts(
|
self.pp_handler.broadcast_drafts(
|
||||||
@@ -2396,6 +2401,7 @@ class ExecuteModelState(NamedTuple):
|
|||||||
finished_req_ids: set[str]
|
finished_req_ids: set[str]
|
||||||
ec_connector_output: ECConnectorOutput | None
|
ec_connector_output: ECConnectorOutput | None
|
||||||
cudagraph_stats: CUDAGraphStat | None
|
cudagraph_stats: CUDAGraphStat | None
|
||||||
|
num_spec_tokens_to_schedule: int
|
||||||
|
|
||||||
|
|
||||||
class BatchReqState(NamedTuple):
|
class BatchReqState(NamedTuple):
|
||||||
|
|||||||
@@ -1,11 +1,13 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
|
from dataclasses import dataclass, replace
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
from vllm.config.compilation import CUDAGraphMode
|
from vllm.config.compilation import CUDAGraphMode
|
||||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||||
|
from vllm.v1.spec_decode.dynamic.utils import build_dynamic_sd_schedule_lookup
|
||||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||||
BatchExecutionDescriptor,
|
BatchExecutionDescriptor,
|
||||||
@@ -17,6 +19,11 @@ from vllm.v1.worker.gpu.model_states.interface import ModelState
|
|||||||
from vllm.v1.worker.utils import AttentionGroup
|
from vllm.v1.worker.utils import AttentionGroup
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class SpeculatorBatchDescriptor(BatchExecutionDescriptor):
|
||||||
|
num_speculative_tokens: int = 0
|
||||||
|
|
||||||
|
|
||||||
class SpeculatorCudaGraphManager(CudaGraphManager):
|
class SpeculatorCudaGraphManager(CudaGraphManager):
|
||||||
"""CudaGraphManager for draft prefill and decode.
|
"""CudaGraphManager for draft prefill and decode.
|
||||||
|
|
||||||
@@ -27,6 +34,18 @@ class SpeculatorCudaGraphManager(CudaGraphManager):
|
|||||||
earlier capture would execute kernels with stale buffer contents.
|
earlier capture would execute kernels with stale buffer contents.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
def specialize_spec_tokens(
|
||||||
|
self, desc: BatchExecutionDescriptor, num_speculative_tokens: int
|
||||||
|
) -> BatchExecutionDescriptor:
|
||||||
|
if desc.cg_mode != CUDAGraphMode.FULL or desc in self.graphs:
|
||||||
|
return desc
|
||||||
|
desc = SpeculatorBatchDescriptor(
|
||||||
|
**vars(desc), num_speculative_tokens=num_speculative_tokens
|
||||||
|
)
|
||||||
|
return (
|
||||||
|
desc if desc in self.graphs else replace(desc, cg_mode=CUDAGraphMode.NONE)
|
||||||
|
)
|
||||||
|
|
||||||
def capture(
|
def capture(
|
||||||
self,
|
self,
|
||||||
forward_fn: Callable,
|
forward_fn: Callable,
|
||||||
@@ -36,7 +55,25 @@ class SpeculatorCudaGraphManager(CudaGraphManager):
|
|||||||
attn_groups: list[list[AttentionGroup]],
|
attn_groups: list[list[AttentionGroup]],
|
||||||
kv_cache_config: KVCacheConfig,
|
kv_cache_config: KVCacheConfig,
|
||||||
progress_bar_desc: str = "Capturing CUDA graphs",
|
progress_bar_desc: str = "Capturing CUDA graphs",
|
||||||
|
specialize_spec_tokens: bool = False,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
if specialize_spec_tokens:
|
||||||
|
speculative_config = self.vllm_config.speculative_config
|
||||||
|
assert speculative_config is not None
|
||||||
|
dense_schedule = build_dynamic_sd_schedule_lookup(
|
||||||
|
speculative_config.num_speculative_tokens_per_batch_size,
|
||||||
|
vllm_max_batch_size=self.max_num_reqs,
|
||||||
|
vllm_num_speculative_tokens=self.vllm_config.num_speculative_tokens,
|
||||||
|
)
|
||||||
|
self._capture_descs = {
|
||||||
|
mode: [
|
||||||
|
SpeculatorBatchDescriptor(**vars(desc), num_speculative_tokens=k)
|
||||||
|
for desc in descs
|
||||||
|
for k in sorted(set(dense_schedule) - {0, 1}, reverse=True)
|
||||||
|
]
|
||||||
|
for mode, descs in self._capture_descs.items()
|
||||||
|
}
|
||||||
|
|
||||||
def create_forward_fn(
|
def create_forward_fn(
|
||||||
desc: BatchExecutionDescriptor,
|
desc: BatchExecutionDescriptor,
|
||||||
warmup: bool,
|
warmup: bool,
|
||||||
@@ -60,6 +97,13 @@ class SpeculatorCudaGraphManager(CudaGraphManager):
|
|||||||
max_query_len=desc.max_query_len or desc.uniform_token_count,
|
max_query_len=desc.max_query_len or desc.uniform_token_count,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
forward_kwargs = {}
|
||||||
|
if self.decode_query_len == 1:
|
||||||
|
forward_kwargs["num_speculative_steps"] = (
|
||||||
|
desc.num_speculative_tokens
|
||||||
|
if isinstance(desc, SpeculatorBatchDescriptor)
|
||||||
|
else self.vllm_config.num_speculative_tokens
|
||||||
|
)
|
||||||
return lambda cg_mode: forward_fn(
|
return lambda cg_mode: forward_fn(
|
||||||
num_reqs,
|
num_reqs,
|
||||||
num_tokens,
|
num_tokens,
|
||||||
@@ -67,6 +111,7 @@ class SpeculatorCudaGraphManager(CudaGraphManager):
|
|||||||
slot_mappings,
|
slot_mappings,
|
||||||
num_tokens_across_dp,
|
num_tokens_across_dp,
|
||||||
cg_mode,
|
cg_mode,
|
||||||
|
**forward_kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
super().capture(create_forward_fn, progress_bar_desc)
|
super().capture(create_forward_fn, progress_bar_desc)
|
||||||
|
|||||||
@@ -190,6 +190,8 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
|||||||
self.attn_groups,
|
self.attn_groups,
|
||||||
self.kv_cache_config,
|
self.kv_cache_config,
|
||||||
progress_bar_desc="Capturing decode CUDA graphs",
|
progress_bar_desc="Capturing decode CUDA graphs",
|
||||||
|
specialize_spec_tokens=self.use_fused_multi_step_decode
|
||||||
|
and self.speculative_config.uses_dynamic_speculative_decoding(),
|
||||||
)
|
)
|
||||||
self.on_multi_step_decode_end(self.max_num_reqs)
|
self.on_multi_step_decode_end(self.max_num_reqs)
|
||||||
|
|
||||||
@@ -220,14 +222,17 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
|||||||
skip_attn_for_dummy_run: bool = False,
|
skip_attn_for_dummy_run: bool = False,
|
||||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
||||||
is_profile: bool = False,
|
is_profile: bool = False,
|
||||||
|
num_speculative_tokens: int | None = None,
|
||||||
) -> torch.Tensor:
|
) -> torch.Tensor:
|
||||||
|
num_reqs = input_batch.num_reqs
|
||||||
|
if num_speculative_tokens is None:
|
||||||
|
num_speculative_tokens = self.num_speculative_steps
|
||||||
num_tokens = input_batch.num_tokens
|
num_tokens = input_batch.num_tokens
|
||||||
num_tokens_padded = input_batch.num_tokens_after_padding
|
num_tokens_padded = input_batch.num_tokens_after_padding
|
||||||
num_reqs = input_batch.num_reqs
|
|
||||||
max_query_len = input_batch.num_scheduled_tokens.max()
|
max_query_len = input_batch.num_scheduled_tokens.max()
|
||||||
max_seq_len = input_batch.seq_lens_cpu_upper_bound[:num_reqs].max().item()
|
max_seq_len = input_batch.seq_lens_cpu_upper_bound[:num_reqs].max().item()
|
||||||
self.draft_max_seq_len = min(
|
self.draft_max_seq_len = min(
|
||||||
max_seq_len + self.num_speculative_steps, self.max_model_len
|
max_seq_len + num_speculative_tokens, self.max_model_len
|
||||||
)
|
)
|
||||||
|
|
||||||
# NOTE(woosuk): To avoid CPU-GPU synchronization without CPU knowing the
|
# NOTE(woosuk): To avoid CPU-GPU synchronization without CPU knowing the
|
||||||
@@ -322,9 +327,10 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
|||||||
)
|
)
|
||||||
self.on_prefill_end(num_reqs)
|
self.on_prefill_end(num_reqs)
|
||||||
|
|
||||||
if self.num_speculative_steps == 1:
|
if num_speculative_tokens <= 1:
|
||||||
# Early exit.
|
if num_speculative_tokens == 0:
|
||||||
return self.draft_tokens[:num_reqs, :1]
|
self.draft_tokens[:num_reqs].fill_(-1)
|
||||||
|
return self.draft_tokens[:num_reqs]
|
||||||
|
|
||||||
if self.pcp_manager is not None and not dummy_run:
|
if self.pcp_manager is not None and not dummy_run:
|
||||||
self.block_tables.gather_block_tables(
|
self.block_tables.gather_block_tables(
|
||||||
@@ -360,6 +366,10 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
|||||||
need_eager=is_profile,
|
need_eager=is_profile,
|
||||||
dp_sync=decode_batch_sync,
|
dp_sync=decode_batch_sync,
|
||||||
)
|
)
|
||||||
|
if self.decode_cudagraph_manager is not None:
|
||||||
|
decode_batch_desc = self.decode_cudagraph_manager.specialize_spec_tokens(
|
||||||
|
decode_batch_desc, num_speculative_tokens
|
||||||
|
)
|
||||||
num_tokens_across_dp = (
|
num_tokens_across_dp = (
|
||||||
decode_batch_sync.num_tokens_across_dp
|
decode_batch_sync.num_tokens_across_dp
|
||||||
if decode_batch_sync is not None
|
if decode_batch_sync is not None
|
||||||
@@ -379,6 +389,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
|||||||
decode_batch_desc,
|
decode_batch_desc,
|
||||||
num_tokens_across_dp,
|
num_tokens_across_dp,
|
||||||
input_batch.seq_lens_cpu_upper_bound,
|
input_batch.seq_lens_cpu_upper_bound,
|
||||||
|
num_speculative_tokens,
|
||||||
)
|
)
|
||||||
self.on_multi_step_decode_end(num_reqs)
|
self.on_multi_step_decode_end(num_reqs)
|
||||||
|
|
||||||
@@ -505,6 +516,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
|||||||
batch_desc: BatchExecutionDescriptor,
|
batch_desc: BatchExecutionDescriptor,
|
||||||
num_tokens_across_dp: torch.Tensor | None,
|
num_tokens_across_dp: torch.Tensor | None,
|
||||||
seq_lens_cpu_upper_bound: torch.Tensor,
|
seq_lens_cpu_upper_bound: torch.Tensor,
|
||||||
|
num_speculative_steps: int,
|
||||||
) -> None:
|
) -> None:
|
||||||
positions = self.input_buffers.positions[:num_reqs]
|
positions = self.input_buffers.positions[:num_reqs]
|
||||||
query_start_loc = self.input_buffers.query_start_loc[: num_reqs + 1]
|
query_start_loc = self.input_buffers.query_start_loc[: num_reqs + 1]
|
||||||
@@ -512,7 +524,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
|||||||
|
|
||||||
attn_metadata = None
|
attn_metadata = None
|
||||||
slot_mappings_by_layer = None
|
slot_mappings_by_layer = None
|
||||||
for step in range(1, self.num_speculative_steps):
|
for step in range(1, num_speculative_steps):
|
||||||
# Rebuild every step when positions advance, or just once
|
# Rebuild every step when positions advance, or just once
|
||||||
# on the first step when positions are constant (Gemma4 MTP).
|
# on the first step when positions are constant (Gemma4 MTP).
|
||||||
if not skip_attn and (self.advance_draft_positions or step == 1):
|
if not skip_attn and (self.advance_draft_positions or step == 1):
|
||||||
@@ -546,6 +558,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
|||||||
slot_mappings_by_layer,
|
slot_mappings_by_layer,
|
||||||
num_tokens_across_dp=num_tokens_across_dp,
|
num_tokens_across_dp=num_tokens_across_dp,
|
||||||
cudagraph_runtime_mode=batch_desc.cg_mode,
|
cudagraph_runtime_mode=batch_desc.cg_mode,
|
||||||
|
num_speculative_steps=num_speculative_steps,
|
||||||
)
|
)
|
||||||
|
|
||||||
def _fused_multi_step_decode(
|
def _fused_multi_step_decode(
|
||||||
@@ -555,6 +568,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
|||||||
batch_desc: BatchExecutionDescriptor,
|
batch_desc: BatchExecutionDescriptor,
|
||||||
num_tokens_across_dp: torch.Tensor | None,
|
num_tokens_across_dp: torch.Tensor | None,
|
||||||
seq_lens_cpu_upper_bound: torch.Tensor,
|
seq_lens_cpu_upper_bound: torch.Tensor,
|
||||||
|
num_speculative_steps: int,
|
||||||
) -> None:
|
) -> None:
|
||||||
positions = self.input_buffers.positions[:num_reqs]
|
positions = self.input_buffers.positions[:num_reqs]
|
||||||
query_start_loc = self.input_buffers.query_start_loc[: num_reqs + 1]
|
query_start_loc = self.input_buffers.query_start_loc[: num_reqs + 1]
|
||||||
@@ -593,6 +607,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
|||||||
slot_mappings_by_layer,
|
slot_mappings_by_layer,
|
||||||
num_tokens_across_dp,
|
num_tokens_across_dp,
|
||||||
batch_desc.cg_mode,
|
batch_desc.cg_mode,
|
||||||
|
num_speculative_steps,
|
||||||
)
|
)
|
||||||
|
|
||||||
def _generate_fused_drafts(
|
def _generate_fused_drafts(
|
||||||
@@ -602,7 +617,8 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
|||||||
attn_metadata: dict[str, Any] | None,
|
attn_metadata: dict[str, Any] | None,
|
||||||
slot_mappings: dict[str, torch.Tensor] | None,
|
slot_mappings: dict[str, torch.Tensor] | None,
|
||||||
num_tokens_across_dp: torch.Tensor | None,
|
num_tokens_across_dp: torch.Tensor | None,
|
||||||
cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE,
|
cudagraph_runtime_mode: CUDAGraphMode,
|
||||||
|
num_speculative_steps: int,
|
||||||
) -> None:
|
) -> None:
|
||||||
idx_mapping = self.idx_mapping[:num_reqs]
|
idx_mapping = self.idx_mapping[:num_reqs]
|
||||||
positions = self.input_buffers.positions[:num_reqs]
|
positions = self.input_buffers.positions[:num_reqs]
|
||||||
@@ -613,7 +629,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
|||||||
else []
|
else []
|
||||||
)
|
)
|
||||||
|
|
||||||
for step in range(1, self.num_speculative_steps):
|
for step in range(1, num_speculative_steps):
|
||||||
self.current_draft_step.fill_(step)
|
self.current_draft_step.fill_(step)
|
||||||
self._generate_draft(
|
self._generate_draft(
|
||||||
num_reqs,
|
num_reqs,
|
||||||
@@ -622,9 +638,10 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
|||||||
slot_mappings,
|
slot_mappings,
|
||||||
num_tokens_across_dp,
|
num_tokens_across_dp,
|
||||||
cudagraph_runtime_mode,
|
cudagraph_runtime_mode,
|
||||||
|
num_speculative_steps,
|
||||||
)
|
)
|
||||||
if (
|
if (
|
||||||
step < self.num_speculative_steps - 1
|
step < num_speculative_steps - 1
|
||||||
and attn_metadata is not None
|
and attn_metadata is not None
|
||||||
and self.advance_draft_positions
|
and self.advance_draft_positions
|
||||||
):
|
):
|
||||||
@@ -644,7 +661,8 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
|||||||
attn_metadata: dict[str, Any] | None,
|
attn_metadata: dict[str, Any] | None,
|
||||||
slot_mappings: dict[str, torch.Tensor] | None,
|
slot_mappings: dict[str, torch.Tensor] | None,
|
||||||
num_tokens_across_dp: torch.Tensor | None,
|
num_tokens_across_dp: torch.Tensor | None,
|
||||||
cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE,
|
cudagraph_runtime_mode: CUDAGraphMode,
|
||||||
|
num_speculative_steps: int,
|
||||||
) -> None:
|
) -> None:
|
||||||
self._prepare_eplb_forward(num_reqs)
|
self._prepare_eplb_forward(num_reqs)
|
||||||
|
|
||||||
@@ -682,7 +700,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
|||||||
self.sample_src_positions,
|
self.sample_src_positions,
|
||||||
num_reqs,
|
num_reqs,
|
||||||
self.max_model_len,
|
self.max_model_len,
|
||||||
self.num_speculative_steps,
|
num_speculative_steps,
|
||||||
advance_draft_positions=self.advance_draft_positions,
|
advance_draft_positions=self.advance_draft_positions,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -360,6 +360,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
|||||||
skip_attn_for_dummy_run: bool = False,
|
skip_attn_for_dummy_run: bool = False,
|
||||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
||||||
is_profile: bool = False,
|
is_profile: bool = False,
|
||||||
|
num_speculative_tokens: int | None = None,
|
||||||
) -> torch.Tensor:
|
) -> torch.Tensor:
|
||||||
num_reqs = input_batch.num_reqs
|
num_reqs = input_batch.num_reqs
|
||||||
num_target_tokens = input_batch.num_tokens
|
num_target_tokens = input_batch.num_tokens
|
||||||
|
|||||||
@@ -355,6 +355,7 @@ class PlainDraftModelSpeculator(DraftModelSpeculator):
|
|||||||
skip_attn_for_dummy_run: bool = False,
|
skip_attn_for_dummy_run: bool = False,
|
||||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
||||||
is_profile: bool = False,
|
is_profile: bool = False,
|
||||||
|
num_speculative_tokens: int | None = None,
|
||||||
) -> torch.Tensor:
|
) -> torch.Tensor:
|
||||||
assert self.model is not None
|
assert self.model is not None
|
||||||
|
|
||||||
|
|||||||
@@ -103,6 +103,7 @@ class ExtractHiddenStatesSpeculator(DraftModelSpeculator):
|
|||||||
skip_attn_for_dummy_run: bool = False,
|
skip_attn_for_dummy_run: bool = False,
|
||||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
||||||
is_profile: bool = False,
|
is_profile: bool = False,
|
||||||
|
num_speculative_tokens: int | None = None,
|
||||||
) -> torch.Tensor:
|
) -> torch.Tensor:
|
||||||
del (
|
del (
|
||||||
last_hidden_states,
|
last_hidden_states,
|
||||||
|
|||||||
@@ -155,6 +155,7 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator):
|
|||||||
skip_attn_for_dummy_run: bool = False,
|
skip_attn_for_dummy_run: bool = False,
|
||||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
||||||
is_profile: bool = False,
|
is_profile: bool = False,
|
||||||
|
num_speculative_tokens: int | None = None,
|
||||||
) -> torch.Tensor:
|
) -> torch.Tensor:
|
||||||
num_reqs = input_batch.num_reqs
|
num_reqs = input_batch.num_reqs
|
||||||
seq_lens_cpu_upper_bound = input_batch.seq_lens_cpu_upper_bound
|
seq_lens_cpu_upper_bound = input_batch.seq_lens_cpu_upper_bound
|
||||||
|
|||||||
@@ -97,6 +97,7 @@ class BaseSpeculator(ABC):
|
|||||||
skip_attn_for_dummy_run: bool = False,
|
skip_attn_for_dummy_run: bool = False,
|
||||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
||||||
is_profile: bool = False,
|
is_profile: bool = False,
|
||||||
|
num_speculative_tokens: int | None = None,
|
||||||
) -> torch.Tensor:
|
) -> torch.Tensor:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|||||||
@@ -146,6 +146,9 @@ def run_mixed_prefill_decode_warmup(
|
|||||||
)
|
)
|
||||||
|
|
||||||
decode_prefill_output = SchedulerOutput.make_empty()
|
decode_prefill_output = SchedulerOutput.make_empty()
|
||||||
|
decode_prefill_output.num_spec_tokens_to_schedule = (
|
||||||
|
model_runner.num_speculative_steps
|
||||||
|
)
|
||||||
decode_prefill_output.scheduled_new_reqs = [
|
decode_prefill_output.scheduled_new_reqs = [
|
||||||
NewRequestData(
|
NewRequestData(
|
||||||
req_id=decode_req_id,
|
req_id=decode_req_id,
|
||||||
@@ -175,6 +178,7 @@ def run_mixed_prefill_decode_warmup(
|
|||||||
]
|
]
|
||||||
|
|
||||||
mixed_output = SchedulerOutput.make_empty()
|
mixed_output = SchedulerOutput.make_empty()
|
||||||
|
mixed_output.num_spec_tokens_to_schedule = model_runner.num_speculative_steps
|
||||||
mixed_output.scheduled_cached_reqs = cached_decode_req
|
mixed_output.scheduled_cached_reqs = cached_decode_req
|
||||||
mixed_output.scheduled_new_reqs = [
|
mixed_output.scheduled_new_reqs = [
|
||||||
NewRequestData(
|
NewRequestData(
|
||||||
@@ -353,6 +357,7 @@ def _warmup_kernels(
|
|||||||
]
|
]
|
||||||
|
|
||||||
prefill_output = SchedulerOutput.make_empty()
|
prefill_output = SchedulerOutput.make_empty()
|
||||||
|
prefill_output.num_spec_tokens_to_schedule = num_spec_steps
|
||||||
prefill_output.scheduled_new_reqs = new_reqs
|
prefill_output.scheduled_new_reqs = new_reqs
|
||||||
prefill_output.num_scheduled_tokens = {rid: prompt_len for rid in req_ids}
|
prefill_output.num_scheduled_tokens = {rid: prompt_len for rid in req_ids}
|
||||||
prefill_output.total_num_scheduled_tokens = prompt_len * num_reqs
|
prefill_output.total_num_scheduled_tokens = prompt_len * num_reqs
|
||||||
@@ -412,6 +417,7 @@ def _warmup_kernels(
|
|||||||
step_spec_tokens[req_ids[i]] = [0] * num_spec_steps
|
step_spec_tokens[req_ids[i]] = [0] * num_spec_steps
|
||||||
|
|
||||||
decode_output = SchedulerOutput.make_empty()
|
decode_output = SchedulerOutput.make_empty()
|
||||||
|
decode_output.num_spec_tokens_to_schedule = num_spec_steps
|
||||||
decode_output.scheduled_cached_reqs = cached_req_data
|
decode_output.scheduled_cached_reqs = cached_req_data
|
||||||
decode_output.num_scheduled_tokens = step_num_scheduled_tokens
|
decode_output.num_scheduled_tokens = step_num_scheduled_tokens
|
||||||
decode_output.scheduled_spec_decode_tokens = step_spec_tokens
|
decode_output.scheduled_spec_decode_tokens = step_spec_tokens
|
||||||
|
|||||||
Reference in New Issue
Block a user