[Spec Decode] Rename AR speculators to StandaloneAR / TargetDependentAR (#60335)
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
@@ -87,7 +87,7 @@ steps:
|
||||
- |
|
||||
bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 30m "
|
||||
pytest -x -v -s \
|
||||
tests/v1/worker/test_gpu_autoregressive_speculator.py \
|
||||
tests/v1/worker/test_gpu_target_dependent_ar_speculator.py \
|
||||
tests/v1/worker/test_gpu_extract_hidden_states_speculator.py \
|
||||
tests/v1/worker/test_gpu_batch_ordering.py \
|
||||
tests/v1/worker/test_gpu_rejection_sampler_chunking.py \
|
||||
|
||||
@@ -23,10 +23,10 @@ from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
|
||||
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
|
||||
from vllm.v1.worker.gpu.input_batch import InputBuffers
|
||||
from vllm.v1.worker.gpu.pcp_manager import PCPManager
|
||||
from vllm.v1.worker.gpu.spec_decode.autoregressive import (
|
||||
from vllm.v1.worker.gpu.spec_decode.target_dependent_ar import (
|
||||
cudagraph_utils as spec_cudagraph_utils,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import (
|
||||
from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.cudagraph_utils import (
|
||||
SpeculatorCudaGraphManager,
|
||||
)
|
||||
|
||||
|
||||
+5
-5
@@ -22,17 +22,17 @@ from vllm.v1.attention.backends.flash_attn import FlashAttentionMetadata
|
||||
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
|
||||
from vllm.v1.worker.gpu.spec_decode import speculator as base_spec_module
|
||||
from vllm.v1.worker.gpu.spec_decode.autoregressive import speculator as spec_module
|
||||
from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
|
||||
AutoRegressiveSpeculator,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.multi_module_mtp.speculator import (
|
||||
MultiModuleMTPSpeculator,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
|
||||
from vllm.v1.worker.gpu.spec_decode.target_dependent_ar import speculator as spec_module
|
||||
from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.speculator import (
|
||||
TargetDependentARSpeculator,
|
||||
)
|
||||
|
||||
|
||||
class _TestSpeculator(AutoRegressiveSpeculator):
|
||||
class _TestSpeculator(TargetDependentARSpeculator):
|
||||
def load_draft_model(self, target_model, target_attn_layer_names):
|
||||
return self.test_draft_model
|
||||
|
||||
@@ -820,7 +820,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
"skip_attn must only be True for initial memory profiling."
|
||||
)
|
||||
|
||||
# Create a dummy scheduler output. Plain draft-model speculation adds
|
||||
# Create a dummy scheduler output. Standalone AR speculation adds
|
||||
# one correction slot per request during prefill. The scheduler
|
||||
# accounts for these slots, while dummy runs bypass the scheduler.
|
||||
# Adjust the dummy token count only when the expanded draft batch
|
||||
|
||||
@@ -71,11 +71,11 @@ def init_speculator(
|
||||
|
||||
return EagleSpeculator(vllm_config, device)
|
||||
elif speculative_config.uses_draft_model():
|
||||
from vllm.v1.worker.gpu.spec_decode.draft_model.speculator import (
|
||||
PlainDraftModelSpeculator,
|
||||
from vllm.v1.worker.gpu.spec_decode.standalone_ar.speculator import (
|
||||
StandaloneARSpeculator,
|
||||
)
|
||||
|
||||
return PlainDraftModelSpeculator(vllm_config, device)
|
||||
return StandaloneARSpeculator(vllm_config, device)
|
||||
elif speculative_config.use_ngram():
|
||||
from vllm.v1.worker.gpu.spec_decode.ngram.speculator import (
|
||||
NgramGPUSpeculator,
|
||||
|
||||
@@ -3,13 +3,13 @@
|
||||
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
|
||||
AutoRegressiveSpeculator,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model
|
||||
from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.speculator import (
|
||||
TargetDependentARSpeculator,
|
||||
)
|
||||
|
||||
|
||||
class EagleSpeculator(AutoRegressiveSpeculator):
|
||||
class EagleSpeculator(TargetDependentARSpeculator):
|
||||
def load_draft_model(
|
||||
self,
|
||||
target_model: nn.Module,
|
||||
|
||||
@@ -17,8 +17,8 @@ from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
from vllm.model_executor.model_loader.utils import get_draft_load_config
|
||||
from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
|
||||
AutoRegressiveSpeculator,
|
||||
from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.speculator import (
|
||||
TargetDependentARSpeculator,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -45,7 +45,7 @@ def _copy_target_kv_scales(attn: nn.Module, target_attn: nn.Module) -> None:
|
||||
getattr(attn, scale_name).copy_(getattr(target_attn, scale_name))
|
||||
|
||||
|
||||
class Gemma4Speculator(AutoRegressiveSpeculator):
|
||||
class Gemma4Speculator(TargetDependentARSpeculator):
|
||||
@property
|
||||
def advance_draft_positions(self) -> bool:
|
||||
# Gemma4 MTP is Q-only and reads K/V from the target's existing cache.
|
||||
|
||||
@@ -3,13 +3,13 @@
|
||||
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
|
||||
AutoRegressiveSpeculator,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model
|
||||
from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.speculator import (
|
||||
TargetDependentARSpeculator,
|
||||
)
|
||||
|
||||
|
||||
class MTPSpeculator(AutoRegressiveSpeculator):
|
||||
class MTPSpeculator(TargetDependentARSpeculator):
|
||||
share_mtp_topk_indices: bool = False
|
||||
|
||||
def load_draft_model(
|
||||
|
||||
@@ -14,11 +14,11 @@ from vllm.v1.attention.backends.utils import PAD_SLOT_ID
|
||||
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
|
||||
from vllm.v1.worker.gpu.dp_utils import DPSyncState, dispatch_cg_and_sync_dp
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import (
|
||||
SpeculatorCudaGraphManager,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model
|
||||
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
|
||||
from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.cudagraph_utils import (
|
||||
SpeculatorCudaGraphManager,
|
||||
)
|
||||
from vllm.v1.worker.utils import get_uniform_decode_token_count
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
+7
-7
@@ -26,22 +26,22 @@ from vllm.v1.worker.utils import AttentionGroup
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class PlainDraftModelSpeculator(DraftModelSpeculator):
|
||||
"""Speculative decoding using a separate smaller draft LM.
|
||||
class StandaloneARSpeculator(DraftModelSpeculator):
|
||||
"""Standalone autoregressive (AR) drafting with a separate smaller LM.
|
||||
|
||||
Unlike Eagle, the draft model runs fully independently of the target model.
|
||||
Step 0 builds an expanded buffer (accepted + correction token + rejected
|
||||
slots) via a Triton kernel; steps 1..k-1 are single-token decode steps.
|
||||
"""
|
||||
|
||||
# Plain draft model needs one extra slot per request for correction.
|
||||
# Standalone AR drafting needs one extra slot per request for correction.
|
||||
num_extra_query_per_req = 1
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, device: torch.device):
|
||||
super().__init__(vllm_config, device)
|
||||
|
||||
# draft_max_seq_len is read by the parent's attention metadata builder.
|
||||
# Plain draft model doesn't do per-batch adjustment; cap at max.
|
||||
# Standalone AR drafting doesn't do per-batch adjustment; cap at max.
|
||||
self.draft_max_seq_len = self.max_model_len
|
||||
|
||||
self.last_token_indices = torch.zeros(
|
||||
@@ -65,10 +65,10 @@ class PlainDraftModelSpeculator(DraftModelSpeculator):
|
||||
self.supports_mm_inputs = False
|
||||
|
||||
def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None:
|
||||
pass # CUDA graph not yet supported for plain draft model speculator.
|
||||
pass # CUDA graph not yet supported for standalone AR speculator.
|
||||
|
||||
def capture(self) -> None:
|
||||
pass # CUDA graph not yet supported for plain draft model speculator.
|
||||
pass # CUDA graph not yet supported for standalone AR speculator.
|
||||
|
||||
def set_attn(
|
||||
self,
|
||||
@@ -415,7 +415,7 @@ def _prepare_prefill_inputs_kernel(
|
||||
max_model_len,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
"""Per-request step-0 input preparation for the plain draft-model speculator.
|
||||
"""Per-request step-0 input preparation for the standalone AR speculator.
|
||||
|
||||
Output layout for request i (out_start = query_start_loc[i] + i):
|
||||
[out_start, out_start + num_valid) accepted tokens
|
||||
+9
-3
@@ -17,16 +17,22 @@ from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
|
||||
from vllm.v1.worker.gpu.dp_utils import DPSyncState, dispatch_cg_and_sync_dp
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import (
|
||||
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
|
||||
from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.cudagraph_utils import (
|
||||
SpeculatorCudaGraphManager,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
|
||||
from vllm.v1.worker.utils import AttentionGroup, get_uniform_decode_token_count
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
class TargetDependentARSpeculator(DraftModelSpeculator):
|
||||
"""Autoregressive (AR) drafting that depends on target model state.
|
||||
|
||||
Base for Eagle, MTP, and Gemma4 drafters, which consume the target's
|
||||
hidden states (and, for Gemma4, its KV cache) at each draft step.
|
||||
"""
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, device: torch.device):
|
||||
super().__init__(vllm_config, device)
|
||||
|
||||
Reference in New Issue
Block a user