[Spec Decode] Rename AR speculators to StandaloneAR / TargetDependentAR (#60335)

Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
Giancarlo Delfin
2026-10-07 14:01:59 -04:00
committed by GitHub
parent fba462e008
commit 6ee9be6b3c
14 changed files with 42 additions and 36 deletions
+1 -1
View File
@@ -87,7 +87,7 @@ steps:
- |
bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 30m "
pytest -x -v -s \
tests/v1/worker/test_gpu_autoregressive_speculator.py \
tests/v1/worker/test_gpu_target_dependent_ar_speculator.py \
tests/v1/worker/test_gpu_extract_hidden_states_speculator.py \
tests/v1/worker/test_gpu_batch_ordering.py \
tests/v1/worker/test_gpu_rejection_sampler_chunking.py \
+2 -2
View File
@@ -23,10 +23,10 @@ from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
from vllm.v1.worker.gpu.input_batch import InputBuffers
from vllm.v1.worker.gpu.pcp_manager import PCPManager
from vllm.v1.worker.gpu.spec_decode.autoregressive import (
from vllm.v1.worker.gpu.spec_decode.target_dependent_ar import (
cudagraph_utils as spec_cudagraph_utils,
)
from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import (
from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.cudagraph_utils import (
SpeculatorCudaGraphManager,
)
@@ -22,17 +22,17 @@ from vllm.v1.attention.backends.flash_attn import FlashAttentionMetadata
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
from vllm.v1.worker.gpu.spec_decode import speculator as base_spec_module
from vllm.v1.worker.gpu.spec_decode.autoregressive import speculator as spec_module
from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
AutoRegressiveSpeculator,
)
from vllm.v1.worker.gpu.spec_decode.multi_module_mtp.speculator import (
MultiModuleMTPSpeculator,
)
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
from vllm.v1.worker.gpu.spec_decode.target_dependent_ar import speculator as spec_module
from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.speculator import (
TargetDependentARSpeculator,
)
class _TestSpeculator(AutoRegressiveSpeculator):
class _TestSpeculator(TargetDependentARSpeculator):
def load_draft_model(self, target_model, target_attn_layer_names):
return self.test_draft_model
+1 -1
View File
@@ -820,7 +820,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
"skip_attn must only be True for initial memory profiling."
)
# Create a dummy scheduler output. Plain draft-model speculation adds
# Create a dummy scheduler output. Standalone AR speculation adds
# one correction slot per request during prefill. The scheduler
# accounts for these slots, while dummy runs bypass the scheduler.
# Adjust the dummy token count only when the expanded draft batch
+3 -3
View File
@@ -71,11 +71,11 @@ def init_speculator(
return EagleSpeculator(vllm_config, device)
elif speculative_config.uses_draft_model():
from vllm.v1.worker.gpu.spec_decode.draft_model.speculator import (
PlainDraftModelSpeculator,
from vllm.v1.worker.gpu.spec_decode.standalone_ar.speculator import (
StandaloneARSpeculator,
)
return PlainDraftModelSpeculator(vllm_config, device)
return StandaloneARSpeculator(vllm_config, device)
elif speculative_config.use_ngram():
from vllm.v1.worker.gpu.spec_decode.ngram.speculator import (
NgramGPUSpeculator,
@@ -3,13 +3,13 @@
import torch.nn as nn
from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
AutoRegressiveSpeculator,
)
from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model
from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.speculator import (
TargetDependentARSpeculator,
)
class EagleSpeculator(AutoRegressiveSpeculator):
class EagleSpeculator(TargetDependentARSpeculator):
def load_draft_model(
self,
target_model: nn.Module,
@@ -17,8 +17,8 @@ from vllm.distributed.parallel_state import get_pp_group
from vllm.logger import init_logger
from vllm.model_executor.model_loader import get_model
from vllm.model_executor.model_loader.utils import get_draft_load_config
from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
AutoRegressiveSpeculator,
from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.speculator import (
TargetDependentARSpeculator,
)
logger = init_logger(__name__)
@@ -45,7 +45,7 @@ def _copy_target_kv_scales(attn: nn.Module, target_attn: nn.Module) -> None:
getattr(attn, scale_name).copy_(getattr(target_attn, scale_name))
class Gemma4Speculator(AutoRegressiveSpeculator):
class Gemma4Speculator(TargetDependentARSpeculator):
@property
def advance_draft_positions(self) -> bool:
# Gemma4 MTP is Q-only and reads K/V from the target's existing cache.
@@ -3,13 +3,13 @@
import torch.nn as nn
from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
AutoRegressiveSpeculator,
)
from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model
from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.speculator import (
TargetDependentARSpeculator,
)
class MTPSpeculator(AutoRegressiveSpeculator):
class MTPSpeculator(TargetDependentARSpeculator):
share_mtp_topk_indices: bool = False
def load_draft_model(
@@ -14,11 +14,11 @@ from vllm.v1.attention.backends.utils import PAD_SLOT_ID
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
from vllm.v1.worker.gpu.dp_utils import DPSyncState, dispatch_cg_and_sync_dp
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import (
SpeculatorCudaGraphManager,
)
from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.cudagraph_utils import (
SpeculatorCudaGraphManager,
)
from vllm.v1.worker.utils import get_uniform_decode_token_count
logger = init_logger(__name__)
@@ -26,22 +26,22 @@ from vllm.v1.worker.utils import AttentionGroup
logger = init_logger(__name__)
class PlainDraftModelSpeculator(DraftModelSpeculator):
"""Speculative decoding using a separate smaller draft LM.
class StandaloneARSpeculator(DraftModelSpeculator):
"""Standalone autoregressive (AR) drafting with a separate smaller LM.
Unlike Eagle, the draft model runs fully independently of the target model.
Step 0 builds an expanded buffer (accepted + correction token + rejected
slots) via a Triton kernel; steps 1..k-1 are single-token decode steps.
"""
# Plain draft model needs one extra slot per request for correction.
# Standalone AR drafting needs one extra slot per request for correction.
num_extra_query_per_req = 1
def __init__(self, vllm_config: VllmConfig, device: torch.device):
super().__init__(vllm_config, device)
# draft_max_seq_len is read by the parent's attention metadata builder.
# Plain draft model doesn't do per-batch adjustment; cap at max.
# Standalone AR drafting doesn't do per-batch adjustment; cap at max.
self.draft_max_seq_len = self.max_model_len
self.last_token_indices = torch.zeros(
@@ -65,10 +65,10 @@ class PlainDraftModelSpeculator(DraftModelSpeculator):
self.supports_mm_inputs = False
def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None:
pass # CUDA graph not yet supported for plain draft model speculator.
pass # CUDA graph not yet supported for standalone AR speculator.
def capture(self) -> None:
pass # CUDA graph not yet supported for plain draft model speculator.
pass # CUDA graph not yet supported for standalone AR speculator.
def set_attn(
self,
@@ -415,7 +415,7 @@ def _prepare_prefill_inputs_kernel(
max_model_len,
BLOCK_SIZE: tl.constexpr,
):
"""Per-request step-0 input preparation for the plain draft-model speculator.
"""Per-request step-0 input preparation for the standalone AR speculator.
Output layout for request i (out_start = query_start_loc[i] + i):
[out_start, out_start + num_valid) accepted tokens
@@ -17,16 +17,22 @@ from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
from vllm.v1.worker.gpu.dp_utils import DPSyncState, dispatch_cg_and_sync_dp
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.model_states.interface import ModelState
from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import (
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.cudagraph_utils import (
SpeculatorCudaGraphManager,
)
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
from vllm.v1.worker.utils import AttentionGroup, get_uniform_decode_token_count
logger = init_logger(__name__)
class AutoRegressiveSpeculator(DraftModelSpeculator):
class TargetDependentARSpeculator(DraftModelSpeculator):
"""Autoregressive (AR) drafting that depends on target model state.
Base for Eagle, MTP, and Gemma4 drafters, which consume the target's
hidden states (and, for Gemma4, its KV cache) at each draft step.
"""
def __init__(self, vllm_config: VllmConfig, device: torch.device):
super().__init__(vllm_config, device)