diff --git a/.buildkite/hardware_tests/cpu.yaml b/.buildkite/hardware_tests/cpu.yaml index fbd43d80dd..3fdce975d3 100644 --- a/.buildkite/hardware_tests/cpu.yaml +++ b/.buildkite/hardware_tests/cpu.yaml @@ -87,7 +87,7 @@ steps: - | bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 30m " pytest -x -v -s \ - tests/v1/worker/test_gpu_autoregressive_speculator.py \ + tests/v1/worker/test_gpu_target_dependent_ar_speculator.py \ tests/v1/worker/test_gpu_extract_hidden_states_speculator.py \ tests/v1/worker/test_gpu_batch_ordering.py \ tests/v1/worker/test_gpu_rejection_sampler_chunking.py \ diff --git a/tests/v1/cudagraph/test_cudagraph_manager.py b/tests/v1/cudagraph/test_cudagraph_manager.py index 668fbdf61b..1e4071bcfb 100644 --- a/tests/v1/cudagraph/test_cudagraph_manager.py +++ b/tests/v1/cudagraph/test_cudagraph_manager.py @@ -23,10 +23,10 @@ from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp from vllm.v1.worker.gpu.input_batch import InputBuffers from vllm.v1.worker.gpu.pcp_manager import PCPManager -from vllm.v1.worker.gpu.spec_decode.autoregressive import ( +from vllm.v1.worker.gpu.spec_decode.target_dependent_ar import ( cudagraph_utils as spec_cudagraph_utils, ) -from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import ( +from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.cudagraph_utils import ( SpeculatorCudaGraphManager, ) diff --git a/tests/v1/worker/test_gpu_autoregressive_speculator.py b/tests/v1/worker/test_gpu_target_dependent_ar_speculator.py similarity index 98% rename from tests/v1/worker/test_gpu_autoregressive_speculator.py rename to tests/v1/worker/test_gpu_target_dependent_ar_speculator.py index 0a8cfd1a13..20e86f4a78 100644 --- a/tests/v1/worker/test_gpu_autoregressive_speculator.py +++ b/tests/v1/worker/test_gpu_target_dependent_ar_speculator.py @@ -22,17 +22,17 @@ from vllm.v1.attention.backends.flash_attn import FlashAttentionMetadata from vllm.v1.attention.backends.utils import split_decodes_and_prefills from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor from vllm.v1.worker.gpu.spec_decode import speculator as base_spec_module -from vllm.v1.worker.gpu.spec_decode.autoregressive import speculator as spec_module -from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import ( - AutoRegressiveSpeculator, -) from vllm.v1.worker.gpu.spec_decode.multi_module_mtp.speculator import ( MultiModuleMTPSpeculator, ) from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator +from vllm.v1.worker.gpu.spec_decode.target_dependent_ar import speculator as spec_module +from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.speculator import ( + TargetDependentARSpeculator, +) -class _TestSpeculator(AutoRegressiveSpeculator): +class _TestSpeculator(TargetDependentARSpeculator): def load_draft_model(self, target_model, target_attn_layer_names): return self.test_draft_model diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index d0c08707f4..373042a415 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -820,7 +820,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): "skip_attn must only be True for initial memory profiling." ) - # Create a dummy scheduler output. Plain draft-model speculation adds + # Create a dummy scheduler output. Standalone AR speculation adds # one correction slot per request during prefill. The scheduler # accounts for these slots, while dummy runs bypass the scheduler. # Adjust the dummy token count only when the expanded draft batch diff --git a/vllm/v1/worker/gpu/spec_decode/__init__.py b/vllm/v1/worker/gpu/spec_decode/__init__.py index 902a4f085c..fc028941cf 100644 --- a/vllm/v1/worker/gpu/spec_decode/__init__.py +++ b/vllm/v1/worker/gpu/spec_decode/__init__.py @@ -71,11 +71,11 @@ def init_speculator( return EagleSpeculator(vllm_config, device) elif speculative_config.uses_draft_model(): - from vllm.v1.worker.gpu.spec_decode.draft_model.speculator import ( - PlainDraftModelSpeculator, + from vllm.v1.worker.gpu.spec_decode.standalone_ar.speculator import ( + StandaloneARSpeculator, ) - return PlainDraftModelSpeculator(vllm_config, device) + return StandaloneARSpeculator(vllm_config, device) elif speculative_config.use_ngram(): from vllm.v1.worker.gpu.spec_decode.ngram.speculator import ( NgramGPUSpeculator, diff --git a/vllm/v1/worker/gpu/spec_decode/eagle/speculator.py b/vllm/v1/worker/gpu/spec_decode/eagle/speculator.py index e878872e62..0611d42b1c 100644 --- a/vllm/v1/worker/gpu/spec_decode/eagle/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/eagle/speculator.py @@ -3,13 +3,13 @@ import torch.nn as nn -from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import ( - AutoRegressiveSpeculator, -) from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model +from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.speculator import ( + TargetDependentARSpeculator, +) -class EagleSpeculator(AutoRegressiveSpeculator): +class EagleSpeculator(TargetDependentARSpeculator): def load_draft_model( self, target_model: nn.Module, diff --git a/vllm/v1/worker/gpu/spec_decode/gemma4/speculator.py b/vllm/v1/worker/gpu/spec_decode/gemma4/speculator.py index 6f76e5055a..9d553830ae 100644 --- a/vllm/v1/worker/gpu/spec_decode/gemma4/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/gemma4/speculator.py @@ -17,8 +17,8 @@ from vllm.distributed.parallel_state import get_pp_group from vllm.logger import init_logger from vllm.model_executor.model_loader import get_model from vllm.model_executor.model_loader.utils import get_draft_load_config -from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import ( - AutoRegressiveSpeculator, +from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.speculator import ( + TargetDependentARSpeculator, ) logger = init_logger(__name__) @@ -45,7 +45,7 @@ def _copy_target_kv_scales(attn: nn.Module, target_attn: nn.Module) -> None: getattr(attn, scale_name).copy_(getattr(target_attn, scale_name)) -class Gemma4Speculator(AutoRegressiveSpeculator): +class Gemma4Speculator(TargetDependentARSpeculator): @property def advance_draft_positions(self) -> bool: # Gemma4 MTP is Q-only and reads K/V from the target's existing cache. diff --git a/vllm/v1/worker/gpu/spec_decode/mtp/speculator.py b/vllm/v1/worker/gpu/spec_decode/mtp/speculator.py index d94702612d..a0064197f7 100644 --- a/vllm/v1/worker/gpu/spec_decode/mtp/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/mtp/speculator.py @@ -3,13 +3,13 @@ import torch.nn as nn -from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import ( - AutoRegressiveSpeculator, -) from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model +from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.speculator import ( + TargetDependentARSpeculator, +) -class MTPSpeculator(AutoRegressiveSpeculator): +class MTPSpeculator(TargetDependentARSpeculator): share_mtp_topk_indices: bool = False def load_draft_model( diff --git a/vllm/v1/worker/gpu/spec_decode/multi_module_mtp/speculator.py b/vllm/v1/worker/gpu/spec_decode/multi_module_mtp/speculator.py index 71f7f89416..919e939c03 100644 --- a/vllm/v1/worker/gpu/spec_decode/multi_module_mtp/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/multi_module_mtp/speculator.py @@ -14,11 +14,11 @@ from vllm.v1.attention.backends.utils import PAD_SLOT_ID from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer from vllm.v1.worker.gpu.dp_utils import DPSyncState, dispatch_cg_and_sync_dp from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers -from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import ( - SpeculatorCudaGraphManager, -) from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator +from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.cudagraph_utils import ( + SpeculatorCudaGraphManager, +) from vllm.v1.worker.utils import get_uniform_decode_token_count logger = init_logger(__name__) diff --git a/vllm/v1/worker/gpu/spec_decode/autoregressive/__init__.py b/vllm/v1/worker/gpu/spec_decode/standalone_ar/__init__.py similarity index 100% rename from vllm/v1/worker/gpu/spec_decode/autoregressive/__init__.py rename to vllm/v1/worker/gpu/spec_decode/standalone_ar/__init__.py diff --git a/vllm/v1/worker/gpu/spec_decode/draft_model/speculator.py b/vllm/v1/worker/gpu/spec_decode/standalone_ar/speculator.py similarity index 97% rename from vllm/v1/worker/gpu/spec_decode/draft_model/speculator.py rename to vllm/v1/worker/gpu/spec_decode/standalone_ar/speculator.py index 5be7b0c86d..8bdfb8530b 100644 --- a/vllm/v1/worker/gpu/spec_decode/draft_model/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/standalone_ar/speculator.py @@ -26,22 +26,22 @@ from vllm.v1.worker.utils import AttentionGroup logger = init_logger(__name__) -class PlainDraftModelSpeculator(DraftModelSpeculator): - """Speculative decoding using a separate smaller draft LM. +class StandaloneARSpeculator(DraftModelSpeculator): + """Standalone autoregressive (AR) drafting with a separate smaller LM. Unlike Eagle, the draft model runs fully independently of the target model. Step 0 builds an expanded buffer (accepted + correction token + rejected slots) via a Triton kernel; steps 1..k-1 are single-token decode steps. """ - # Plain draft model needs one extra slot per request for correction. + # Standalone AR drafting needs one extra slot per request for correction. num_extra_query_per_req = 1 def __init__(self, vllm_config: VllmConfig, device: torch.device): super().__init__(vllm_config, device) # draft_max_seq_len is read by the parent's attention metadata builder. - # Plain draft model doesn't do per-batch adjustment; cap at max. + # Standalone AR drafting doesn't do per-batch adjustment; cap at max. self.draft_max_seq_len = self.max_model_len self.last_token_indices = torch.zeros( @@ -65,10 +65,10 @@ class PlainDraftModelSpeculator(DraftModelSpeculator): self.supports_mm_inputs = False def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None: - pass # CUDA graph not yet supported for plain draft model speculator. + pass # CUDA graph not yet supported for standalone AR speculator. def capture(self) -> None: - pass # CUDA graph not yet supported for plain draft model speculator. + pass # CUDA graph not yet supported for standalone AR speculator. def set_attn( self, @@ -415,7 +415,7 @@ def _prepare_prefill_inputs_kernel( max_model_len, BLOCK_SIZE: tl.constexpr, ): - """Per-request step-0 input preparation for the plain draft-model speculator. + """Per-request step-0 input preparation for the standalone AR speculator. Output layout for request i (out_start = query_start_loc[i] + i): [out_start, out_start + num_valid) accepted tokens diff --git a/vllm/v1/worker/gpu/spec_decode/draft_model/__init__.py b/vllm/v1/worker/gpu/spec_decode/target_dependent_ar/__init__.py similarity index 100% rename from vllm/v1/worker/gpu/spec_decode/draft_model/__init__.py rename to vllm/v1/worker/gpu/spec_decode/target_dependent_ar/__init__.py diff --git a/vllm/v1/worker/gpu/spec_decode/autoregressive/cudagraph_utils.py b/vllm/v1/worker/gpu/spec_decode/target_dependent_ar/cudagraph_utils.py similarity index 100% rename from vllm/v1/worker/gpu/spec_decode/autoregressive/cudagraph_utils.py rename to vllm/v1/worker/gpu/spec_decode/target_dependent_ar/cudagraph_utils.py diff --git a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py b/vllm/v1/worker/gpu/spec_decode/target_dependent_ar/speculator.py similarity index 99% rename from vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py rename to vllm/v1/worker/gpu/spec_decode/target_dependent_ar/speculator.py index 46ff826815..9a8ed7b8b4 100644 --- a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/target_dependent_ar/speculator.py @@ -17,16 +17,22 @@ from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor from vllm.v1.worker.gpu.dp_utils import DPSyncState, dispatch_cg_and_sync_dp from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers from vllm.v1.worker.gpu.model_states.interface import ModelState -from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import ( +from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator +from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.cudagraph_utils import ( SpeculatorCudaGraphManager, ) -from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator from vllm.v1.worker.utils import AttentionGroup, get_uniform_decode_token_count logger = init_logger(__name__) -class AutoRegressiveSpeculator(DraftModelSpeculator): +class TargetDependentARSpeculator(DraftModelSpeculator): + """Autoregressive (AR) drafting that depends on target model state. + + Base for Eagle, MTP, and Gemma4 drafters, which consume the target's + hidden states (and, for Gemma4, its KV cache) at each draft step. + """ + def __init__(self, vllm_config: VllmConfig, device: torch.device): super().__init__(vllm_config, device)