[Misc] [Core] Implement RFC "Augment BaseExecutor interfaces to enable hardware-agnostic speculative decoding" (#3837)
This commit is contained in:
@@ -3,7 +3,6 @@ from typing import Dict, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import CacheConfig
|
||||
from vllm.model_executor.layers.rejection_sampler import RejectionSampler
|
||||
from vllm.sequence import (SamplerOutput, SequenceGroupMetadata,
|
||||
SequenceGroupOutput, SequenceOutput)
|
||||
@@ -15,9 +14,10 @@ from vllm.spec_decode.multi_step_worker import MultiStepWorker
|
||||
from vllm.spec_decode.util import (get_all_seq_ids, nvtx_range,
|
||||
split_batch_by_proposal_len)
|
||||
from vllm.worker.worker import Worker
|
||||
from vllm.worker.worker_base import LoraNotSupportedWorkerBase
|
||||
|
||||
|
||||
class SpecDecodeWorker:
|
||||
class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
"""Worker which implements speculative decoding.
|
||||
|
||||
Speculative decoding reduces decoding per-token latency by using a proposal
|
||||
@@ -94,10 +94,7 @@ class SpecDecodeWorker:
|
||||
device=self.device,
|
||||
vocab_size=self._vocab_size)
|
||||
|
||||
def profile_num_available_blocks(self, block_size: int,
|
||||
gpu_memory_utilization: float,
|
||||
cpu_swap_space: int,
|
||||
cache_dtype: str) -> Tuple[int, int]:
|
||||
def determine_num_available_blocks(self) -> Tuple[int, int]:
|
||||
"""Determine the number of cache blocks to use.
|
||||
|
||||
This is done by profiling the scorer model (which is typically the
|
||||
@@ -106,27 +103,26 @@ class SpecDecodeWorker:
|
||||
such that the number of blocks is equal in both KV caches.
|
||||
"""
|
||||
num_gpu_blocks, num_cpu_blocks = (
|
||||
self.scorer_worker.profile_num_available_blocks(
|
||||
block_size, gpu_memory_utilization, cpu_swap_space,
|
||||
cache_dtype))
|
||||
self.scorer_worker.determine_num_available_blocks())
|
||||
|
||||
scorer_cache_block_size_bytes = (
|
||||
self.scorer_worker.get_cache_block_size_bytes(
|
||||
block_size, cache_dtype))
|
||||
self.scorer_worker.get_cache_block_size_bytes())
|
||||
proposer_cache_block_size_bytes = (
|
||||
self.proposer_worker.get_cache_block_size_bytes(
|
||||
block_size, cache_dtype))
|
||||
self.proposer_worker.get_cache_block_size_bytes())
|
||||
|
||||
new_num_gpu_blocks = split_num_cache_blocks_evenly(
|
||||
scorer_cache_block_size_bytes, proposer_cache_block_size_bytes,
|
||||
num_gpu_blocks)
|
||||
return new_num_gpu_blocks, num_cpu_blocks
|
||||
|
||||
def init_cache_engine(self, cache_config: CacheConfig):
|
||||
def initialize_cache(self, num_gpu_blocks: int,
|
||||
num_cpu_blocks: int) -> None:
|
||||
"""Initialize the cache engine of the scorer and proposer workers.
|
||||
"""
|
||||
self.scorer_worker.init_cache_engine(cache_config)
|
||||
self.proposer_worker.init_cache_engine(cache_config)
|
||||
self.scorer_worker.initialize_cache(num_gpu_blocks=num_gpu_blocks,
|
||||
num_cpu_blocks=num_cpu_blocks)
|
||||
self.proposer_worker.initialize_cache(num_gpu_blocks=num_gpu_blocks,
|
||||
num_cpu_blocks=num_cpu_blocks)
|
||||
|
||||
@torch.inference_mode()
|
||||
def execute_model(
|
||||
@@ -351,6 +347,16 @@ class SpecDecodeWorker:
|
||||
def device(self):
|
||||
return self.scorer_worker.device
|
||||
|
||||
def get_cache_block_size_bytes(self):
|
||||
"""Return the size of a cache block in bytes.
|
||||
|
||||
This function is only used to compose workers within a SpecDecodeWorker.
|
||||
We leave composing a SpecDecodeWorker within a SpecDecodeWorker
|
||||
undefined for now, although it could be implemented in the future.
|
||||
See https://arxiv.org/abs/2308.04623.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def split_num_cache_blocks_evenly(scorer_cache_block_size_bytes: int,
|
||||
proposer_cache_block_size_bytes: int,
|
||||
|
||||
Reference in New Issue
Block a user