[ModelRunner V2] Misc minor simplifications and optimizations (#33467)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -5,10 +5,7 @@ import torch
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
|
||||
def init_speculator(
|
||||
vllm_config: VllmConfig,
|
||||
device: torch.device,
|
||||
):
|
||||
def init_speculator(vllm_config: VllmConfig, device: torch.device):
|
||||
speculative_config = vllm_config.speculative_config
|
||||
assert speculative_config is not None
|
||||
if speculative_config.use_eagle():
|
||||
|
||||
@@ -54,26 +54,15 @@ class EagleSpeculator:
|
||||
device=device,
|
||||
)
|
||||
self.hidden_states = torch.zeros(
|
||||
self.max_num_tokens,
|
||||
self.hidden_size,
|
||||
dtype=self.dtype,
|
||||
device=device,
|
||||
self.max_num_tokens, self.hidden_size, dtype=self.dtype, device=device
|
||||
)
|
||||
self.idx_mapping = torch.zeros(
|
||||
self.max_num_reqs,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
self.max_num_reqs, dtype=torch.int32, device=device
|
||||
)
|
||||
self.temperature = torch.zeros(
|
||||
self.max_num_reqs,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
self.seeds = torch.zeros(
|
||||
self.max_num_reqs,
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
self.max_num_reqs, dtype=torch.float32, device=device
|
||||
)
|
||||
self.seeds = torch.zeros(self.max_num_reqs, dtype=torch.int64, device=device)
|
||||
self.draft_tokens = torch.zeros(
|
||||
self.max_num_reqs,
|
||||
self.num_speculative_steps,
|
||||
|
||||
@@ -19,11 +19,7 @@ from vllm.v1.worker.gpu.input_batch import InputBuffers
|
||||
|
||||
|
||||
class EagleCudaGraphManager:
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
device: torch.device,
|
||||
):
|
||||
def __init__(self, vllm_config: VllmConfig, device: torch.device):
|
||||
self.vllm_config = vllm_config
|
||||
self.scheduler_config = vllm_config.scheduler_config
|
||||
self.device = device
|
||||
@@ -35,16 +31,10 @@ class EagleCudaGraphManager:
|
||||
self.compilation_config = vllm_config.compilation_config
|
||||
assert self.compilation_config is not None
|
||||
|
||||
cudagraph_mode: CUDAGraphMode
|
||||
if self.compilation_config.cudagraph_mode is None:
|
||||
cudagraph_mode = CUDAGraphMode.NONE
|
||||
else:
|
||||
cudagraph_mode = self.compilation_config.cudagraph_mode
|
||||
if cudagraph_mode == CUDAGraphMode.FULL:
|
||||
# NOTE(woosuk): For Eagle, we only use CUDA graphs for decode.
|
||||
cudagraph_mode = CUDAGraphMode.FULL_DECODE_ONLY
|
||||
|
||||
self.cudagraph_mode = cudagraph_mode
|
||||
self.cudagraph_mode = self.compilation_config.cudagraph_mode
|
||||
if self.cudagraph_mode == CUDAGraphMode.FULL:
|
||||
# NOTE(woosuk): For Eagle, we only use CUDA graphs for decode.
|
||||
self.cudagraph_mode = CUDAGraphMode.FULL_DECODE_ONLY
|
||||
|
||||
self.cudagraph_sizes = get_cudagraph_sizes(
|
||||
self.compilation_config.cudagraph_capture_sizes,
|
||||
|
||||
Reference in New Issue
Block a user