[V1] Support LLM.apply_model (#18465)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -5,11 +5,10 @@ import asyncio
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from functools import cached_property
|
||||
from typing import (Any, Awaitable, Callable, Dict, List, Optional, Set, Tuple,
|
||||
Union)
|
||||
from typing import Any, Awaitable, Callable, List, Optional, Set, Union
|
||||
|
||||
import torch.nn as nn
|
||||
from typing_extensions import TypeVar
|
||||
from typing_extensions import TypeVar, deprecated
|
||||
|
||||
import vllm.platforms
|
||||
from vllm.config import VllmConfig
|
||||
@@ -63,10 +62,10 @@ class ExecutorBase(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def collective_rpc(self,
|
||||
method: Union[str, Callable[..., _R]],
|
||||
method: Union[str, Callable[[WorkerBase], _R]],
|
||||
timeout: Optional[float] = None,
|
||||
args: Tuple = (),
|
||||
kwargs: Optional[Dict[str, Any]] = None) -> List[_R]:
|
||||
args: tuple = (),
|
||||
kwargs: Optional[dict[str, Any]] = None) -> list[_R]:
|
||||
"""
|
||||
Execute an RPC call on all workers.
|
||||
|
||||
@@ -91,7 +90,7 @@ class ExecutorBase(ABC):
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def determine_num_available_blocks(self) -> Tuple[int, int]:
|
||||
def determine_num_available_blocks(self) -> tuple[int, int]:
|
||||
"""Determine the number of available blocks for the GPU KV cache and
|
||||
swappable CPU KV cache.
|
||||
|
||||
@@ -99,9 +98,10 @@ class ExecutorBase(ABC):
|
||||
ExecutorBase may require modification of the result, e.g. to ensure the
|
||||
selected cache sizes are compatible with all workers.
|
||||
|
||||
Returns a Tuple[num_gpu_blocks, num_cpu_blocks], where num_gpu_blocks
|
||||
are blocks that are "active" on the device and can be appended to.
|
||||
num_cpu_blocks refers to "swapped" blocks in CPU memory and cannot be
|
||||
Returns a tuple `(num_gpu_blocks, num_cpu_blocks)`, where
|
||||
`num_gpu_blocks` are blocks that are "active" on the device and can be
|
||||
appended to.
|
||||
`num_cpu_blocks` refers to "swapped" blocks in CPU memory and cannot be
|
||||
appended to.
|
||||
"""
|
||||
results = self.collective_rpc("determine_num_available_blocks")
|
||||
@@ -127,16 +127,15 @@ class ExecutorBase(ABC):
|
||||
self.collective_rpc("initialize_cache",
|
||||
args=(num_gpu_blocks, num_cpu_blocks))
|
||||
|
||||
@deprecated("`llm_engine.model_executor.apply_model` will no longer work "
|
||||
"in V1 Engine. Please replace with `llm_engine.apply_model` "
|
||||
"and set `VLLM_ALLOW_INSECURE_SERIALIZATION=1`.")
|
||||
def apply_model(self, func: Callable[[nn.Module], _R]) -> list[_R]:
|
||||
"""
|
||||
Run a function directly on the model inside each worker,
|
||||
returning the result for each of them.
|
||||
"""
|
||||
|
||||
def rpc_func(worker: WorkerBase) -> _R:
|
||||
return func(worker.get_model())
|
||||
|
||||
return self.collective_rpc(rpc_func)
|
||||
return self.collective_rpc("apply_model", args=(func, ))
|
||||
|
||||
@cached_property # Avoid unnecessary RPC calls
|
||||
def supported_tasks(self) -> tuple[SupportedTask, ...]:
|
||||
@@ -308,8 +307,8 @@ class DistributedExecutorBase(ExecutorBase):
|
||||
def collective_rpc(self,
|
||||
method: Union[str, Callable],
|
||||
timeout: Optional[float] = None,
|
||||
args: Tuple = (),
|
||||
kwargs: Optional[Dict] = None) -> List[Any]:
|
||||
args: tuple = (),
|
||||
kwargs: Optional[dict[str, Any]] = None) -> list[Any]:
|
||||
return self._run_workers(method, *args, **(kwargs or {}))
|
||||
|
||||
@abstractmethod
|
||||
|
||||
Reference in New Issue
Block a user