Convert formatting to use ruff instead of yapf + isort (#26247)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -4,8 +4,9 @@
|
||||
import asyncio
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Awaitable
|
||||
from functools import cached_property
|
||||
from typing import Any, Awaitable, Callable, List, Optional, Set, Union
|
||||
from typing import Any, Callable, List, Optional, Set, Union
|
||||
|
||||
from typing_extensions import TypeVar
|
||||
|
||||
@@ -60,11 +61,13 @@ class ExecutorBase(ABC):
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def collective_rpc(self,
|
||||
method: Union[str, Callable[[WorkerBase], _R]],
|
||||
timeout: Optional[float] = None,
|
||||
args: tuple = (),
|
||||
kwargs: Optional[dict[str, Any]] = None) -> list[_R]:
|
||||
def collective_rpc(
|
||||
self,
|
||||
method: Union[str, Callable[[WorkerBase], _R]],
|
||||
timeout: Optional[float] = None,
|
||||
args: tuple = (),
|
||||
kwargs: Optional[dict[str, Any]] = None,
|
||||
) -> list[_R]:
|
||||
"""
|
||||
Execute an RPC call on all workers.
|
||||
|
||||
@@ -109,22 +112,29 @@ class ExecutorBase(ABC):
|
||||
return a, b
|
||||
|
||||
def initialize_cache(self, num_gpu_blocks: int, num_cpu_blocks) -> None:
|
||||
"""Initialize the KV cache by invoking the underlying worker.
|
||||
"""
|
||||
"""Initialize the KV cache by invoking the underlying worker."""
|
||||
# NOTE: This is logged in the executor because there can be >1 workers.
|
||||
logger.info("# %s blocks: %d, # CPU blocks: %d",
|
||||
vllm.platforms.current_platform.device_name,
|
||||
num_gpu_blocks, num_cpu_blocks)
|
||||
max_concurrency = (num_gpu_blocks * self.cache_config.block_size /
|
||||
self.model_config.max_model_len)
|
||||
logger.info("Maximum concurrency for %s tokens per request: %.2fx",
|
||||
self.model_config.max_model_len, max_concurrency)
|
||||
logger.info(
|
||||
"# %s blocks: %d, # CPU blocks: %d",
|
||||
vllm.platforms.current_platform.device_name,
|
||||
num_gpu_blocks,
|
||||
num_cpu_blocks,
|
||||
)
|
||||
max_concurrency = (
|
||||
num_gpu_blocks
|
||||
* self.cache_config.block_size
|
||||
/ self.model_config.max_model_len
|
||||
)
|
||||
logger.info(
|
||||
"Maximum concurrency for %s tokens per request: %.2fx",
|
||||
self.model_config.max_model_len,
|
||||
max_concurrency,
|
||||
)
|
||||
|
||||
self.cache_config.num_gpu_blocks = num_gpu_blocks
|
||||
self.cache_config.num_cpu_blocks = num_cpu_blocks
|
||||
|
||||
self.collective_rpc("initialize_cache",
|
||||
args=(num_gpu_blocks, num_cpu_blocks))
|
||||
self.collective_rpc("initialize_cache", args=(num_gpu_blocks, num_cpu_blocks))
|
||||
|
||||
@cached_property # Avoid unnecessary RPC calls
|
||||
def supported_tasks(self) -> tuple[SupportedTask, ...]:
|
||||
@@ -134,8 +144,7 @@ class ExecutorBase(ABC):
|
||||
def execute_model(
|
||||
self, execute_model_req: ExecuteModelRequest
|
||||
) -> Optional[List[Union[SamplerOutput, PoolerOutput]]]:
|
||||
output = self.collective_rpc("execute_model",
|
||||
args=(execute_model_req, ))
|
||||
output = self.collective_rpc("execute_model", args=(execute_model_req,))
|
||||
return output[0]
|
||||
|
||||
def stop_remote_worker_execution_loop(self) -> None:
|
||||
@@ -144,15 +153,15 @@ class ExecutorBase(ABC):
|
||||
|
||||
def add_lora(self, lora_request: LoRARequest) -> bool:
|
||||
assert lora_request.lora_int_id > 0, "lora_id must be greater than 0."
|
||||
return all(self.collective_rpc("add_lora", args=(lora_request, )))
|
||||
return all(self.collective_rpc("add_lora", args=(lora_request,)))
|
||||
|
||||
def remove_lora(self, lora_id: int) -> bool:
|
||||
assert lora_id > 0, "lora_id must be greater than 0."
|
||||
return all(self.collective_rpc("remove_lora", args=(lora_id, )))
|
||||
return all(self.collective_rpc("remove_lora", args=(lora_id,)))
|
||||
|
||||
def pin_lora(self, lora_id: int) -> bool:
|
||||
assert lora_id > 0, "lora_id must be greater than 0."
|
||||
return all(self.collective_rpc("pin_lora", args=(lora_id, )))
|
||||
return all(self.collective_rpc("pin_lora", args=(lora_id,)))
|
||||
|
||||
def list_loras(self) -> Set[int]:
|
||||
sets = self.collective_rpc("list_loras")
|
||||
@@ -175,8 +184,9 @@ class ExecutorBase(ABC):
|
||||
time_after_sleep = time.perf_counter()
|
||||
self.sleeping_tags = {"weights", "kv_cache"}
|
||||
self.is_sleeping = True
|
||||
logger.info("It took %.6f seconds to fall asleep.",
|
||||
time_after_sleep - time_before_sleep)
|
||||
logger.info(
|
||||
"It took %.6f seconds to fall asleep.", time_after_sleep - time_before_sleep
|
||||
)
|
||||
|
||||
def wake_up(self, tags: Optional[list[str]] = None):
|
||||
if not self.is_sleeping:
|
||||
@@ -185,15 +195,18 @@ class ExecutorBase(ABC):
|
||||
if tags:
|
||||
for tag in tags:
|
||||
if tag not in self.sleeping_tags:
|
||||
logger.warning("Tag %s is not in sleeping tags %s", tag,
|
||||
self.sleeping_tags)
|
||||
logger.warning(
|
||||
"Tag %s is not in sleeping tags %s", tag, self.sleeping_tags
|
||||
)
|
||||
return
|
||||
time_before_wakeup = time.perf_counter()
|
||||
self.collective_rpc("wake_up", kwargs=dict(tags=tags))
|
||||
time_after_wakeup = time.perf_counter()
|
||||
logger.info("It took %.6f seconds to wake up tags %s.",
|
||||
time_after_wakeup - time_before_wakeup,
|
||||
tags if tags is not None else self.sleeping_tags)
|
||||
logger.info(
|
||||
"It took %.6f seconds to wake up tags %s.",
|
||||
time_after_wakeup - time_before_wakeup,
|
||||
tags if tags is not None else self.sleeping_tags,
|
||||
)
|
||||
if tags:
|
||||
for tag in tags:
|
||||
self.sleeping_tags.remove(tag)
|
||||
@@ -208,10 +221,10 @@ class ExecutorBase(ABC):
|
||||
pattern: Optional[str] = None,
|
||||
max_size: Optional[int] = None,
|
||||
) -> None:
|
||||
self.collective_rpc("save_sharded_state",
|
||||
kwargs=dict(path=path,
|
||||
pattern=pattern,
|
||||
max_size=max_size))
|
||||
self.collective_rpc(
|
||||
"save_sharded_state",
|
||||
kwargs=dict(path=path, pattern=pattern, max_size=max_size),
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
def check_health(self) -> None:
|
||||
@@ -224,8 +237,8 @@ class ExecutorBase(ABC):
|
||||
self.collective_rpc("shutdown")
|
||||
|
||||
async def execute_model_async(
|
||||
self,
|
||||
execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
|
||||
self, execute_model_req: ExecuteModelRequest
|
||||
) -> List[SamplerOutput]:
|
||||
"""Executes one model step on the given sequences."""
|
||||
output = await make_async(self.execute_model)(execute_model_req)
|
||||
return output
|
||||
@@ -242,7 +255,8 @@ class ExecutorBase(ABC):
|
||||
def init_kv_output_aggregator(self, finished_count: Optional[int]) -> None:
|
||||
"""Init KVOutputAggregator"""
|
||||
self.kv_output_aggregator = KVOutputAggregator(
|
||||
finished_count or self.parallel_config.world_size)
|
||||
finished_count or self.parallel_config.world_size
|
||||
)
|
||||
|
||||
|
||||
class DistributedExecutorBase(ExecutorBase):
|
||||
@@ -263,7 +277,8 @@ class DistributedExecutorBase(ExecutorBase):
|
||||
if self.parallel_worker_tasks is None:
|
||||
self.parallel_worker_tasks = self._run_workers(
|
||||
"start_worker_execution_loop",
|
||||
async_run_tensor_parallel_workers_only=True)
|
||||
async_run_tensor_parallel_workers_only=True,
|
||||
)
|
||||
|
||||
# Only the driver worker returns the sampling results.
|
||||
driver_outputs = self._driver_execute_model(execute_model_req)
|
||||
@@ -293,11 +308,13 @@ class DistributedExecutorBase(ExecutorBase):
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def collective_rpc(self,
|
||||
method: Union[str, Callable],
|
||||
timeout: Optional[float] = None,
|
||||
args: tuple = (),
|
||||
kwargs: Optional[dict[str, Any]] = None) -> list[Any]:
|
||||
def collective_rpc(
|
||||
self,
|
||||
method: Union[str, Callable],
|
||||
timeout: Optional[float] = None,
|
||||
args: tuple = (),
|
||||
kwargs: Optional[dict[str, Any]] = None,
|
||||
) -> list[Any]:
|
||||
return self._run_workers(method, *args, **(kwargs or {}))
|
||||
|
||||
@abstractmethod
|
||||
@@ -328,12 +345,13 @@ class DistributedExecutorBase(ExecutorBase):
|
||||
raise NotImplementedError
|
||||
|
||||
async def execute_model_async(
|
||||
self,
|
||||
execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
|
||||
self, execute_model_req: ExecuteModelRequest
|
||||
) -> List[SamplerOutput]:
|
||||
if self.parallel_worker_tasks is None:
|
||||
# Start model execution loop running in the parallel workers
|
||||
self.parallel_worker_tasks = asyncio.create_task(
|
||||
self._start_worker_execution_loop())
|
||||
self._start_worker_execution_loop()
|
||||
)
|
||||
|
||||
# Only the driver worker returns the sampling results.
|
||||
return await self._driver_execute_model_async(execute_model_req)
|
||||
|
||||
Reference in New Issue
Block a user