Convert formatting to use ruff instead of yapf + isort (#26247)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
+126
-100
@@ -27,18 +27,14 @@ from vllm.pooling_params import PoolingParams
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.tasks import SupportedTask
|
||||
from vllm.tracing import init_tracer
|
||||
from vllm.transformers_utils.config import (
|
||||
maybe_register_config_serialize_by_value)
|
||||
from vllm.transformers_utils.tokenizer import (AnyTokenizer,
|
||||
init_tokenizer_from_configs)
|
||||
from vllm.transformers_utils.config import maybe_register_config_serialize_by_value
|
||||
from vllm.transformers_utils.tokenizer import AnyTokenizer, init_tokenizer_from_configs
|
||||
from vllm.usage.usage_lib import UsageContext
|
||||
from vllm.utils import (Device, as_list, cancel_task_threadsafe, cdiv,
|
||||
deprecate_kwargs)
|
||||
from vllm.utils import Device, as_list, cancel_task_threadsafe, cdiv, deprecate_kwargs
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm.v1.engine.core_client import EngineCoreClient
|
||||
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
|
||||
from vllm.v1.engine.output_processor import (OutputProcessor,
|
||||
RequestOutputCollector)
|
||||
from vllm.v1.engine.output_processor import OutputProcessor, RequestOutputCollector
|
||||
from vllm.v1.engine.parallel_sampling import ParentRequest
|
||||
from vllm.v1.engine.processor import Processor
|
||||
from vllm.v1.executor.abstract import Executor
|
||||
@@ -50,7 +46,6 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class AsyncLLM(EngineClient):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
@@ -91,7 +86,8 @@ class AsyncLLM(EngineClient):
|
||||
"Using V1 AsyncLLMEngine, but envs.VLLM_USE_V1=False. "
|
||||
"This should not happen. As a workaround, try using "
|
||||
"AsyncLLMEngine.from_vllm_config(...) or explicitly set "
|
||||
"VLLM_USE_V1=0 or 1 and report this issue on Github.")
|
||||
"VLLM_USE_V1=0 or 1 and report this issue on Github."
|
||||
)
|
||||
|
||||
# Ensure we can serialize custom transformer configs
|
||||
maybe_register_config_serialize_by_value()
|
||||
@@ -105,14 +101,16 @@ class AsyncLLM(EngineClient):
|
||||
if not log_stats and stat_loggers is not None:
|
||||
logger.info(
|
||||
"AsyncLLM created with log_stats=False and non-empty custom "
|
||||
"logger list; enabling logging without default stat loggers")
|
||||
"logger list; enabling logging without default stat loggers"
|
||||
)
|
||||
|
||||
if self.model_config.skip_tokenizer_init:
|
||||
self.tokenizer = None
|
||||
else:
|
||||
# Tokenizer (+ ensure liveness if running in another process).
|
||||
self.tokenizer = init_tokenizer_from_configs(
|
||||
model_config=vllm_config.model_config)
|
||||
model_config=vllm_config.model_config
|
||||
)
|
||||
|
||||
# Processor (converts Inputs --> EngineCoreRequests).
|
||||
self.processor = Processor(
|
||||
@@ -122,12 +120,13 @@ class AsyncLLM(EngineClient):
|
||||
)
|
||||
|
||||
# OutputProcessor (converts EngineCoreOutputs --> RequestOutput).
|
||||
self.output_processor = OutputProcessor(self.tokenizer,
|
||||
log_stats=self.log_stats)
|
||||
self.output_processor = OutputProcessor(
|
||||
self.tokenizer, log_stats=self.log_stats
|
||||
)
|
||||
if self.observability_config.otlp_traces_endpoint is not None:
|
||||
tracer = init_tracer(
|
||||
"vllm.llm_engine",
|
||||
self.observability_config.otlp_traces_endpoint)
|
||||
"vllm.llm_engine", self.observability_config.otlp_traces_endpoint
|
||||
)
|
||||
self.output_processor.tracer = tracer
|
||||
|
||||
# EngineCore (starts the engine in background process).
|
||||
@@ -163,7 +162,8 @@ class AsyncLLM(EngineClient):
|
||||
if envs.VLLM_TORCH_PROFILER_DIR:
|
||||
logger.info(
|
||||
"Torch profiler enabled. AsyncLLM CPU traces will be collected under %s", # noqa: E501
|
||||
envs.VLLM_TORCH_PROFILER_DIR)
|
||||
envs.VLLM_TORCH_PROFILER_DIR,
|
||||
)
|
||||
worker_name = f"{socket.gethostname()}_{os.getpid()}.async_llm"
|
||||
self.profiler = torch.profiler.profile(
|
||||
activities=[
|
||||
@@ -171,37 +171,39 @@ class AsyncLLM(EngineClient):
|
||||
],
|
||||
with_stack=envs.VLLM_TORCH_PROFILER_WITH_STACK,
|
||||
on_trace_ready=torch.profiler.tensorboard_trace_handler(
|
||||
envs.VLLM_TORCH_PROFILER_DIR,
|
||||
worker_name=worker_name,
|
||||
use_gzip=True))
|
||||
envs.VLLM_TORCH_PROFILER_DIR, worker_name=worker_name, use_gzip=True
|
||||
),
|
||||
)
|
||||
else:
|
||||
self.profiler = None
|
||||
|
||||
@classmethod
|
||||
@deprecate_kwargs(
|
||||
"disable_log_requests",
|
||||
additional_message=("This argument will have no effect. "
|
||||
"Use `enable_log_requests` instead."),
|
||||
additional_message=(
|
||||
"This argument will have no effect. Use `enable_log_requests` instead."
|
||||
),
|
||||
)
|
||||
def from_vllm_config(
|
||||
cls,
|
||||
vllm_config: VllmConfig,
|
||||
start_engine_loop: bool = True,
|
||||
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
|
||||
stat_loggers: Optional[list[StatLoggerFactory]] = None,
|
||||
enable_log_requests: bool = False,
|
||||
disable_log_stats: bool = False,
|
||||
client_addresses: Optional[dict[str, str]] = None,
|
||||
client_count: int = 1,
|
||||
client_index: int = 0,
|
||||
disable_log_requests: bool = True, # Deprecated, will be removed
|
||||
cls,
|
||||
vllm_config: VllmConfig,
|
||||
start_engine_loop: bool = True,
|
||||
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
|
||||
stat_loggers: Optional[list[StatLoggerFactory]] = None,
|
||||
enable_log_requests: bool = False,
|
||||
disable_log_stats: bool = False,
|
||||
client_addresses: Optional[dict[str, str]] = None,
|
||||
client_count: int = 1,
|
||||
client_index: int = 0,
|
||||
disable_log_requests: bool = True, # Deprecated, will be removed
|
||||
) -> "AsyncLLM":
|
||||
if not envs.VLLM_USE_V1:
|
||||
raise ValueError(
|
||||
"Using V1 AsyncLLMEngine, but envs.VLLM_USE_V1=False. "
|
||||
"This should not happen. As a workaround, try using "
|
||||
"AsyncLLMEngine.from_vllm_config(...) or explicitly set "
|
||||
"VLLM_USE_V1=0 or 1 and report this issue on Github.")
|
||||
"VLLM_USE_V1=0 or 1 and report this issue on Github."
|
||||
)
|
||||
|
||||
# Create the LLMEngine.
|
||||
return cls(
|
||||
@@ -288,14 +290,20 @@ class AsyncLLM(EngineClient):
|
||||
assert prompt_text is None
|
||||
logger.warning_once(
|
||||
"Processor has been moved under OpenAIServing and will "
|
||||
"be removed from AsyncLLM in v0.13.")
|
||||
request = self.processor.process_inputs(request_id, prompt, params,
|
||||
arrival_time, lora_request,
|
||||
tokenization_kwargs,
|
||||
trace_headers, priority,
|
||||
data_parallel_rank)
|
||||
prompt_text = (prompt if isinstance(prompt, str) else
|
||||
prompt.get("prompt"))
|
||||
"be removed from AsyncLLM in v0.13."
|
||||
)
|
||||
request = self.processor.process_inputs(
|
||||
request_id,
|
||||
prompt,
|
||||
params,
|
||||
arrival_time,
|
||||
lora_request,
|
||||
tokenization_kwargs,
|
||||
trace_headers,
|
||||
priority,
|
||||
data_parallel_rank,
|
||||
)
|
||||
prompt_text = prompt if isinstance(prompt, str) else prompt.get("prompt")
|
||||
|
||||
if is_pooling or params.n == 1:
|
||||
await self._add_request(request, prompt_text, None, 0, queue)
|
||||
@@ -310,22 +318,24 @@ class AsyncLLM(EngineClient):
|
||||
parent_request = ParentRequest(request_id, parent_params)
|
||||
for idx in range(parent_params.n):
|
||||
request_id, child_params = parent_request.get_child_info(idx)
|
||||
child_request = request if idx == parent_params.n - 1 else copy(
|
||||
request)
|
||||
child_request = request if idx == parent_params.n - 1 else copy(request)
|
||||
child_request.request_id = request_id
|
||||
child_request.sampling_params = child_params
|
||||
await self._add_request(child_request, prompt_text, parent_request,
|
||||
idx, queue)
|
||||
await self._add_request(
|
||||
child_request, prompt_text, parent_request, idx, queue
|
||||
)
|
||||
return queue
|
||||
|
||||
async def _add_request(self, request: EngineCoreRequest,
|
||||
prompt: Optional[str],
|
||||
parent_req: Optional[ParentRequest], index: int,
|
||||
queue: RequestOutputCollector):
|
||||
|
||||
async def _add_request(
|
||||
self,
|
||||
request: EngineCoreRequest,
|
||||
prompt: Optional[str],
|
||||
parent_req: Optional[ParentRequest],
|
||||
index: int,
|
||||
queue: RequestOutputCollector,
|
||||
):
|
||||
# Add the request to OutputProcessor (this process).
|
||||
self.output_processor.add_request(request, prompt, parent_req, index,
|
||||
queue)
|
||||
self.output_processor.add_request(request, prompt, parent_req, index, queue)
|
||||
|
||||
# Add the EngineCoreRequest to EngineCore (separate process).
|
||||
await self.engine_core.add_request_async(request)
|
||||
@@ -366,12 +376,15 @@ class AsyncLLM(EngineClient):
|
||||
returning the RequestOutput back to the caller.
|
||||
"""
|
||||
|
||||
if (self.vllm_config.cache_config.kv_sharing_fast_prefill
|
||||
and sampling_params.prompt_logprobs):
|
||||
if (
|
||||
self.vllm_config.cache_config.kv_sharing_fast_prefill
|
||||
and sampling_params.prompt_logprobs
|
||||
):
|
||||
raise ValueError(
|
||||
"--kv-sharing-fast-prefill produces incorrect logprobs for "
|
||||
"prompt tokens, please disable it when the requests need "
|
||||
"prompt logprobs")
|
||||
"prompt logprobs"
|
||||
)
|
||||
|
||||
try:
|
||||
# We start the output_handler on the first call to generate() so
|
||||
@@ -389,15 +402,17 @@ class AsyncLLM(EngineClient):
|
||||
tokenization_kwargs,
|
||||
)
|
||||
|
||||
q = await self.add_request(request_id,
|
||||
prompt,
|
||||
sampling_params,
|
||||
lora_request=lora_request,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
trace_headers=trace_headers,
|
||||
priority=priority,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
prompt_text=prompt_text)
|
||||
q = await self.add_request(
|
||||
request_id,
|
||||
prompt,
|
||||
sampling_params,
|
||||
lora_request=lora_request,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
trace_headers=trace_headers,
|
||||
priority=priority,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
prompt_text=prompt_text,
|
||||
)
|
||||
|
||||
# The output_handler task pushes items into the queue.
|
||||
# This task pulls from the queue and yields to caller.
|
||||
@@ -460,23 +475,26 @@ class AsyncLLM(EngineClient):
|
||||
outputs = await engine_core.get_output_async()
|
||||
num_outputs = len(outputs.outputs)
|
||||
|
||||
iteration_stats = IterationStats() if (
|
||||
log_stats and num_outputs) else None
|
||||
iteration_stats = (
|
||||
IterationStats() if (log_stats and num_outputs) else None
|
||||
)
|
||||
|
||||
# Split outputs into chunks of at most
|
||||
# VLLM_V1_OUTPUT_PROC_CHUNK_SIZE, so that we don't block the
|
||||
# event loop for too long.
|
||||
if num_outputs <= VLLM_V1_OUTPUT_PROC_CHUNK_SIZE:
|
||||
slices = (outputs.outputs, )
|
||||
slices = (outputs.outputs,)
|
||||
else:
|
||||
slices = np.array_split(
|
||||
outputs.outputs,
|
||||
cdiv(num_outputs, VLLM_V1_OUTPUT_PROC_CHUNK_SIZE))
|
||||
cdiv(num_outputs, VLLM_V1_OUTPUT_PROC_CHUNK_SIZE),
|
||||
)
|
||||
|
||||
for i, outputs_slice in enumerate(slices):
|
||||
# 2) Process EngineCoreOutputs.
|
||||
processed_outputs = output_processor.process_outputs(
|
||||
outputs_slice, outputs.timestamp, iteration_stats)
|
||||
outputs_slice, outputs.timestamp, iteration_stats
|
||||
)
|
||||
# NOTE: RequestOutputs are pushed to their queues.
|
||||
assert not processed_outputs.request_outputs
|
||||
|
||||
@@ -486,7 +504,8 @@ class AsyncLLM(EngineClient):
|
||||
|
||||
# 3) Abort any reqs that finished due to stop strings.
|
||||
await engine_core.abort_requests_async(
|
||||
processed_outputs.reqs_to_abort)
|
||||
processed_outputs.reqs_to_abort
|
||||
)
|
||||
|
||||
# 4) Logging.
|
||||
# TODO(rob): make into a coroutine and launch it in
|
||||
@@ -506,8 +525,9 @@ class AsyncLLM(EngineClient):
|
||||
async def abort(self, request_id: Union[str, Iterable[str]]) -> None:
|
||||
"""Abort RequestId in OutputProcessor and EngineCore."""
|
||||
|
||||
request_ids = (request_id, ) if isinstance(
|
||||
request_id, str) else as_list(request_id)
|
||||
request_ids = (
|
||||
(request_id,) if isinstance(request_id, str) else as_list(request_id)
|
||||
)
|
||||
all_request_ids = self.output_processor.abort_requests(request_ids)
|
||||
await self.engine_core.abort_requests_async(all_request_ids)
|
||||
|
||||
@@ -614,8 +634,9 @@ class AsyncLLM(EngineClient):
|
||||
|
||||
async def get_tokenizer(self) -> AnyTokenizer:
|
||||
if self.tokenizer is None:
|
||||
raise ValueError("Unable to get tokenizer because "
|
||||
"skip_tokenizer_init is True")
|
||||
raise ValueError(
|
||||
"Unable to get tokenizer because skip_tokenizer_init is True"
|
||||
)
|
||||
|
||||
return self.tokenizer
|
||||
|
||||
@@ -647,8 +668,7 @@ class AsyncLLM(EngineClient):
|
||||
self.processor.clear_cache()
|
||||
await self.engine_core.reset_mm_cache_async()
|
||||
|
||||
async def reset_prefix_cache(self,
|
||||
device: Optional[Device] = None) -> None:
|
||||
async def reset_prefix_cache(self, device: Optional[Device] = None) -> None:
|
||||
if device == Device.CPU:
|
||||
raise ValueError("Not supported on CPU.")
|
||||
await self.engine_core.reset_prefix_cache_async()
|
||||
@@ -679,16 +699,19 @@ class AsyncLLM(EngineClient):
|
||||
"""Prevent an adapter from being evicted."""
|
||||
return await self.engine_core.pin_lora_async(lora_id)
|
||||
|
||||
async def collective_rpc(self,
|
||||
method: str,
|
||||
timeout: Optional[float] = None,
|
||||
args: tuple = (),
|
||||
kwargs: Optional[dict] = None):
|
||||
async def collective_rpc(
|
||||
self,
|
||||
method: str,
|
||||
timeout: Optional[float] = None,
|
||||
args: tuple = (),
|
||||
kwargs: Optional[dict] = None,
|
||||
):
|
||||
"""
|
||||
Perform a collective RPC call to the given path.
|
||||
"""
|
||||
return await self.engine_core.collective_rpc_async(
|
||||
method, timeout, args, kwargs)
|
||||
method, timeout, args, kwargs
|
||||
)
|
||||
|
||||
async def wait_for_requests_to_drain(self, drain_timeout: int = 300):
|
||||
"""Wait for all requests to be drained."""
|
||||
@@ -698,16 +721,17 @@ class AsyncLLM(EngineClient):
|
||||
logger.info("Engines are idle, requests have been drained")
|
||||
return
|
||||
|
||||
logger.info(
|
||||
"Engines are still running, waiting for requests to drain...")
|
||||
logger.info("Engines are still running, waiting for requests to drain...")
|
||||
await asyncio.sleep(1) # Wait 1 second before checking again
|
||||
|
||||
raise TimeoutError(f"Timeout reached after {drain_timeout} seconds "
|
||||
"waiting for requests to drain.")
|
||||
raise TimeoutError(
|
||||
f"Timeout reached after {drain_timeout} seconds "
|
||||
"waiting for requests to drain."
|
||||
)
|
||||
|
||||
async def scale_elastic_ep(self,
|
||||
new_data_parallel_size: int,
|
||||
drain_timeout: int = 300):
|
||||
async def scale_elastic_ep(
|
||||
self, new_data_parallel_size: int, drain_timeout: int = 300
|
||||
):
|
||||
"""
|
||||
Scale up or down the data parallel size by adding or removing
|
||||
engine cores.
|
||||
@@ -716,22 +740,24 @@ class AsyncLLM(EngineClient):
|
||||
drain_timeout:
|
||||
Maximum time to wait for requests to drain (seconds)
|
||||
"""
|
||||
old_data_parallel_size = \
|
||||
self.vllm_config.parallel_config.data_parallel_size
|
||||
old_data_parallel_size = self.vllm_config.parallel_config.data_parallel_size
|
||||
if old_data_parallel_size == new_data_parallel_size:
|
||||
logger.info("Data parallel size is already %s, skipping scale",
|
||||
new_data_parallel_size)
|
||||
logger.info(
|
||||
"Data parallel size is already %s, skipping scale",
|
||||
new_data_parallel_size,
|
||||
)
|
||||
return
|
||||
logger.info(
|
||||
"Waiting for requests to drain before "
|
||||
"scaling up to %s engines...", new_data_parallel_size)
|
||||
"Waiting for requests to drain before scaling up to %s engines...",
|
||||
new_data_parallel_size,
|
||||
)
|
||||
await self.wait_for_requests_to_drain(drain_timeout)
|
||||
logger.info(
|
||||
"Requests have been drained, proceeding with scale "
|
||||
"to %s engines", new_data_parallel_size)
|
||||
"Requests have been drained, proceeding with scale to %s engines",
|
||||
new_data_parallel_size,
|
||||
)
|
||||
await self.engine_core.scale_elastic_ep(new_data_parallel_size)
|
||||
self.vllm_config.parallel_config.data_parallel_size = \
|
||||
new_data_parallel_size
|
||||
self.vllm_config.parallel_config.data_parallel_size = new_data_parallel_size
|
||||
|
||||
# recreate stat loggers
|
||||
if new_data_parallel_size > old_data_parallel_size and self.log_stats:
|
||||
|
||||
Reference in New Issue
Block a user