Convert formatting to use ruff instead of yapf + isort (#26247)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
+32
-27
@@ -32,6 +32,7 @@ class FinishReason(enum.IntEnum):
|
||||
abort - aborted for another reason
|
||||
|
||||
"""
|
||||
|
||||
STOP = 0
|
||||
LENGTH = 1
|
||||
ABORT = 2
|
||||
@@ -41,11 +42,11 @@ class FinishReason(enum.IntEnum):
|
||||
|
||||
|
||||
class EngineCoreRequest(
|
||||
msgspec.Struct,
|
||||
array_like=True, # type: ignore[call-arg]
|
||||
omit_defaults=True, # type: ignore[call-arg]
|
||||
gc=False): # type: ignore[call-arg]
|
||||
|
||||
msgspec.Struct,
|
||||
array_like=True, # type: ignore[call-arg]
|
||||
omit_defaults=True, # type: ignore[call-arg]
|
||||
gc=False,
|
||||
): # type: ignore[call-arg]
|
||||
request_id: str
|
||||
prompt_token_ids: Optional[list[int]]
|
||||
mm_features: Optional[list[MultiModalFeatureSpec]]
|
||||
@@ -73,6 +74,7 @@ class EngineCoreRequest(
|
||||
|
||||
class EngineCoreEventType(enum.IntEnum):
|
||||
"""The type of engine core request event."""
|
||||
|
||||
QUEUED = 1
|
||||
SCHEDULED = 2
|
||||
PREEMPTED = 3
|
||||
@@ -85,23 +87,24 @@ class EngineCoreEvent(msgspec.Struct):
|
||||
frontend to calculate intervals between engine core events. These
|
||||
timestamps should not be compared with timestamps from other processes.
|
||||
"""
|
||||
|
||||
type: EngineCoreEventType
|
||||
timestamp: float
|
||||
|
||||
@classmethod
|
||||
def new_event(cls,
|
||||
event_type: EngineCoreEventType,
|
||||
timestamp: Optional[float] = None) -> "EngineCoreEvent":
|
||||
def new_event(
|
||||
cls, event_type: EngineCoreEventType, timestamp: Optional[float] = None
|
||||
) -> "EngineCoreEvent":
|
||||
timestamp = time.monotonic() if timestamp is None else timestamp
|
||||
return cls(event_type, timestamp)
|
||||
|
||||
|
||||
class EngineCoreOutput(
|
||||
msgspec.Struct,
|
||||
array_like=True, # type: ignore[call-arg]
|
||||
omit_defaults=True, # type: ignore[call-arg]
|
||||
gc=False): # type: ignore[call-arg]
|
||||
|
||||
msgspec.Struct,
|
||||
array_like=True, # type: ignore[call-arg]
|
||||
omit_defaults=True, # type: ignore[call-arg]
|
||||
gc=False,
|
||||
): # type: ignore[call-arg]
|
||||
request_id: str
|
||||
new_token_ids: list[int]
|
||||
|
||||
@@ -132,10 +135,10 @@ class UtilityResult:
|
||||
|
||||
|
||||
class UtilityOutput(
|
||||
msgspec.Struct,
|
||||
array_like=True, # type: ignore[call-arg]
|
||||
gc=False): # type: ignore[call-arg]
|
||||
|
||||
msgspec.Struct,
|
||||
array_like=True, # type: ignore[call-arg]
|
||||
gc=False,
|
||||
): # type: ignore[call-arg]
|
||||
call_id: int
|
||||
|
||||
# Non-None implies the call failed, result should be None.
|
||||
@@ -144,11 +147,11 @@ class UtilityOutput(
|
||||
|
||||
|
||||
class EngineCoreOutputs(
|
||||
msgspec.Struct,
|
||||
array_like=True, # type: ignore[call-arg]
|
||||
omit_defaults=True, # type: ignore[call-arg]
|
||||
gc=False): # type: ignore[call-arg]
|
||||
|
||||
msgspec.Struct,
|
||||
array_like=True, # type: ignore[call-arg]
|
||||
omit_defaults=True, # type: ignore[call-arg]
|
||||
gc=False,
|
||||
): # type: ignore[call-arg]
|
||||
# NOTE(Nick): We could consider ways to make this more compact,
|
||||
# e.g. columnwise layout
|
||||
|
||||
@@ -179,12 +182,13 @@ class EngineCoreRequestType(enum.Enum):
|
||||
Request types defined as hex byte strings, so it can be sent over sockets
|
||||
without separate encoding step.
|
||||
"""
|
||||
ADD = b'\x00'
|
||||
ABORT = b'\x01'
|
||||
START_DP_WAVE = b'\x02'
|
||||
UTILITY = b'\x03'
|
||||
|
||||
ADD = b"\x00"
|
||||
ABORT = b"\x01"
|
||||
START_DP_WAVE = b"\x02"
|
||||
UTILITY = b"\x03"
|
||||
# Sentinel used within EngineCoreProc.
|
||||
EXECUTOR_FAILED = b'\x04'
|
||||
EXECUTOR_FAILED = b"\x04"
|
||||
|
||||
|
||||
class ReconfigureDistributedRequest(msgspec.Struct):
|
||||
@@ -199,5 +203,6 @@ class ReconfigureRankType(enum.IntEnum):
|
||||
"""
|
||||
Rank type for reconfiguring distributed request.
|
||||
"""
|
||||
|
||||
KEEP_CURRENT_RANK = -1
|
||||
SHUTDOWN_CURRENT_RANK = -2
|
||||
|
||||
+126
-100
@@ -27,18 +27,14 @@ from vllm.pooling_params import PoolingParams
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.tasks import SupportedTask
|
||||
from vllm.tracing import init_tracer
|
||||
from vllm.transformers_utils.config import (
|
||||
maybe_register_config_serialize_by_value)
|
||||
from vllm.transformers_utils.tokenizer import (AnyTokenizer,
|
||||
init_tokenizer_from_configs)
|
||||
from vllm.transformers_utils.config import maybe_register_config_serialize_by_value
|
||||
from vllm.transformers_utils.tokenizer import AnyTokenizer, init_tokenizer_from_configs
|
||||
from vllm.usage.usage_lib import UsageContext
|
||||
from vllm.utils import (Device, as_list, cancel_task_threadsafe, cdiv,
|
||||
deprecate_kwargs)
|
||||
from vllm.utils import Device, as_list, cancel_task_threadsafe, cdiv, deprecate_kwargs
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm.v1.engine.core_client import EngineCoreClient
|
||||
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
|
||||
from vllm.v1.engine.output_processor import (OutputProcessor,
|
||||
RequestOutputCollector)
|
||||
from vllm.v1.engine.output_processor import OutputProcessor, RequestOutputCollector
|
||||
from vllm.v1.engine.parallel_sampling import ParentRequest
|
||||
from vllm.v1.engine.processor import Processor
|
||||
from vllm.v1.executor.abstract import Executor
|
||||
@@ -50,7 +46,6 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class AsyncLLM(EngineClient):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
@@ -91,7 +86,8 @@ class AsyncLLM(EngineClient):
|
||||
"Using V1 AsyncLLMEngine, but envs.VLLM_USE_V1=False. "
|
||||
"This should not happen. As a workaround, try using "
|
||||
"AsyncLLMEngine.from_vllm_config(...) or explicitly set "
|
||||
"VLLM_USE_V1=0 or 1 and report this issue on Github.")
|
||||
"VLLM_USE_V1=0 or 1 and report this issue on Github."
|
||||
)
|
||||
|
||||
# Ensure we can serialize custom transformer configs
|
||||
maybe_register_config_serialize_by_value()
|
||||
@@ -105,14 +101,16 @@ class AsyncLLM(EngineClient):
|
||||
if not log_stats and stat_loggers is not None:
|
||||
logger.info(
|
||||
"AsyncLLM created with log_stats=False and non-empty custom "
|
||||
"logger list; enabling logging without default stat loggers")
|
||||
"logger list; enabling logging without default stat loggers"
|
||||
)
|
||||
|
||||
if self.model_config.skip_tokenizer_init:
|
||||
self.tokenizer = None
|
||||
else:
|
||||
# Tokenizer (+ ensure liveness if running in another process).
|
||||
self.tokenizer = init_tokenizer_from_configs(
|
||||
model_config=vllm_config.model_config)
|
||||
model_config=vllm_config.model_config
|
||||
)
|
||||
|
||||
# Processor (converts Inputs --> EngineCoreRequests).
|
||||
self.processor = Processor(
|
||||
@@ -122,12 +120,13 @@ class AsyncLLM(EngineClient):
|
||||
)
|
||||
|
||||
# OutputProcessor (converts EngineCoreOutputs --> RequestOutput).
|
||||
self.output_processor = OutputProcessor(self.tokenizer,
|
||||
log_stats=self.log_stats)
|
||||
self.output_processor = OutputProcessor(
|
||||
self.tokenizer, log_stats=self.log_stats
|
||||
)
|
||||
if self.observability_config.otlp_traces_endpoint is not None:
|
||||
tracer = init_tracer(
|
||||
"vllm.llm_engine",
|
||||
self.observability_config.otlp_traces_endpoint)
|
||||
"vllm.llm_engine", self.observability_config.otlp_traces_endpoint
|
||||
)
|
||||
self.output_processor.tracer = tracer
|
||||
|
||||
# EngineCore (starts the engine in background process).
|
||||
@@ -163,7 +162,8 @@ class AsyncLLM(EngineClient):
|
||||
if envs.VLLM_TORCH_PROFILER_DIR:
|
||||
logger.info(
|
||||
"Torch profiler enabled. AsyncLLM CPU traces will be collected under %s", # noqa: E501
|
||||
envs.VLLM_TORCH_PROFILER_DIR)
|
||||
envs.VLLM_TORCH_PROFILER_DIR,
|
||||
)
|
||||
worker_name = f"{socket.gethostname()}_{os.getpid()}.async_llm"
|
||||
self.profiler = torch.profiler.profile(
|
||||
activities=[
|
||||
@@ -171,37 +171,39 @@ class AsyncLLM(EngineClient):
|
||||
],
|
||||
with_stack=envs.VLLM_TORCH_PROFILER_WITH_STACK,
|
||||
on_trace_ready=torch.profiler.tensorboard_trace_handler(
|
||||
envs.VLLM_TORCH_PROFILER_DIR,
|
||||
worker_name=worker_name,
|
||||
use_gzip=True))
|
||||
envs.VLLM_TORCH_PROFILER_DIR, worker_name=worker_name, use_gzip=True
|
||||
),
|
||||
)
|
||||
else:
|
||||
self.profiler = None
|
||||
|
||||
@classmethod
|
||||
@deprecate_kwargs(
|
||||
"disable_log_requests",
|
||||
additional_message=("This argument will have no effect. "
|
||||
"Use `enable_log_requests` instead."),
|
||||
additional_message=(
|
||||
"This argument will have no effect. Use `enable_log_requests` instead."
|
||||
),
|
||||
)
|
||||
def from_vllm_config(
|
||||
cls,
|
||||
vllm_config: VllmConfig,
|
||||
start_engine_loop: bool = True,
|
||||
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
|
||||
stat_loggers: Optional[list[StatLoggerFactory]] = None,
|
||||
enable_log_requests: bool = False,
|
||||
disable_log_stats: bool = False,
|
||||
client_addresses: Optional[dict[str, str]] = None,
|
||||
client_count: int = 1,
|
||||
client_index: int = 0,
|
||||
disable_log_requests: bool = True, # Deprecated, will be removed
|
||||
cls,
|
||||
vllm_config: VllmConfig,
|
||||
start_engine_loop: bool = True,
|
||||
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
|
||||
stat_loggers: Optional[list[StatLoggerFactory]] = None,
|
||||
enable_log_requests: bool = False,
|
||||
disable_log_stats: bool = False,
|
||||
client_addresses: Optional[dict[str, str]] = None,
|
||||
client_count: int = 1,
|
||||
client_index: int = 0,
|
||||
disable_log_requests: bool = True, # Deprecated, will be removed
|
||||
) -> "AsyncLLM":
|
||||
if not envs.VLLM_USE_V1:
|
||||
raise ValueError(
|
||||
"Using V1 AsyncLLMEngine, but envs.VLLM_USE_V1=False. "
|
||||
"This should not happen. As a workaround, try using "
|
||||
"AsyncLLMEngine.from_vllm_config(...) or explicitly set "
|
||||
"VLLM_USE_V1=0 or 1 and report this issue on Github.")
|
||||
"VLLM_USE_V1=0 or 1 and report this issue on Github."
|
||||
)
|
||||
|
||||
# Create the LLMEngine.
|
||||
return cls(
|
||||
@@ -288,14 +290,20 @@ class AsyncLLM(EngineClient):
|
||||
assert prompt_text is None
|
||||
logger.warning_once(
|
||||
"Processor has been moved under OpenAIServing and will "
|
||||
"be removed from AsyncLLM in v0.13.")
|
||||
request = self.processor.process_inputs(request_id, prompt, params,
|
||||
arrival_time, lora_request,
|
||||
tokenization_kwargs,
|
||||
trace_headers, priority,
|
||||
data_parallel_rank)
|
||||
prompt_text = (prompt if isinstance(prompt, str) else
|
||||
prompt.get("prompt"))
|
||||
"be removed from AsyncLLM in v0.13."
|
||||
)
|
||||
request = self.processor.process_inputs(
|
||||
request_id,
|
||||
prompt,
|
||||
params,
|
||||
arrival_time,
|
||||
lora_request,
|
||||
tokenization_kwargs,
|
||||
trace_headers,
|
||||
priority,
|
||||
data_parallel_rank,
|
||||
)
|
||||
prompt_text = prompt if isinstance(prompt, str) else prompt.get("prompt")
|
||||
|
||||
if is_pooling or params.n == 1:
|
||||
await self._add_request(request, prompt_text, None, 0, queue)
|
||||
@@ -310,22 +318,24 @@ class AsyncLLM(EngineClient):
|
||||
parent_request = ParentRequest(request_id, parent_params)
|
||||
for idx in range(parent_params.n):
|
||||
request_id, child_params = parent_request.get_child_info(idx)
|
||||
child_request = request if idx == parent_params.n - 1 else copy(
|
||||
request)
|
||||
child_request = request if idx == parent_params.n - 1 else copy(request)
|
||||
child_request.request_id = request_id
|
||||
child_request.sampling_params = child_params
|
||||
await self._add_request(child_request, prompt_text, parent_request,
|
||||
idx, queue)
|
||||
await self._add_request(
|
||||
child_request, prompt_text, parent_request, idx, queue
|
||||
)
|
||||
return queue
|
||||
|
||||
async def _add_request(self, request: EngineCoreRequest,
|
||||
prompt: Optional[str],
|
||||
parent_req: Optional[ParentRequest], index: int,
|
||||
queue: RequestOutputCollector):
|
||||
|
||||
async def _add_request(
|
||||
self,
|
||||
request: EngineCoreRequest,
|
||||
prompt: Optional[str],
|
||||
parent_req: Optional[ParentRequest],
|
||||
index: int,
|
||||
queue: RequestOutputCollector,
|
||||
):
|
||||
# Add the request to OutputProcessor (this process).
|
||||
self.output_processor.add_request(request, prompt, parent_req, index,
|
||||
queue)
|
||||
self.output_processor.add_request(request, prompt, parent_req, index, queue)
|
||||
|
||||
# Add the EngineCoreRequest to EngineCore (separate process).
|
||||
await self.engine_core.add_request_async(request)
|
||||
@@ -366,12 +376,15 @@ class AsyncLLM(EngineClient):
|
||||
returning the RequestOutput back to the caller.
|
||||
"""
|
||||
|
||||
if (self.vllm_config.cache_config.kv_sharing_fast_prefill
|
||||
and sampling_params.prompt_logprobs):
|
||||
if (
|
||||
self.vllm_config.cache_config.kv_sharing_fast_prefill
|
||||
and sampling_params.prompt_logprobs
|
||||
):
|
||||
raise ValueError(
|
||||
"--kv-sharing-fast-prefill produces incorrect logprobs for "
|
||||
"prompt tokens, please disable it when the requests need "
|
||||
"prompt logprobs")
|
||||
"prompt logprobs"
|
||||
)
|
||||
|
||||
try:
|
||||
# We start the output_handler on the first call to generate() so
|
||||
@@ -389,15 +402,17 @@ class AsyncLLM(EngineClient):
|
||||
tokenization_kwargs,
|
||||
)
|
||||
|
||||
q = await self.add_request(request_id,
|
||||
prompt,
|
||||
sampling_params,
|
||||
lora_request=lora_request,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
trace_headers=trace_headers,
|
||||
priority=priority,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
prompt_text=prompt_text)
|
||||
q = await self.add_request(
|
||||
request_id,
|
||||
prompt,
|
||||
sampling_params,
|
||||
lora_request=lora_request,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
trace_headers=trace_headers,
|
||||
priority=priority,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
prompt_text=prompt_text,
|
||||
)
|
||||
|
||||
# The output_handler task pushes items into the queue.
|
||||
# This task pulls from the queue and yields to caller.
|
||||
@@ -460,23 +475,26 @@ class AsyncLLM(EngineClient):
|
||||
outputs = await engine_core.get_output_async()
|
||||
num_outputs = len(outputs.outputs)
|
||||
|
||||
iteration_stats = IterationStats() if (
|
||||
log_stats and num_outputs) else None
|
||||
iteration_stats = (
|
||||
IterationStats() if (log_stats and num_outputs) else None
|
||||
)
|
||||
|
||||
# Split outputs into chunks of at most
|
||||
# VLLM_V1_OUTPUT_PROC_CHUNK_SIZE, so that we don't block the
|
||||
# event loop for too long.
|
||||
if num_outputs <= VLLM_V1_OUTPUT_PROC_CHUNK_SIZE:
|
||||
slices = (outputs.outputs, )
|
||||
slices = (outputs.outputs,)
|
||||
else:
|
||||
slices = np.array_split(
|
||||
outputs.outputs,
|
||||
cdiv(num_outputs, VLLM_V1_OUTPUT_PROC_CHUNK_SIZE))
|
||||
cdiv(num_outputs, VLLM_V1_OUTPUT_PROC_CHUNK_SIZE),
|
||||
)
|
||||
|
||||
for i, outputs_slice in enumerate(slices):
|
||||
# 2) Process EngineCoreOutputs.
|
||||
processed_outputs = output_processor.process_outputs(
|
||||
outputs_slice, outputs.timestamp, iteration_stats)
|
||||
outputs_slice, outputs.timestamp, iteration_stats
|
||||
)
|
||||
# NOTE: RequestOutputs are pushed to their queues.
|
||||
assert not processed_outputs.request_outputs
|
||||
|
||||
@@ -486,7 +504,8 @@ class AsyncLLM(EngineClient):
|
||||
|
||||
# 3) Abort any reqs that finished due to stop strings.
|
||||
await engine_core.abort_requests_async(
|
||||
processed_outputs.reqs_to_abort)
|
||||
processed_outputs.reqs_to_abort
|
||||
)
|
||||
|
||||
# 4) Logging.
|
||||
# TODO(rob): make into a coroutine and launch it in
|
||||
@@ -506,8 +525,9 @@ class AsyncLLM(EngineClient):
|
||||
async def abort(self, request_id: Union[str, Iterable[str]]) -> None:
|
||||
"""Abort RequestId in OutputProcessor and EngineCore."""
|
||||
|
||||
request_ids = (request_id, ) if isinstance(
|
||||
request_id, str) else as_list(request_id)
|
||||
request_ids = (
|
||||
(request_id,) if isinstance(request_id, str) else as_list(request_id)
|
||||
)
|
||||
all_request_ids = self.output_processor.abort_requests(request_ids)
|
||||
await self.engine_core.abort_requests_async(all_request_ids)
|
||||
|
||||
@@ -614,8 +634,9 @@ class AsyncLLM(EngineClient):
|
||||
|
||||
async def get_tokenizer(self) -> AnyTokenizer:
|
||||
if self.tokenizer is None:
|
||||
raise ValueError("Unable to get tokenizer because "
|
||||
"skip_tokenizer_init is True")
|
||||
raise ValueError(
|
||||
"Unable to get tokenizer because skip_tokenizer_init is True"
|
||||
)
|
||||
|
||||
return self.tokenizer
|
||||
|
||||
@@ -647,8 +668,7 @@ class AsyncLLM(EngineClient):
|
||||
self.processor.clear_cache()
|
||||
await self.engine_core.reset_mm_cache_async()
|
||||
|
||||
async def reset_prefix_cache(self,
|
||||
device: Optional[Device] = None) -> None:
|
||||
async def reset_prefix_cache(self, device: Optional[Device] = None) -> None:
|
||||
if device == Device.CPU:
|
||||
raise ValueError("Not supported on CPU.")
|
||||
await self.engine_core.reset_prefix_cache_async()
|
||||
@@ -679,16 +699,19 @@ class AsyncLLM(EngineClient):
|
||||
"""Prevent an adapter from being evicted."""
|
||||
return await self.engine_core.pin_lora_async(lora_id)
|
||||
|
||||
async def collective_rpc(self,
|
||||
method: str,
|
||||
timeout: Optional[float] = None,
|
||||
args: tuple = (),
|
||||
kwargs: Optional[dict] = None):
|
||||
async def collective_rpc(
|
||||
self,
|
||||
method: str,
|
||||
timeout: Optional[float] = None,
|
||||
args: tuple = (),
|
||||
kwargs: Optional[dict] = None,
|
||||
):
|
||||
"""
|
||||
Perform a collective RPC call to the given path.
|
||||
"""
|
||||
return await self.engine_core.collective_rpc_async(
|
||||
method, timeout, args, kwargs)
|
||||
method, timeout, args, kwargs
|
||||
)
|
||||
|
||||
async def wait_for_requests_to_drain(self, drain_timeout: int = 300):
|
||||
"""Wait for all requests to be drained."""
|
||||
@@ -698,16 +721,17 @@ class AsyncLLM(EngineClient):
|
||||
logger.info("Engines are idle, requests have been drained")
|
||||
return
|
||||
|
||||
logger.info(
|
||||
"Engines are still running, waiting for requests to drain...")
|
||||
logger.info("Engines are still running, waiting for requests to drain...")
|
||||
await asyncio.sleep(1) # Wait 1 second before checking again
|
||||
|
||||
raise TimeoutError(f"Timeout reached after {drain_timeout} seconds "
|
||||
"waiting for requests to drain.")
|
||||
raise TimeoutError(
|
||||
f"Timeout reached after {drain_timeout} seconds "
|
||||
"waiting for requests to drain."
|
||||
)
|
||||
|
||||
async def scale_elastic_ep(self,
|
||||
new_data_parallel_size: int,
|
||||
drain_timeout: int = 300):
|
||||
async def scale_elastic_ep(
|
||||
self, new_data_parallel_size: int, drain_timeout: int = 300
|
||||
):
|
||||
"""
|
||||
Scale up or down the data parallel size by adding or removing
|
||||
engine cores.
|
||||
@@ -716,22 +740,24 @@ class AsyncLLM(EngineClient):
|
||||
drain_timeout:
|
||||
Maximum time to wait for requests to drain (seconds)
|
||||
"""
|
||||
old_data_parallel_size = \
|
||||
self.vllm_config.parallel_config.data_parallel_size
|
||||
old_data_parallel_size = self.vllm_config.parallel_config.data_parallel_size
|
||||
if old_data_parallel_size == new_data_parallel_size:
|
||||
logger.info("Data parallel size is already %s, skipping scale",
|
||||
new_data_parallel_size)
|
||||
logger.info(
|
||||
"Data parallel size is already %s, skipping scale",
|
||||
new_data_parallel_size,
|
||||
)
|
||||
return
|
||||
logger.info(
|
||||
"Waiting for requests to drain before "
|
||||
"scaling up to %s engines...", new_data_parallel_size)
|
||||
"Waiting for requests to drain before scaling up to %s engines...",
|
||||
new_data_parallel_size,
|
||||
)
|
||||
await self.wait_for_requests_to_drain(drain_timeout)
|
||||
logger.info(
|
||||
"Requests have been drained, proceeding with scale "
|
||||
"to %s engines", new_data_parallel_size)
|
||||
"Requests have been drained, proceeding with scale to %s engines",
|
||||
new_data_parallel_size,
|
||||
)
|
||||
await self.engine_core.scale_elastic_ep(new_data_parallel_size)
|
||||
self.vllm_config.parallel_config.data_parallel_size = \
|
||||
new_data_parallel_size
|
||||
self.vllm_config.parallel_config.data_parallel_size = new_data_parallel_size
|
||||
|
||||
# recreate stat loggers
|
||||
if new_data_parallel_size > old_data_parallel_size and self.log_stats:
|
||||
|
||||
@@ -56,7 +56,6 @@ class DPCoordinator:
|
||||
"""
|
||||
|
||||
def __init__(self, parallel_config: ParallelConfig):
|
||||
|
||||
dp_size = parallel_config.data_parallel_size
|
||||
assert dp_size > 1, "Coordinator only used for data parallel"
|
||||
|
||||
@@ -68,7 +67,8 @@ class DPCoordinator:
|
||||
# either external or hybrid DP LB mode.
|
||||
local_only = not (external_lb or hybrid_lb)
|
||||
front_publish_address = get_engine_client_zmq_addr(
|
||||
local_only=local_only, host=host)
|
||||
local_only=local_only, host=host
|
||||
)
|
||||
|
||||
local_only_eng = dp_size == parallel_config.data_parallel_size_local
|
||||
back_publish_address = get_engine_client_zmq_addr(local_only_eng, host)
|
||||
@@ -84,7 +84,8 @@ class DPCoordinator:
|
||||
"back_output_address": back_output_address,
|
||||
"back_publish_address": back_publish_address,
|
||||
},
|
||||
daemon=True)
|
||||
daemon=True,
|
||||
)
|
||||
self.proc.start()
|
||||
|
||||
self.stats_publish_address = front_publish_address
|
||||
@@ -104,16 +105,12 @@ class DPCoordinator:
|
||||
|
||||
|
||||
class EngineState:
|
||||
|
||||
def __init__(self):
|
||||
self.request_counts = [0, 0] # [waiting, running]
|
||||
|
||||
|
||||
class DPCoordinatorProc:
|
||||
|
||||
def __init__(self,
|
||||
engine_count: int,
|
||||
min_stats_update_interval_ms: int = 100):
|
||||
def __init__(self, engine_count: int, min_stats_update_interval_ms: int = 100):
|
||||
set_process_title("DPCoordinator")
|
||||
self.ctx = zmq.Context()
|
||||
|
||||
@@ -131,7 +128,8 @@ class DPCoordinatorProc:
|
||||
):
|
||||
coordinator = DPCoordinatorProc(
|
||||
engine_count=engine_count,
|
||||
min_stats_update_interval_ms=min_stats_update_interval_ms)
|
||||
min_stats_update_interval_ms=min_stats_update_interval_ms,
|
||||
)
|
||||
try:
|
||||
coordinator.process_input_socket(
|
||||
front_publish_address,
|
||||
@@ -141,10 +139,12 @@ class DPCoordinatorProc:
|
||||
except KeyboardInterrupt:
|
||||
logger.info("DP Coordinator process exiting")
|
||||
|
||||
def process_input_socket(self, front_publish_address: str,
|
||||
back_output_address: str,
|
||||
back_publish_address: str):
|
||||
|
||||
def process_input_socket(
|
||||
self,
|
||||
front_publish_address: str,
|
||||
back_output_address: str,
|
||||
back_publish_address: str,
|
||||
):
|
||||
decoder = MsgpackDecoder(EngineCoreOutputs)
|
||||
|
||||
# For tracking request wave progression.
|
||||
@@ -157,29 +157,33 @@ class DPCoordinatorProc:
|
||||
last_stats_wave = -1
|
||||
last_step_counts: Optional[list[list[int]]] = None
|
||||
|
||||
with make_zmq_socket(
|
||||
with (
|
||||
make_zmq_socket(
|
||||
path=front_publish_address, # IPC
|
||||
ctx=self.ctx,
|
||||
socket_type=zmq.XPUB,
|
||||
bind=True,
|
||||
) as publish_front, make_zmq_socket(
|
||||
) as publish_front,
|
||||
make_zmq_socket(
|
||||
path=back_output_address, # IPC or TCP
|
||||
ctx=self.ctx,
|
||||
socket_type=zmq.PULL,
|
||||
bind=True,
|
||||
) as output_back, make_zmq_socket(
|
||||
) as output_back,
|
||||
make_zmq_socket(
|
||||
path=back_publish_address, # IPC or TCP
|
||||
ctx=self.ctx,
|
||||
socket_type=zmq.XPUB,
|
||||
bind=True,
|
||||
) as publish_back:
|
||||
|
||||
) as publish_back,
|
||||
):
|
||||
# Wait until all engines subscribe.
|
||||
for _ in self.engines:
|
||||
if publish_back.recv() != b'\x01':
|
||||
if publish_back.recv() != b"\x01":
|
||||
logger.error(
|
||||
"DP Coordinator received unexpected message while "
|
||||
"waiting for engines to subscribe")
|
||||
"waiting for engines to subscribe"
|
||||
)
|
||||
return
|
||||
# Send ready message to engines.
|
||||
publish_back.send(b"READY")
|
||||
@@ -194,15 +198,13 @@ class DPCoordinatorProc:
|
||||
elapsed = int(time.time() * 1000) - last_publish_time
|
||||
# Send at stats_update_interval_ms interval if the stats have
|
||||
# changed, or otherwise every 5 seconds.
|
||||
wait_for = (self.stats_update_interval_ms
|
||||
if stats_changed else 5000)
|
||||
wait_for = self.stats_update_interval_ms if stats_changed else 5000
|
||||
|
||||
# Wait at least 50ms to ensure we've received all stats for
|
||||
# the current step.
|
||||
min_timeout = 50 if last_step_counts is None else 0
|
||||
|
||||
events = poller.poll(timeout=max(min_timeout, wait_for -
|
||||
elapsed))
|
||||
events = poller.poll(timeout=max(min_timeout, wait_for - elapsed))
|
||||
if not events:
|
||||
# Poller timeout - publish current stats to front-ends.
|
||||
if last_step_counts is not None:
|
||||
@@ -212,8 +214,7 @@ class DPCoordinatorProc:
|
||||
engine_req_counts_list = self._get_engine_counts()
|
||||
stats_changed = False
|
||||
|
||||
to_publish = (engine_req_counts_list, current_wave,
|
||||
engines_running)
|
||||
to_publish = (engine_req_counts_list, current_wave, engines_running)
|
||||
publish_front.send(msgspec.msgpack.encode(to_publish))
|
||||
last_publish_time = int(time.time() * 1000)
|
||||
continue
|
||||
@@ -223,13 +224,16 @@ class DPCoordinatorProc:
|
||||
|
||||
if publish_front in events:
|
||||
buffer = publish_front.recv()
|
||||
if buffer in (b'\x01', b'\x00'):
|
||||
if buffer in (b"\x01", b"\x00"):
|
||||
# Ignore subscription messages.
|
||||
continue
|
||||
|
||||
decoded = msgspec.msgpack.decode(buffer)
|
||||
if isinstance(decoded, (list, tuple)) and len(
|
||||
decoded) == 2 and decoded[0] == "SCALE_ELASTIC_EP":
|
||||
if (
|
||||
isinstance(decoded, (list, tuple))
|
||||
and len(decoded) == 2
|
||||
and decoded[0] == "SCALE_ELASTIC_EP"
|
||||
):
|
||||
# Handle scale up notification
|
||||
new_engine_count = decoded[1]
|
||||
current_count = len(self.engines)
|
||||
@@ -248,13 +252,17 @@ class DPCoordinatorProc:
|
||||
# engine
|
||||
engines_running = False
|
||||
logger.info(
|
||||
"DPCoordinator scaled up from %s to %s "
|
||||
"engines", current_count, new_engine_count)
|
||||
"DPCoordinator scaled up from %s to %s engines",
|
||||
current_count,
|
||||
new_engine_count,
|
||||
)
|
||||
else:
|
||||
self.engines = self.engines[:new_engine_count]
|
||||
logger.info(
|
||||
"DPCoordinator scaled down from %s to %s "
|
||||
"engines", current_count, new_engine_count)
|
||||
"DPCoordinator scaled down from %s to %s engines",
|
||||
current_count,
|
||||
new_engine_count,
|
||||
)
|
||||
continue # Skip normal engine notification processing
|
||||
|
||||
# We received a message on the front-end XPUB socket,
|
||||
@@ -270,8 +278,9 @@ class DPCoordinatorProc:
|
||||
|
||||
engines_running = True
|
||||
wave_state_changed = True
|
||||
self._send_start_wave(publish_back, current_wave,
|
||||
engine_to_exclude)
|
||||
self._send_start_wave(
|
||||
publish_back, current_wave, engine_to_exclude
|
||||
)
|
||||
|
||||
if output_back in events:
|
||||
# We received a message from one of the engines.
|
||||
@@ -290,21 +299,28 @@ class DPCoordinatorProc:
|
||||
stats = self.engines[eng_index].request_counts
|
||||
stats_step = scheduler_stats.step_counter
|
||||
stats_wave = scheduler_stats.current_wave
|
||||
if (stats_wave > last_stats_wave
|
||||
or stats_wave == last_stats_wave
|
||||
and stats_step > last_stats_step):
|
||||
if (
|
||||
stats_wave > last_stats_wave
|
||||
or stats_wave == last_stats_wave
|
||||
and stats_step > last_stats_step
|
||||
):
|
||||
if stats_changed:
|
||||
last_step_counts = self._get_engine_counts(
|
||||
do_copy=True)
|
||||
last_step_counts = self._get_engine_counts(do_copy=True)
|
||||
last_stats_step = stats_step
|
||||
last_stats_wave = stats_wave
|
||||
elif stats_wave != last_stats_wave or (
|
||||
stats_step != last_stats_step):
|
||||
stats_step != last_stats_step
|
||||
):
|
||||
logger.warning(
|
||||
"Received stats for out-of-order "
|
||||
"step (%d, %d) from engine %d (expected "
|
||||
"> (%d, %d))", stats_wave, stats_step,
|
||||
eng_index, last_stats_wave, last_stats_step)
|
||||
"> (%d, %d))",
|
||||
stats_wave,
|
||||
stats_step,
|
||||
eng_index,
|
||||
last_stats_wave,
|
||||
last_stats_step,
|
||||
)
|
||||
stats[0] = scheduler_stats.num_waiting_reqs
|
||||
stats[1] = scheduler_stats.num_running_reqs
|
||||
stats_changed = True
|
||||
@@ -315,20 +331,24 @@ class DPCoordinatorProc:
|
||||
# (engines_running==False).
|
||||
if current_wave <= wave:
|
||||
new_wave = wave + 1
|
||||
logger.debug("Moving DP wave from %d to %d.",
|
||||
current_wave, new_wave)
|
||||
logger.debug(
|
||||
"Moving DP wave from %d to %d.", current_wave, new_wave
|
||||
)
|
||||
current_wave = new_wave
|
||||
engines_running = False
|
||||
wave_state_changed = True
|
||||
elif (wave := outputs.start_wave) is not None and (
|
||||
wave > current_wave or
|
||||
(wave == current_wave and not engines_running)):
|
||||
wave > current_wave
|
||||
or (wave == current_wave and not engines_running)
|
||||
):
|
||||
# 3. The engine received request for a non-current wave
|
||||
# so we must ensure that other engines progress to the
|
||||
# next wave (race condition handling).
|
||||
logger.debug(
|
||||
"Starting wave %d after notification of "
|
||||
"stale wave request from engine.", wave)
|
||||
"stale wave request from engine.",
|
||||
wave,
|
||||
)
|
||||
current_wave = wave
|
||||
engines_running = True
|
||||
wave_state_changed = True
|
||||
@@ -339,16 +359,16 @@ class DPCoordinatorProc:
|
||||
publish_front.send(msgspec.msgpack.encode(message))
|
||||
|
||||
@staticmethod
|
||||
def _send_start_wave(socket: zmq.Socket, wave: int,
|
||||
exclude_engine_index: Optional[int]):
|
||||
def _send_start_wave(
|
||||
socket: zmq.Socket, wave: int, exclude_engine_index: Optional[int]
|
||||
):
|
||||
"""Broadcast the START_DP_WAVE message to all the engines.
|
||||
It includes the current wave number and index of engine which
|
||||
has already received a request with this wave number and so doesn't
|
||||
require additional notification.
|
||||
"""
|
||||
wave_encoded = msgspec.msgpack.encode((wave, exclude_engine_index))
|
||||
socket.send_multipart(
|
||||
(EngineCoreRequestType.START_DP_WAVE.value, wave_encoded))
|
||||
socket.send_multipart((EngineCoreRequestType.START_DP_WAVE.value, wave_encoded))
|
||||
|
||||
def _get_engine_counts(self, do_copy=False) -> list[list[int]]:
|
||||
"""Return list of [waiting, running] count lists for each engine."""
|
||||
|
||||
+390
-274
File diff suppressed because it is too large
Load Diff
+343
-275
File diff suppressed because it is too large
Load Diff
@@ -11,7 +11,10 @@ from transformers import PreTrainedTokenizerFast
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.transformers_utils.detokenizer_utils import (
|
||||
AnyTokenizer, convert_prompt_ids_to_tokens, detokenize_incrementally)
|
||||
AnyTokenizer,
|
||||
convert_prompt_ids_to_tokens,
|
||||
detokenize_incrementally,
|
||||
)
|
||||
from vllm.utils import length_from_prompt_token_ids_or_embeds
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
|
||||
@@ -19,15 +22,13 @@ logger = init_logger(__name__)
|
||||
|
||||
# Only tokenizers >= 0.21.1 supports DecodeStream used for
|
||||
# FastIncrementalDetokenizer.
|
||||
USE_FAST_DETOKENIZER = version.parse(
|
||||
tokenizers.__version__) >= version.parse("0.21.1")
|
||||
USE_FAST_DETOKENIZER = version.parse(tokenizers.__version__) >= version.parse("0.21.1")
|
||||
|
||||
# Error string from https://github.com/huggingface/tokenizers/blob/909fdde2a4ffedd9295206f705eb612be2a91b12/tokenizers/src/tokenizer/mod.rs#L1042
|
||||
INVALID_PREFIX_ERR_MSG = "Invalid prefix encountered"
|
||||
|
||||
|
||||
class IncrementalDetokenizer:
|
||||
|
||||
def __init__(self):
|
||||
self.token_ids: list[int] = []
|
||||
|
||||
@@ -35,8 +36,7 @@ class IncrementalDetokenizer:
|
||||
def output_token_ids(self) -> list[int]:
|
||||
return self.token_ids
|
||||
|
||||
def update(self, new_token_ids: list[int],
|
||||
stop_terminated: bool) -> Optional[str]:
|
||||
def update(self, new_token_ids: list[int], stop_terminated: bool) -> Optional[str]:
|
||||
self.token_ids.extend(new_token_ids)
|
||||
return None
|
||||
|
||||
@@ -49,15 +49,13 @@ class IncrementalDetokenizer:
|
||||
tokenizer: Optional[AnyTokenizer],
|
||||
request: EngineCoreRequest,
|
||||
) -> "IncrementalDetokenizer":
|
||||
|
||||
assert request.sampling_params is not None
|
||||
|
||||
if tokenizer is None:
|
||||
# No tokenizer => skipping detokenization.
|
||||
return IncrementalDetokenizer()
|
||||
|
||||
if USE_FAST_DETOKENIZER and isinstance(tokenizer,
|
||||
PreTrainedTokenizerFast):
|
||||
if USE_FAST_DETOKENIZER and isinstance(tokenizer, PreTrainedTokenizerFast):
|
||||
# Fast tokenizer => use tokenizers library DecodeStream.
|
||||
return FastIncrementalDetokenizer(tokenizer, request)
|
||||
|
||||
@@ -66,7 +64,6 @@ class IncrementalDetokenizer:
|
||||
|
||||
|
||||
class BaseIncrementalDetokenizer(IncrementalDetokenizer, ABC):
|
||||
|
||||
def __init__(self, request: EngineCoreRequest):
|
||||
super().__init__()
|
||||
|
||||
@@ -88,8 +85,7 @@ class BaseIncrementalDetokenizer(IncrementalDetokenizer, ABC):
|
||||
# Generation data
|
||||
self.output_text = ""
|
||||
|
||||
def update(self, new_token_ids: list[int],
|
||||
stop_terminated: bool) -> Optional[str]:
|
||||
def update(self, new_token_ids: list[int], stop_terminated: bool) -> Optional[str]:
|
||||
"""
|
||||
Update RequestState for the request_id by:
|
||||
1) Detokenize the new token ids incrementally.
|
||||
@@ -117,8 +113,7 @@ class BaseIncrementalDetokenizer(IncrementalDetokenizer, ABC):
|
||||
self.token_ids.append(new_token_id)
|
||||
self.output_text += self.decode_next(new_token_id)
|
||||
# Support min_tokens, see https://github.com/vllm-project/vllm/pull/22014
|
||||
if self.min_tokens and len(
|
||||
self.output_token_ids) <= self.min_tokens:
|
||||
if self.min_tokens and len(self.output_token_ids) <= self.min_tokens:
|
||||
stop_check_offset = len(self.output_text)
|
||||
|
||||
if skipped_stop_token_id is not None:
|
||||
@@ -152,8 +147,11 @@ class BaseIncrementalDetokenizer(IncrementalDetokenizer, ABC):
|
||||
# We return the full output text if the sequence is finished.
|
||||
buffer_length = 0 if finished else self.stop_buffer_length
|
||||
if not delta:
|
||||
return self.output_text[:-buffer_length] if buffer_length else (
|
||||
self.output_text)
|
||||
return (
|
||||
self.output_text[:-buffer_length]
|
||||
if buffer_length
|
||||
else (self.output_text)
|
||||
)
|
||||
length = len(self.output_text) - buffer_length
|
||||
last_offset = self._last_output_text_offset
|
||||
if last_offset < length:
|
||||
@@ -163,9 +161,7 @@ class BaseIncrementalDetokenizer(IncrementalDetokenizer, ABC):
|
||||
|
||||
|
||||
class FastIncrementalDetokenizer(BaseIncrementalDetokenizer):
|
||||
|
||||
def __init__(self, tokenizer: PreTrainedTokenizerFast,
|
||||
request: EngineCoreRequest):
|
||||
def __init__(self, tokenizer: PreTrainedTokenizerFast, request: EngineCoreRequest):
|
||||
super().__init__(request)
|
||||
|
||||
sampling_params = request.sampling_params
|
||||
@@ -173,8 +169,7 @@ class FastIncrementalDetokenizer(BaseIncrementalDetokenizer):
|
||||
|
||||
self.request_id = request.request_id
|
||||
self.skip_special_tokens = sampling_params.skip_special_tokens
|
||||
self.stream = DecodeStream(
|
||||
skip_special_tokens=self.skip_special_tokens)
|
||||
self.stream = DecodeStream(skip_special_tokens=self.skip_special_tokens)
|
||||
|
||||
self.tokenizer: Tokenizer = tokenizer._tokenizer
|
||||
|
||||
@@ -185,7 +180,7 @@ class FastIncrementalDetokenizer(BaseIncrementalDetokenizer):
|
||||
if prompt_len > 4:
|
||||
for i in range(4, min(prompt_len + 1, 24)):
|
||||
suffix = prompt_token_ids[-i:]
|
||||
if '�' not in self.tokenizer.decode(suffix):
|
||||
if "�" not in self.tokenizer.decode(suffix):
|
||||
prompt_suffix = suffix
|
||||
break
|
||||
|
||||
@@ -195,17 +190,18 @@ class FastIncrementalDetokenizer(BaseIncrementalDetokenizer):
|
||||
|
||||
self.spaces_between_special_tokens = (
|
||||
sampling_params.skip_special_tokens
|
||||
or sampling_params.spaces_between_special_tokens)
|
||||
or sampling_params.spaces_between_special_tokens
|
||||
)
|
||||
|
||||
if not self.spaces_between_special_tokens:
|
||||
# Store dict of added token ids so that we can suppress
|
||||
# the spaces between them.
|
||||
if (added_token_ids := getattr(self.tokenizer, "added_token_ids",
|
||||
None)) is None:
|
||||
if (
|
||||
added_token_ids := getattr(self.tokenizer, "added_token_ids", None)
|
||||
) is None:
|
||||
self.tokenizer.added_token_ids = added_token_ids = {
|
||||
tid: tok.content
|
||||
for tid, tok in
|
||||
self.tokenizer.get_added_tokens_decoder().items()
|
||||
for tid, tok in self.tokenizer.get_added_tokens_decoder().items()
|
||||
}
|
||||
|
||||
if added_token_ids:
|
||||
@@ -245,15 +241,15 @@ class FastIncrementalDetokenizer(BaseIncrementalDetokenizer):
|
||||
# See https://github.com/vllm-project/vllm/issues/17448.
|
||||
logger.warning(
|
||||
"Encountered invalid prefix detokenization error"
|
||||
" for request %s, resetting decode stream.", self.request_id)
|
||||
self.stream = DecodeStream(
|
||||
skip_special_tokens=self.skip_special_tokens)
|
||||
" for request %s, resetting decode stream.",
|
||||
self.request_id,
|
||||
)
|
||||
self.stream = DecodeStream(skip_special_tokens=self.skip_special_tokens)
|
||||
token = self.stream.step(self.tokenizer, next_token_id)
|
||||
return token
|
||||
|
||||
|
||||
class SlowIncrementalDetokenizer(BaseIncrementalDetokenizer):
|
||||
|
||||
def __init__(self, tokenizer: AnyTokenizer, request: EngineCoreRequest):
|
||||
super().__init__(request)
|
||||
|
||||
@@ -262,7 +258,8 @@ class SlowIncrementalDetokenizer(BaseIncrementalDetokenizer):
|
||||
assert params is not None
|
||||
|
||||
self.prompt_len = length_from_prompt_token_ids_or_embeds(
|
||||
request.prompt_token_ids, request.prompt_embeds)
|
||||
request.prompt_token_ids, request.prompt_embeds
|
||||
)
|
||||
|
||||
# Metadata for incremental detokenization.
|
||||
if request.prompt_token_ids is not None:
|
||||
@@ -271,37 +268,37 @@ class SlowIncrementalDetokenizer(BaseIncrementalDetokenizer):
|
||||
tokenizer=tokenizer,
|
||||
prompt_ids=request.prompt_token_ids,
|
||||
skip_special_tokens=params.skip_special_tokens,
|
||||
))
|
||||
)
|
||||
)
|
||||
else:
|
||||
# Prompt embedding requests cannot be detokenized, in general.
|
||||
self.tokens = [""] * self.prompt_len
|
||||
self.prefix_offset = 0
|
||||
self.read_offest = 0
|
||||
|
||||
self.token_ids.extend(request.prompt_token_ids
|
||||
or [0] * self.prompt_len)
|
||||
self.token_ids.extend(request.prompt_token_ids or [0] * self.prompt_len)
|
||||
|
||||
self.skip_special_tokens = params.skip_special_tokens
|
||||
self.spaces_between_special_tokens = (
|
||||
params.spaces_between_special_tokens)
|
||||
self.spaces_between_special_tokens = params.spaces_between_special_tokens
|
||||
|
||||
@property
|
||||
def output_token_ids(self) -> list[int]:
|
||||
return self.token_ids if not self.prompt_len else (
|
||||
self.token_ids[self.prompt_len:])
|
||||
return (
|
||||
self.token_ids
|
||||
if not self.prompt_len
|
||||
else (self.token_ids[self.prompt_len :])
|
||||
)
|
||||
|
||||
def decode_next(self, next_token_id: int) -> str:
|
||||
new_tokens, decoded_text, prefix_offset, read_offset = (
|
||||
detokenize_incrementally(
|
||||
tokenizer=self.tokenizer,
|
||||
all_input_ids=self.token_ids,
|
||||
prev_tokens=self.tokens,
|
||||
prefix_offset=self.prefix_offset,
|
||||
read_offset=self.read_offset,
|
||||
skip_special_tokens=self.skip_special_tokens,
|
||||
spaces_between_special_tokens=self.
|
||||
spaces_between_special_tokens,
|
||||
))
|
||||
new_tokens, decoded_text, prefix_offset, read_offset = detokenize_incrementally(
|
||||
tokenizer=self.tokenizer,
|
||||
all_input_ids=self.token_ids,
|
||||
prev_tokens=self.tokens,
|
||||
prefix_offset=self.prefix_offset,
|
||||
read_offset=self.read_offset,
|
||||
skip_special_tokens=self.skip_special_tokens,
|
||||
spaces_between_special_tokens=self.spaces_between_special_tokens,
|
||||
)
|
||||
|
||||
self.tokens.extend(new_tokens)
|
||||
self.prefix_offset = prefix_offset
|
||||
@@ -331,8 +328,7 @@ def check_stop_strings(
|
||||
for stop_str in stop:
|
||||
stop_string_len = len(stop_str)
|
||||
# Avoid searching already-searched text.
|
||||
stop_index = output_text.find(stop_str,
|
||||
1 - new_char_count - stop_string_len)
|
||||
stop_index = output_text.find(stop_str, 1 - new_char_count - stop_string_len)
|
||||
if stop_index == -1:
|
||||
continue
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
class EngineGenerateError(Exception):
|
||||
"""Raised when a AsyncLLM.generate() fails. Recoverable."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
|
||||
@@ -23,8 +23,7 @@ from vllm.pooling_params import PoolingParams
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.tasks import SupportedTask
|
||||
from vllm.tracing import init_tracer
|
||||
from vllm.transformers_utils.tokenizer import (AnyTokenizer,
|
||||
init_tokenizer_from_configs)
|
||||
from vllm.transformers_utils.tokenizer import AnyTokenizer, init_tokenizer_from_configs
|
||||
from vllm.usage.usage_lib import UsageContext
|
||||
from vllm.utils import Device
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
@@ -62,12 +61,14 @@ class LLMEngine:
|
||||
"Using V1 LLMEngine, but envs.VLLM_USE_V1=False. "
|
||||
"This should not happen. As a workaround, try using "
|
||||
"LLMEngine.from_vllm_config(...) or explicitly set "
|
||||
"VLLM_USE_V1=0 or 1 and report this issue on Github.")
|
||||
"VLLM_USE_V1=0 or 1 and report this issue on Github."
|
||||
)
|
||||
|
||||
if stat_loggers is not None:
|
||||
raise NotImplementedError(
|
||||
"Passing StatLoggers to LLMEngine in V1 is not yet supported. "
|
||||
"Set VLLM_USE_V1=0 and file and issue on Github.")
|
||||
"Set VLLM_USE_V1=0 and file and issue on Github."
|
||||
)
|
||||
|
||||
self.vllm_config = vllm_config
|
||||
self.observability_config = vllm_config.observability_config
|
||||
@@ -76,15 +77,19 @@ class LLMEngine:
|
||||
|
||||
self.log_stats = log_stats
|
||||
|
||||
executor_backend = (
|
||||
self.vllm_config.parallel_config.distributed_executor_backend)
|
||||
executor_backend = self.vllm_config.parallel_config.distributed_executor_backend
|
||||
parallel_config = vllm_config.parallel_config
|
||||
self.external_launcher_dp = (parallel_config.data_parallel_size > 1 and
|
||||
executor_backend == "external_launcher")
|
||||
self.external_launcher_dp = (
|
||||
parallel_config.data_parallel_size > 1
|
||||
and executor_backend == "external_launcher"
|
||||
)
|
||||
# important: init dp group before init the engine_core
|
||||
# In the decoupled engine case this is handled in EngineCoreProc.
|
||||
if not multiprocess_mode and parallel_config.data_parallel_size > 1 \
|
||||
and not self.external_launcher_dp:
|
||||
if (
|
||||
not multiprocess_mode
|
||||
and parallel_config.data_parallel_size > 1
|
||||
and not self.external_launcher_dp
|
||||
):
|
||||
self.dp_group = parallel_config.stateless_init_dp_group()
|
||||
else:
|
||||
self.dp_group = None
|
||||
@@ -95,20 +100,22 @@ class LLMEngine:
|
||||
else:
|
||||
# Tokenizer (+ ensure liveness if running in another process).
|
||||
self.tokenizer = init_tokenizer_from_configs(
|
||||
model_config=vllm_config.model_config)
|
||||
model_config=vllm_config.model_config
|
||||
)
|
||||
|
||||
# Processor (convert Inputs --> EngineCoreRequests)
|
||||
self.processor = Processor(vllm_config=vllm_config,
|
||||
tokenizer=self.tokenizer,
|
||||
mm_registry=mm_registry)
|
||||
self.processor = Processor(
|
||||
vllm_config=vllm_config, tokenizer=self.tokenizer, mm_registry=mm_registry
|
||||
)
|
||||
|
||||
# OutputProcessor (convert EngineCoreOutputs --> RequestOutput).
|
||||
self.output_processor = OutputProcessor(self.tokenizer,
|
||||
log_stats=self.log_stats)
|
||||
self.output_processor = OutputProcessor(
|
||||
self.tokenizer, log_stats=self.log_stats
|
||||
)
|
||||
if self.observability_config.otlp_traces_endpoint is not None:
|
||||
tracer = init_tracer(
|
||||
"vllm.llm_engine",
|
||||
self.observability_config.otlp_traces_endpoint)
|
||||
"vllm.llm_engine", self.observability_config.otlp_traces_endpoint
|
||||
)
|
||||
self.output_processor.tracer = tracer
|
||||
|
||||
# EngineCore (gets EngineCoreRequests and gives EngineCoreOutputs)
|
||||
@@ -149,12 +156,14 @@ class LLMEngine:
|
||||
stat_loggers: Optional[list[StatLoggerFactory]] = None,
|
||||
disable_log_stats: bool = False,
|
||||
) -> "LLMEngine":
|
||||
return cls(vllm_config=vllm_config,
|
||||
executor_class=Executor.get_class(vllm_config),
|
||||
log_stats=(not disable_log_stats),
|
||||
usage_context=usage_context,
|
||||
stat_loggers=stat_loggers,
|
||||
multiprocess_mode=envs.VLLM_ENABLE_V1_MULTIPROCESSING)
|
||||
return cls(
|
||||
vllm_config=vllm_config,
|
||||
executor_class=Executor.get_class(vllm_config),
|
||||
log_stats=(not disable_log_stats),
|
||||
usage_context=usage_context,
|
||||
stat_loggers=stat_loggers,
|
||||
multiprocess_mode=envs.VLLM_ENABLE_V1_MULTIPROCESSING,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_engine_args(
|
||||
@@ -175,12 +184,14 @@ class LLMEngine:
|
||||
enable_multiprocessing = True
|
||||
|
||||
# Create the LLMEngine.
|
||||
return cls(vllm_config=vllm_config,
|
||||
executor_class=executor_class,
|
||||
log_stats=not engine_args.disable_log_stats,
|
||||
usage_context=usage_context,
|
||||
stat_loggers=stat_loggers,
|
||||
multiprocess_mode=enable_multiprocessing)
|
||||
return cls(
|
||||
vllm_config=vllm_config,
|
||||
executor_class=executor_class,
|
||||
log_stats=not engine_args.disable_log_stats,
|
||||
usage_context=usage_context,
|
||||
stat_loggers=stat_loggers,
|
||||
multiprocess_mode=enable_multiprocessing,
|
||||
)
|
||||
|
||||
def get_num_unfinished_requests(self) -> int:
|
||||
return self.output_processor.get_num_unfinished_requests()
|
||||
@@ -193,7 +204,8 @@ class LLMEngine:
|
||||
|
||||
def has_unfinished_requests_dp(self, has_unfinished: bool) -> bool:
|
||||
aggregated_has_unfinished = ParallelConfig.has_unfinished_dp(
|
||||
self.dp_group, has_unfinished)
|
||||
self.dp_group, has_unfinished
|
||||
)
|
||||
if not has_unfinished and aggregated_has_unfinished:
|
||||
self.should_execute_dummy_batch = True
|
||||
return aggregated_has_unfinished
|
||||
@@ -225,22 +237,28 @@ class LLMEngine:
|
||||
) -> None:
|
||||
# Validate the request_id type.
|
||||
if not isinstance(request_id, str):
|
||||
raise TypeError(
|
||||
f"request_id must be a string, got {type(request_id)}")
|
||||
raise TypeError(f"request_id must be a string, got {type(request_id)}")
|
||||
|
||||
# Process raw inputs into the request.
|
||||
if isinstance(prompt, EngineCoreRequest):
|
||||
request = prompt
|
||||
else:
|
||||
assert prompt_text is None
|
||||
logger.warning_once("Processor has been moved under LLM and will "
|
||||
"be removed from LLMEngine in v0.13.")
|
||||
request = self.processor.process_inputs(request_id, prompt, params,
|
||||
arrival_time, lora_request,
|
||||
tokenization_kwargs,
|
||||
trace_headers, priority)
|
||||
prompt_text = (prompt if isinstance(prompt, str) else
|
||||
prompt.get("prompt"))
|
||||
logger.warning_once(
|
||||
"Processor has been moved under LLM and will "
|
||||
"be removed from LLMEngine in v0.13."
|
||||
)
|
||||
request = self.processor.process_inputs(
|
||||
request_id,
|
||||
prompt,
|
||||
params,
|
||||
arrival_time,
|
||||
lora_request,
|
||||
tokenization_kwargs,
|
||||
trace_headers,
|
||||
priority,
|
||||
)
|
||||
prompt_text = prompt if isinstance(prompt, str) else prompt.get("prompt")
|
||||
|
||||
n = params.n if isinstance(params, SamplingParams) else 1
|
||||
|
||||
@@ -260,13 +278,13 @@ class LLMEngine:
|
||||
child_request.sampling_params = params
|
||||
|
||||
# Make a new RequestState and queue.
|
||||
self.output_processor.add_request(child_request, prompt_text,
|
||||
parent_req, idx)
|
||||
self.output_processor.add_request(
|
||||
child_request, prompt_text, parent_req, idx
|
||||
)
|
||||
# Add the request to EngineCore.
|
||||
self.engine_core.add_request(child_request)
|
||||
|
||||
def step(self) -> Union[list[RequestOutput], list[PoolingRequestOutput]]:
|
||||
|
||||
if self.should_execute_dummy_batch:
|
||||
self.should_execute_dummy_batch = False
|
||||
self.engine_core.execute_dummy_batch()
|
||||
@@ -280,7 +298,8 @@ class LLMEngine:
|
||||
processed_outputs = self.output_processor.process_outputs(
|
||||
outputs.outputs,
|
||||
engine_core_timestamp=outputs.timestamp,
|
||||
iteration_stats=iteration_stats)
|
||||
iteration_stats=iteration_stats,
|
||||
)
|
||||
|
||||
# 3) Abort any reqs that finished due to stop strings.
|
||||
self.engine_core.abort_requests(processed_outputs.reqs_to_abort)
|
||||
@@ -330,8 +349,9 @@ class LLMEngine:
|
||||
|
||||
def get_tokenizer(self) -> AnyTokenizer:
|
||||
if self.tokenizer is None:
|
||||
raise ValueError("Unable to get tokenizer because "
|
||||
"skip_tokenizer_init is True")
|
||||
raise ValueError(
|
||||
"Unable to get tokenizer because skip_tokenizer_init is True"
|
||||
)
|
||||
|
||||
return self.tokenizer
|
||||
|
||||
@@ -365,17 +385,21 @@ class LLMEngine:
|
||||
"""Prevent an adapter from being evicted."""
|
||||
return self.engine_core.pin_lora(lora_id)
|
||||
|
||||
def collective_rpc(self,
|
||||
method: Union[str, Callable[[WorkerBase], _R]],
|
||||
timeout: Optional[float] = None,
|
||||
args: tuple = (),
|
||||
kwargs: Optional[dict[str, Any]] = None) -> list[_R]:
|
||||
def collective_rpc(
|
||||
self,
|
||||
method: Union[str, Callable[[WorkerBase], _R]],
|
||||
timeout: Optional[float] = None,
|
||||
args: tuple = (),
|
||||
kwargs: Optional[dict[str, Any]] = None,
|
||||
) -> list[_R]:
|
||||
return self.engine_core.collective_rpc(method, timeout, args, kwargs)
|
||||
|
||||
def apply_model(self, func: Callable[[nn.Module], _R]) -> list[_R]:
|
||||
return self.collective_rpc("apply_model", args=(func, ))
|
||||
return self.collective_rpc("apply_model", args=(func,))
|
||||
|
||||
def __del__(self):
|
||||
if dp_group := getattr(self, "dp_group",
|
||||
None) and not self.external_launcher_dp:
|
||||
if (
|
||||
dp_group := getattr(self, "dp_group", None)
|
||||
and not self.external_launcher_dp
|
||||
):
|
||||
stateless_destroy_torch_distributed_process_group(dp_group)
|
||||
|
||||
+33
-20
@@ -9,7 +9,9 @@ from typing import Optional
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob, PromptLogprobs, SampleLogprobs
|
||||
from vllm.transformers_utils.detokenizer_utils import (
|
||||
AnyTokenizer, convert_ids_list_to_tokens)
|
||||
AnyTokenizer,
|
||||
convert_ids_list_to_tokens,
|
||||
)
|
||||
from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest
|
||||
from vllm.v1.outputs import LogprobsLists, LogprobsTensors
|
||||
|
||||
@@ -20,7 +22,6 @@ NONES = itertools.repeat(None)
|
||||
|
||||
@dataclass
|
||||
class LogprobsProcessor:
|
||||
|
||||
# Tokenizer for this request,
|
||||
# None if detokenization is disabled.
|
||||
tokenizer: Optional[AnyTokenizer]
|
||||
@@ -43,7 +44,7 @@ class LogprobsProcessor:
|
||||
num_prompt_logprobs = request.sampling_params.prompt_logprobs
|
||||
return cls(
|
||||
tokenizer=tokenizer,
|
||||
cumulative_logprob=(None if num_logprobs is None else 0.),
|
||||
cumulative_logprob=(None if num_logprobs is None else 0.0),
|
||||
logprobs=(None if num_logprobs is None else []),
|
||||
# NOTE: logprob of first prompt token is None.
|
||||
prompt_logprobs=(None if num_prompt_logprobs is None else [None]),
|
||||
@@ -68,12 +69,13 @@ class LogprobsProcessor:
|
||||
|
||||
token_ids_lst, logprobs_lst, ranks_lst = logprobs_lists
|
||||
|
||||
for rank, logprobs, token_ids in zip(ranks_lst, logprobs_lst,
|
||||
token_ids_lst):
|
||||
|
||||
for rank, logprobs, token_ids in zip(ranks_lst, logprobs_lst, token_ids_lst):
|
||||
# Detokenize (non-incrementally).
|
||||
decoded_tokens = NONES if self.tokenizer is None else (
|
||||
convert_ids_list_to_tokens(self.tokenizer, token_ids))
|
||||
decoded_tokens = (
|
||||
NONES
|
||||
if self.tokenizer is None
|
||||
else (convert_ids_list_to_tokens(self.tokenizer, token_ids))
|
||||
)
|
||||
|
||||
# Sampler puts the sampled logprob in first.
|
||||
sampled_token_logprob = logprobs[0]
|
||||
@@ -87,7 +89,8 @@ class LogprobsProcessor:
|
||||
decoded_tokens,
|
||||
rank,
|
||||
self.num_logprobs,
|
||||
))
|
||||
)
|
||||
)
|
||||
|
||||
def _update_prompt_logprobs(
|
||||
self,
|
||||
@@ -109,9 +112,13 @@ class LogprobsProcessor:
|
||||
|
||||
# Detokenize non-incrementally.
|
||||
# Output is flat: [num_tok, num_lps] -> [num_tok * num_lps]
|
||||
decoded_tokens = None if self.tokenizer is None else (
|
||||
convert_ids_list_to_tokens(self.tokenizer,
|
||||
token_ids.flatten().tolist()))
|
||||
decoded_tokens = (
|
||||
None
|
||||
if self.tokenizer is None
|
||||
else (
|
||||
convert_ids_list_to_tokens(self.tokenizer, token_ids.flatten().tolist())
|
||||
)
|
||||
)
|
||||
|
||||
# Recover shapes.
|
||||
num_prompt_tokens, num_logprobs = logprobs.shape
|
||||
@@ -126,15 +133,20 @@ class LogprobsProcessor:
|
||||
# Handle flattening.
|
||||
offset = pos * num_logprobs
|
||||
offset_end = offset + num_logprobs
|
||||
decoded_tokens_for_pos = NONES \
|
||||
if decoded_tokens is None else decoded_tokens[offset:offset_end]
|
||||
decoded_tokens_for_pos = (
|
||||
NONES if decoded_tokens is None else decoded_tokens[offset:offset_end]
|
||||
)
|
||||
|
||||
# Update with the Logprob dictionary for this pos.
|
||||
self.prompt_logprobs.append(
|
||||
self._make_logprob_dict(prompt_logprobs[pos], token_ids[pos],
|
||||
decoded_tokens_for_pos,
|
||||
prompt_token_ranks[pos],
|
||||
self.num_prompt_logprobs))
|
||||
self._make_logprob_dict(
|
||||
prompt_logprobs[pos],
|
||||
token_ids[pos],
|
||||
decoded_tokens_for_pos,
|
||||
prompt_token_ranks[pos],
|
||||
self.num_prompt_logprobs,
|
||||
)
|
||||
)
|
||||
|
||||
def pop_prompt_logprobs(self) -> Optional[PromptLogprobs]:
|
||||
"""Pop and return all request prompt logprobs
|
||||
@@ -182,7 +194,7 @@ class LogprobsProcessor:
|
||||
# being in the topk, since inserting duplicated data
|
||||
# into a dictionary twice is the same as doing it once.
|
||||
topk_ranks = range(1, num_logprobs + 1)
|
||||
ranks = itertools.chain((rank, ), topk_ranks)
|
||||
ranks = itertools.chain((rank,), topk_ranks)
|
||||
|
||||
return {
|
||||
token_id: Logprob(
|
||||
@@ -191,7 +203,8 @@ class LogprobsProcessor:
|
||||
decoded_token=token,
|
||||
)
|
||||
for token_id, logprob, rank, token in zip(
|
||||
logprob_token_ids, logprobs, ranks, decoded_tokens)
|
||||
logprob_token_ids, logprobs, ranks, decoded_tokens
|
||||
)
|
||||
}
|
||||
|
||||
def update_from_output(self, output: EngineCoreOutput) -> None:
|
||||
|
||||
+146
-121
@@ -8,19 +8,21 @@ from typing import Any, Optional, Union, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.outputs import (CompletionOutput, PoolingOutput,
|
||||
PoolingRequestOutput, RequestOutput)
|
||||
from vllm.outputs import (
|
||||
CompletionOutput,
|
||||
PoolingOutput,
|
||||
PoolingRequestOutput,
|
||||
RequestOutput,
|
||||
)
|
||||
from vllm.sampling_params import RequestOutputKind
|
||||
from vllm.tracing import (SpanAttributes, SpanKind, Tracer,
|
||||
extract_trace_context)
|
||||
from vllm.tracing import SpanAttributes, SpanKind, Tracer, extract_trace_context
|
||||
from vllm.transformers_utils.tokenizer import AnyTokenizer
|
||||
from vllm.utils import length_from_prompt_token_ids_or_embeds
|
||||
from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest, FinishReason
|
||||
from vllm.v1.engine.detokenizer import IncrementalDetokenizer
|
||||
from vllm.v1.engine.logprobs import LogprobsProcessor
|
||||
from vllm.v1.engine.parallel_sampling import ParentRequest
|
||||
from vllm.v1.metrics.stats import (IterationStats, LoRARequestStates,
|
||||
RequestStateStats)
|
||||
from vllm.v1.metrics.stats import IterationStats, LoRARequestStates, RequestStateStats
|
||||
|
||||
|
||||
class RequestOutputCollector:
|
||||
@@ -34,12 +36,14 @@ class RequestOutputCollector:
|
||||
|
||||
def __init__(self, output_kind: RequestOutputKind):
|
||||
self.aggregate = output_kind == RequestOutputKind.DELTA
|
||||
self.output: Optional[Union[RequestOutput, PoolingRequestOutput,
|
||||
Exception]] = None
|
||||
self.output: Optional[Union[RequestOutput, PoolingRequestOutput, Exception]] = (
|
||||
None
|
||||
)
|
||||
self.ready = asyncio.Event()
|
||||
|
||||
def put(self, output: Union[RequestOutput, PoolingRequestOutput,
|
||||
Exception]) -> None:
|
||||
def put(
|
||||
self, output: Union[RequestOutput, PoolingRequestOutput, Exception]
|
||||
) -> None:
|
||||
"""Non-blocking put operation."""
|
||||
if self.output is None or isinstance(output, Exception):
|
||||
self.output = output
|
||||
@@ -59,8 +63,7 @@ class RequestOutputCollector:
|
||||
raise output
|
||||
return output
|
||||
|
||||
def get_nowait(
|
||||
self) -> Optional[Union[RequestOutput, PoolingRequestOutput]]:
|
||||
def get_nowait(self) -> Optional[Union[RequestOutput, PoolingRequestOutput]]:
|
||||
"""Non-blocking get operation."""
|
||||
output = self.output
|
||||
if output is not None:
|
||||
@@ -78,7 +81,6 @@ class OutputProcessorOutput:
|
||||
|
||||
|
||||
class RequestState:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
request_id: str,
|
||||
@@ -108,7 +110,8 @@ class RequestState:
|
||||
self.prompt_token_ids = prompt_token_ids
|
||||
self.prompt_embeds = prompt_embeds
|
||||
self.prompt_len = length_from_prompt_token_ids_or_embeds(
|
||||
self.prompt_token_ids, self.prompt_embeds)
|
||||
self.prompt_token_ids, self.prompt_embeds
|
||||
)
|
||||
self.logprobs_processor = logprobs_processor
|
||||
self.detokenizer = detokenizer
|
||||
self.max_tokens_param = max_tokens_param
|
||||
@@ -119,8 +122,7 @@ class RequestState:
|
||||
self.queue = queue
|
||||
self.num_cached_tokens = 0
|
||||
|
||||
self.stats = RequestStateStats(
|
||||
arrival_time=arrival_time) if log_stats else None
|
||||
self.stats = RequestStateStats(arrival_time=arrival_time) if log_stats else None
|
||||
|
||||
@classmethod
|
||||
def from_new_request(
|
||||
@@ -133,7 +135,6 @@ class RequestState:
|
||||
queue: Optional[RequestOutputCollector],
|
||||
log_stats: bool,
|
||||
) -> "RequestState":
|
||||
|
||||
if sampling_params := request.sampling_params:
|
||||
if not sampling_params.detokenize:
|
||||
tokenizer = None
|
||||
@@ -164,8 +165,9 @@ class RequestState:
|
||||
request_id=request.request_id,
|
||||
parent_req=parent_req,
|
||||
request_index=request_index,
|
||||
lora_name=(request.lora_request.name
|
||||
if request.lora_request is not None else None),
|
||||
lora_name=(
|
||||
request.lora_request.name if request.lora_request is not None else None
|
||||
),
|
||||
output_kind=output_kind,
|
||||
prompt=prompt,
|
||||
prompt_token_ids=request.prompt_token_ids,
|
||||
@@ -189,7 +191,6 @@ class RequestState:
|
||||
stop_reason: Union[int, str, None],
|
||||
kv_transfer_params: Optional[dict[str, Any]] = None,
|
||||
) -> Optional[Union[RequestOutput, PoolingRequestOutput]]:
|
||||
|
||||
finished = finish_reason is not None
|
||||
final_only = self.output_kind == RequestOutputKind.FINAL_ONLY
|
||||
|
||||
@@ -200,22 +201,23 @@ class RequestState:
|
||||
request_id = self.request_id
|
||||
if pooling_output is not None:
|
||||
return self._new_request_output(
|
||||
request_id, [self._new_pooling_output(pooling_output)],
|
||||
finished)
|
||||
request_id, [self._new_pooling_output(pooling_output)], finished
|
||||
)
|
||||
|
||||
output = self._new_completion_output(new_token_ids, finish_reason,
|
||||
stop_reason)
|
||||
output = self._new_completion_output(new_token_ids, finish_reason, stop_reason)
|
||||
|
||||
if self.parent_req is None:
|
||||
outputs = [output]
|
||||
else:
|
||||
request_id, outputs, finished = self.parent_req.get_outputs(
|
||||
request_id, output)
|
||||
request_id, output
|
||||
)
|
||||
if not outputs:
|
||||
return None
|
||||
|
||||
return self._new_request_output(request_id, outputs, finished,
|
||||
kv_transfer_params)
|
||||
return self._new_request_output(
|
||||
request_id, outputs, finished, kv_transfer_params
|
||||
)
|
||||
|
||||
def _new_request_output(
|
||||
self,
|
||||
@@ -224,7 +226,6 @@ class RequestState:
|
||||
finished: bool,
|
||||
kv_transfer_params: Optional[dict[str, Any]] = None,
|
||||
) -> Union[RequestOutput, PoolingRequestOutput]:
|
||||
|
||||
first_output = outputs[0]
|
||||
if isinstance(first_output, PoolingOutput):
|
||||
assert len(outputs) == 1
|
||||
@@ -248,15 +249,17 @@ class RequestState:
|
||||
if prompt_token_ids is None and self.prompt_embeds is not None:
|
||||
prompt_token_ids = [0] * len(self.prompt_embeds)
|
||||
|
||||
return RequestOutput(request_id=request_id,
|
||||
prompt=self.prompt,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
prompt_logprobs=prompt_logprobs,
|
||||
outputs=cast(list[CompletionOutput], outputs),
|
||||
finished=finished,
|
||||
kv_transfer_params=kv_transfer_params,
|
||||
num_cached_tokens=self.num_cached_tokens,
|
||||
metrics=self.stats)
|
||||
return RequestOutput(
|
||||
request_id=request_id,
|
||||
prompt=self.prompt,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
prompt_logprobs=prompt_logprobs,
|
||||
outputs=cast(list[CompletionOutput], outputs),
|
||||
finished=finished,
|
||||
kv_transfer_params=kv_transfer_params,
|
||||
num_cached_tokens=self.num_cached_tokens,
|
||||
metrics=self.stats,
|
||||
)
|
||||
|
||||
def _new_completion_output(
|
||||
self,
|
||||
@@ -264,7 +267,6 @@ class RequestState:
|
||||
finish_reason: Optional[FinishReason],
|
||||
stop_reason: Union[int, str, None],
|
||||
) -> CompletionOutput:
|
||||
|
||||
assert self.detokenizer is not None
|
||||
assert self.logprobs_processor is not None
|
||||
finished = finish_reason is not None
|
||||
@@ -278,7 +280,7 @@ class RequestState:
|
||||
# Prepare logprobs, based on delta mode
|
||||
logprobs = self.logprobs_processor.logprobs
|
||||
if delta and logprobs:
|
||||
logprobs = logprobs[-len(token_ids):]
|
||||
logprobs = logprobs[-len(token_ids) :]
|
||||
|
||||
return CompletionOutput(
|
||||
index=self.request_index,
|
||||
@@ -287,13 +289,13 @@ class RequestState:
|
||||
logprobs=logprobs,
|
||||
cumulative_logprob=self.logprobs_processor.cumulative_logprob,
|
||||
finish_reason=str(finish_reason) if finished else None,
|
||||
stop_reason=stop_reason if finished else None)
|
||||
stop_reason=stop_reason if finished else None,
|
||||
)
|
||||
|
||||
def _new_pooling_output(
|
||||
self,
|
||||
pooling_output: torch.Tensor,
|
||||
) -> PoolingOutput:
|
||||
|
||||
return PoolingOutput(data=pooling_output)
|
||||
|
||||
|
||||
@@ -333,15 +335,18 @@ class OutputProcessor:
|
||||
request_ids_to_abort.append(request_id)
|
||||
# Produce final abort output.
|
||||
if req_state.queue is not None and (
|
||||
request_output := req_state.make_request_output(
|
||||
new_token_ids=[],
|
||||
# Set pooling_output is not None to
|
||||
# correctly enter the abort pooling branch
|
||||
pooling_output=torch.randn(0, device="cpu")
|
||||
if req_state.detokenizer is None else None,
|
||||
finish_reason=FinishReason.ABORT,
|
||||
stop_reason=None,
|
||||
kv_transfer_params=None)):
|
||||
request_output := req_state.make_request_output(
|
||||
new_token_ids=[],
|
||||
# Set pooling_output is not None to
|
||||
# correctly enter the abort pooling branch
|
||||
pooling_output=torch.randn(0, device="cpu")
|
||||
if req_state.detokenizer is None
|
||||
else None,
|
||||
finish_reason=FinishReason.ABORT,
|
||||
stop_reason=None,
|
||||
kv_transfer_params=None,
|
||||
)
|
||||
):
|
||||
req_state.queue.put(request_output)
|
||||
elif parent := self.parent_requests.get(request_id):
|
||||
# Abort children prior to removing the parent.
|
||||
@@ -364,13 +369,15 @@ class OutputProcessor:
|
||||
if request_id in self.request_states:
|
||||
raise ValueError(f"Request id {request_id} already running.")
|
||||
|
||||
req_state = RequestState.from_new_request(tokenizer=self.tokenizer,
|
||||
request=request,
|
||||
prompt=prompt,
|
||||
parent_req=parent_req,
|
||||
request_index=request_index,
|
||||
queue=queue,
|
||||
log_stats=self.log_stats)
|
||||
req_state = RequestState.from_new_request(
|
||||
tokenizer=self.tokenizer,
|
||||
request=request,
|
||||
prompt=prompt,
|
||||
parent_req=parent_req,
|
||||
request_index=request_index,
|
||||
queue=queue,
|
||||
log_stats=self.log_stats,
|
||||
)
|
||||
self.request_states[request_id] = req_state
|
||||
self.lora_states.add_request(req_state)
|
||||
if parent_req:
|
||||
@@ -404,8 +411,7 @@ class OutputProcessor:
|
||||
within the loop below.
|
||||
"""
|
||||
|
||||
request_outputs: Union[list[RequestOutput],
|
||||
list[PoolingRequestOutput]] = []
|
||||
request_outputs: Union[list[RequestOutput], list[PoolingRequestOutput]] = []
|
||||
reqs_to_abort: list[str] = []
|
||||
for engine_core_output in engine_core_outputs:
|
||||
req_id = engine_core_output.request_id
|
||||
@@ -415,9 +421,9 @@ class OutputProcessor:
|
||||
continue
|
||||
|
||||
# 1) Compute stats for this iteration.
|
||||
self._update_stats_from_output(req_state, engine_core_output,
|
||||
engine_core_timestamp,
|
||||
iteration_stats)
|
||||
self._update_stats_from_output(
|
||||
req_state, engine_core_output, engine_core_timestamp, iteration_stats
|
||||
)
|
||||
|
||||
new_token_ids = engine_core_output.new_token_ids
|
||||
pooling_output = engine_core_output.pooling_output
|
||||
@@ -432,20 +438,24 @@ class OutputProcessor:
|
||||
assert req_state.logprobs_processor is not None
|
||||
# 2) Detokenize the token ids into text and perform stop checks.
|
||||
stop_string = req_state.detokenizer.update(
|
||||
new_token_ids, finish_reason == FinishReason.STOP)
|
||||
new_token_ids, finish_reason == FinishReason.STOP
|
||||
)
|
||||
if stop_string:
|
||||
finish_reason = FinishReason.STOP
|
||||
stop_reason = stop_string
|
||||
|
||||
# 3) Compute sample and prompt logprobs for request,
|
||||
# if required.
|
||||
req_state.logprobs_processor.update_from_output(
|
||||
engine_core_output)
|
||||
req_state.logprobs_processor.update_from_output(engine_core_output)
|
||||
|
||||
# 4) Create and handle RequestOutput objects.
|
||||
if request_output := req_state.make_request_output(
|
||||
new_token_ids, pooling_output, finish_reason, stop_reason,
|
||||
kv_transfer_params):
|
||||
new_token_ids,
|
||||
pooling_output,
|
||||
finish_reason,
|
||||
stop_reason,
|
||||
kv_transfer_params,
|
||||
):
|
||||
if req_state.queue is not None:
|
||||
# AsyncLLM: put into queue for handling by generate().
|
||||
req_state.queue.put(request_output)
|
||||
@@ -466,11 +476,11 @@ class OutputProcessor:
|
||||
reqs_to_abort.append(req_id)
|
||||
|
||||
# Track per-request stats
|
||||
self._update_stats_from_finished(req_state, finish_reason,
|
||||
iteration_stats)
|
||||
self._update_stats_from_finished(
|
||||
req_state, finish_reason, iteration_stats
|
||||
)
|
||||
if self.tracer:
|
||||
self.do_tracing(engine_core_output, req_state,
|
||||
iteration_stats)
|
||||
self.do_tracing(engine_core_output, req_state, iteration_stats)
|
||||
self.lora_states.update_iteration_stats(iteration_stats)
|
||||
|
||||
return OutputProcessorOutput(
|
||||
@@ -478,9 +488,12 @@ class OutputProcessor:
|
||||
reqs_to_abort=reqs_to_abort,
|
||||
)
|
||||
|
||||
def do_tracing(self, engine_core_output: EngineCoreOutput,
|
||||
req_state: RequestState,
|
||||
iteration_stats: Optional[IterationStats]) -> None:
|
||||
def do_tracing(
|
||||
self,
|
||||
engine_core_output: EngineCoreOutput,
|
||||
req_state: RequestState,
|
||||
iteration_stats: Optional[IterationStats],
|
||||
) -> None:
|
||||
assert req_state.stats is not None
|
||||
assert iteration_stats is not None
|
||||
assert self.tracer is not None
|
||||
@@ -488,59 +501,63 @@ class OutputProcessor:
|
||||
arrival_time_nano_seconds = int(req_state.stats.arrival_time * 1e9)
|
||||
trace_context = extract_trace_context(engine_core_output.trace_headers)
|
||||
prompt_length = length_from_prompt_token_ids_or_embeds(
|
||||
req_state.prompt_token_ids, req_state.prompt_embeds)
|
||||
with (self.tracer.start_as_current_span(
|
||||
"llm_request",
|
||||
kind=SpanKind.SERVER,
|
||||
context=trace_context,
|
||||
start_time=arrival_time_nano_seconds) as span):
|
||||
req_state.prompt_token_ids, req_state.prompt_embeds
|
||||
)
|
||||
with self.tracer.start_as_current_span(
|
||||
"llm_request",
|
||||
kind=SpanKind.SERVER,
|
||||
context=trace_context,
|
||||
start_time=arrival_time_nano_seconds,
|
||||
) as span:
|
||||
metrics = req_state.stats
|
||||
e2e_time = iteration_stats.iteration_timestamp - \
|
||||
metrics.arrival_time
|
||||
e2e_time = iteration_stats.iteration_timestamp - metrics.arrival_time
|
||||
queued_time = metrics.scheduled_ts - metrics.queued_ts
|
||||
prefill_time = metrics.first_token_ts - metrics.scheduled_ts
|
||||
decode_time = metrics.last_token_ts - metrics.first_token_ts
|
||||
inference_time = metrics.last_token_ts - metrics.scheduled_ts
|
||||
span.set_attribute(
|
||||
SpanAttributes.GEN_AI_LATENCY_TIME_TO_FIRST_TOKEN,
|
||||
metrics.first_token_latency)
|
||||
metrics.first_token_latency,
|
||||
)
|
||||
span.set_attribute(SpanAttributes.GEN_AI_LATENCY_E2E, e2e_time)
|
||||
span.set_attribute(SpanAttributes.GEN_AI_LATENCY_TIME_IN_QUEUE,
|
||||
queued_time)
|
||||
span.set_attribute(SpanAttributes.GEN_AI_USAGE_PROMPT_TOKENS,
|
||||
prompt_length)
|
||||
span.set_attribute(SpanAttributes.GEN_AI_USAGE_COMPLETION_TOKENS,
|
||||
metrics.num_generation_tokens)
|
||||
span.set_attribute(SpanAttributes.GEN_AI_LATENCY_TIME_IN_QUEUE, queued_time)
|
||||
span.set_attribute(SpanAttributes.GEN_AI_USAGE_PROMPT_TOKENS, prompt_length)
|
||||
span.set_attribute(
|
||||
SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_PREFILL,
|
||||
prefill_time)
|
||||
SpanAttributes.GEN_AI_USAGE_COMPLETION_TOKENS,
|
||||
metrics.num_generation_tokens,
|
||||
)
|
||||
span.set_attribute(
|
||||
SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_DECODE,
|
||||
decode_time)
|
||||
SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_PREFILL, prefill_time
|
||||
)
|
||||
span.set_attribute(
|
||||
SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_INFERENCE,
|
||||
inference_time)
|
||||
SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_DECODE, decode_time
|
||||
)
|
||||
span.set_attribute(
|
||||
SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_INFERENCE, inference_time
|
||||
)
|
||||
|
||||
# meta
|
||||
span.set_attribute(SpanAttributes.GEN_AI_REQUEST_ID,
|
||||
req_state.request_id)
|
||||
span.set_attribute(SpanAttributes.GEN_AI_REQUEST_ID, req_state.request_id)
|
||||
if req_state.top_p:
|
||||
span.set_attribute(SpanAttributes.GEN_AI_REQUEST_TOP_P,
|
||||
req_state.top_p)
|
||||
span.set_attribute(SpanAttributes.GEN_AI_REQUEST_TOP_P, req_state.top_p)
|
||||
if req_state.max_tokens_param:
|
||||
span.set_attribute(SpanAttributes.GEN_AI_REQUEST_MAX_TOKENS,
|
||||
req_state.max_tokens_param)
|
||||
span.set_attribute(
|
||||
SpanAttributes.GEN_AI_REQUEST_MAX_TOKENS, req_state.max_tokens_param
|
||||
)
|
||||
if req_state.temperature:
|
||||
span.set_attribute(SpanAttributes.GEN_AI_REQUEST_TEMPERATURE,
|
||||
req_state.temperature)
|
||||
span.set_attribute(
|
||||
SpanAttributes.GEN_AI_REQUEST_TEMPERATURE, req_state.temperature
|
||||
)
|
||||
if req_state.n:
|
||||
span.set_attribute(SpanAttributes.GEN_AI_REQUEST_N,
|
||||
req_state.n)
|
||||
span.set_attribute(SpanAttributes.GEN_AI_REQUEST_N, req_state.n)
|
||||
|
||||
def _update_stats_from_output(self, req_state: RequestState,
|
||||
engine_core_output: EngineCoreOutput,
|
||||
engine_core_timestamp: Optional[float],
|
||||
iteration_stats: Optional[IterationStats]):
|
||||
def _update_stats_from_output(
|
||||
self,
|
||||
req_state: RequestState,
|
||||
engine_core_output: EngineCoreOutput,
|
||||
engine_core_timestamp: Optional[float],
|
||||
iteration_stats: Optional[IterationStats],
|
||||
):
|
||||
if iteration_stats is None:
|
||||
return
|
||||
|
||||
@@ -548,15 +565,21 @@ class OutputProcessor:
|
||||
|
||||
assert engine_core_timestamp is not None
|
||||
assert req_state.stats is not None
|
||||
iteration_stats.update_from_output(engine_core_output,
|
||||
engine_core_timestamp,
|
||||
req_state.is_prefilling,
|
||||
req_state.prompt_len,
|
||||
req_state.stats, lora_stats)
|
||||
iteration_stats.update_from_output(
|
||||
engine_core_output,
|
||||
engine_core_timestamp,
|
||||
req_state.is_prefilling,
|
||||
req_state.prompt_len,
|
||||
req_state.stats,
|
||||
lora_stats,
|
||||
)
|
||||
|
||||
def _update_stats_from_finished(self, req_state: RequestState,
|
||||
finish_reason: Optional[FinishReason],
|
||||
iteration_stats: Optional[IterationStats]):
|
||||
def _update_stats_from_finished(
|
||||
self,
|
||||
req_state: RequestState,
|
||||
finish_reason: Optional[FinishReason],
|
||||
iteration_stats: Optional[IterationStats],
|
||||
):
|
||||
if iteration_stats is None:
|
||||
return
|
||||
|
||||
@@ -565,11 +588,13 @@ class OutputProcessor:
|
||||
iteration_stats.update_from_finished_request(
|
||||
finish_reason=finish_reason,
|
||||
num_prompt_tokens=length_from_prompt_token_ids_or_embeds(
|
||||
req_state.prompt_token_ids, req_state.prompt_embeds),
|
||||
req_state.prompt_token_ids, req_state.prompt_embeds
|
||||
),
|
||||
max_tokens_param=req_state.max_tokens_param,
|
||||
req_stats=req_state.stats)
|
||||
req_stats=req_state.stats,
|
||||
)
|
||||
self.lora_states.finish_request(req_state)
|
||||
|
||||
ParentRequest.observe_finished_request(
|
||||
req_state.parent_req, iteration_stats,
|
||||
req_state.stats.num_generation_tokens)
|
||||
req_state.parent_req, iteration_stats, req_state.stats.num_generation_tokens
|
||||
)
|
||||
|
||||
@@ -31,15 +31,16 @@ class ParentRequest:
|
||||
# To efficiently obtain child sampling params
|
||||
cached_child_sampling_params: Optional[SamplingParams]
|
||||
|
||||
def __init__(self, request_id: str,
|
||||
sampling_params: SamplingParams) -> None:
|
||||
def __init__(self, request_id: str, sampling_params: SamplingParams) -> None:
|
||||
self.request_id = request_id
|
||||
self.sampling_params = sampling_params
|
||||
|
||||
self.child_requests = set()
|
||||
self.output_aggregator = [None] * sampling_params.n if (
|
||||
sampling_params.output_kind
|
||||
== RequestOutputKind.FINAL_ONLY) else []
|
||||
self.output_aggregator = (
|
||||
[None] * sampling_params.n
|
||||
if (sampling_params.output_kind == RequestOutputKind.FINAL_ONLY)
|
||||
else []
|
||||
)
|
||||
self.max_num_generation_tokens = 0
|
||||
self.cached_child_sampling_params = None
|
||||
|
||||
@@ -49,7 +50,7 @@ class ParentRequest:
|
||||
) -> SamplingParams:
|
||||
"""Efficiently obtain child `sampling_params`
|
||||
|
||||
If `sampling_params.seed` is not `None` then
|
||||
If `sampling_params.seed` is not `None` then
|
||||
each child request requires a unique clone of
|
||||
parent `sampling_params` with a unique seed.
|
||||
|
||||
@@ -76,10 +77,10 @@ class ParentRequest:
|
||||
|
||||
def get_child_info(self, index: int) -> tuple[str, SamplingParams]:
|
||||
"""Get child request ID and sampling params.
|
||||
|
||||
|
||||
Args:
|
||||
index: index within `n` child requests.
|
||||
|
||||
|
||||
Returns:
|
||||
(request ID, sampling_params) tuple
|
||||
"""
|
||||
@@ -111,23 +112,25 @@ class ParentRequest:
|
||||
return self.request_id, outputs, finished
|
||||
|
||||
def observe_num_generation_tokens(self, num_generation_tokens: int):
|
||||
self.max_num_generation_tokens = max(num_generation_tokens,
|
||||
self.max_num_generation_tokens)
|
||||
self.max_num_generation_tokens = max(
|
||||
num_generation_tokens, self.max_num_generation_tokens
|
||||
)
|
||||
return self.max_num_generation_tokens
|
||||
|
||||
@staticmethod
|
||||
def observe_finished_request(parent_req: Optional['ParentRequest'],
|
||||
iteration_stats: IterationStats,
|
||||
num_generation_tokens: int):
|
||||
|
||||
def observe_finished_request(
|
||||
parent_req: Optional["ParentRequest"],
|
||||
iteration_stats: IterationStats,
|
||||
num_generation_tokens: int,
|
||||
):
|
||||
n_param = parent_req.n if parent_req is not None else 1
|
||||
|
||||
if parent_req is not None:
|
||||
num_generation_tokens = parent_req.observe_num_generation_tokens(
|
||||
num_generation_tokens)
|
||||
num_generation_tokens
|
||||
)
|
||||
|
||||
# Child requests finished, we can now record to iteration stats
|
||||
if parent_req is None or not parent_req.child_requests:
|
||||
iteration_stats.max_num_generation_tokens_iter.append(
|
||||
num_generation_tokens)
|
||||
iteration_stats.max_num_generation_tokens_iter.append(num_generation_tokens)
|
||||
iteration_stats.n_params_iter.append(n_param)
|
||||
|
||||
+97
-66
@@ -21,27 +21,25 @@ from vllm.sampling_params import SamplingParams
|
||||
from vllm.transformers_utils.tokenizer import AnyTokenizer
|
||||
from vllm.utils import length_from_prompt_token_ids_or_embeds
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm.v1.structured_output.backend_guidance import (
|
||||
validate_guidance_grammar)
|
||||
from vllm.v1.structured_output.backend_guidance import validate_guidance_grammar
|
||||
from vllm.v1.structured_output.backend_lm_format_enforcer import (
|
||||
validate_structured_output_request_lm_format_enforcer)
|
||||
validate_structured_output_request_lm_format_enforcer,
|
||||
)
|
||||
from vllm.v1.structured_output.backend_outlines import (
|
||||
validate_structured_output_request_outlines)
|
||||
from vllm.v1.structured_output.backend_xgrammar import (
|
||||
validate_xgrammar_grammar)
|
||||
validate_structured_output_request_outlines,
|
||||
)
|
||||
from vllm.v1.structured_output.backend_xgrammar import validate_xgrammar_grammar
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class Processor:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
tokenizer: AnyTokenizer,
|
||||
mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
|
||||
):
|
||||
|
||||
self.vllm_config = vllm_config
|
||||
self.model_config = vllm_config.model_config
|
||||
self.cache_config = vllm_config.cache_config
|
||||
@@ -49,12 +47,10 @@ class Processor:
|
||||
self.structured_outputs_config = vllm_config.structured_outputs_config
|
||||
self.tokenizer = tokenizer
|
||||
|
||||
self.generation_config_fields = (
|
||||
self.model_config.try_get_generation_config())
|
||||
self.generation_config_fields = self.model_config.try_get_generation_config()
|
||||
|
||||
self.mm_registry = mm_registry
|
||||
self.mm_processor_cache = processor_cache_from_config(
|
||||
vllm_config, mm_registry)
|
||||
self.mm_processor_cache = processor_cache_from_config(vllm_config, mm_registry)
|
||||
|
||||
self.input_preprocessor = InputPreprocessor(
|
||||
self.model_config,
|
||||
@@ -79,7 +75,8 @@ class Processor:
|
||||
if num_logprobs > max_logprobs:
|
||||
raise ValueError(
|
||||
f"Requested sample logprobs of {num_logprobs}, "
|
||||
f"which is greater than max allowed: {max_logprobs}")
|
||||
f"which is greater than max allowed: {max_logprobs}"
|
||||
)
|
||||
|
||||
# Validate prompt logprobs.
|
||||
if params.prompt_logprobs:
|
||||
@@ -89,7 +86,8 @@ class Processor:
|
||||
if num_prompt_logprobs > max_logprobs:
|
||||
raise ValueError(
|
||||
f"Requested prompt logprobs of {num_prompt_logprobs}, "
|
||||
f"which is greater than max allowed: {max_logprobs}")
|
||||
f"which is greater than max allowed: {max_logprobs}"
|
||||
)
|
||||
|
||||
def _validate_sampling_params(
|
||||
self,
|
||||
@@ -108,8 +106,7 @@ class Processor:
|
||||
return
|
||||
vocab_size = len(self.tokenizer)
|
||||
if not all(0 <= tid < vocab_size for tid in params.allowed_token_ids):
|
||||
raise ValueError(
|
||||
"allowed_token_ids contains out-of-vocab token id!")
|
||||
raise ValueError("allowed_token_ids contains out-of-vocab token id!")
|
||||
|
||||
def _validate_logit_bias(
|
||||
self,
|
||||
@@ -129,7 +126,8 @@ class Processor:
|
||||
if invalid_token_ids:
|
||||
raise ValueError(
|
||||
f"token_id(s) {invalid_token_ids} in logit_bias contain "
|
||||
f"out-of-vocab token ids. Vocabulary size: {vocab_size}")
|
||||
f"out-of-vocab token ids. Vocabulary size: {vocab_size}"
|
||||
)
|
||||
|
||||
def _validate_supported_sampling_params(
|
||||
self,
|
||||
@@ -140,8 +138,9 @@ class Processor:
|
||||
raise ValueError("vLLM V1 does not yet support best_of.")
|
||||
# Logits processors not supported.
|
||||
if params.logits_processors:
|
||||
raise ValueError("vLLM V1 does not support per request "
|
||||
"user provided logits processors.")
|
||||
raise ValueError(
|
||||
"vLLM V1 does not support per request user provided logits processors."
|
||||
)
|
||||
|
||||
def _validate_params(
|
||||
self,
|
||||
@@ -178,18 +177,23 @@ class Processor:
|
||||
for modality, items in mm_data.items():
|
||||
if modality in mm_uuids:
|
||||
data_len = len(items) if isinstance(items, list) else 1
|
||||
uuid_len = len(mm_uuids[modality]) if isinstance(
|
||||
mm_uuids[modality], list) else 1
|
||||
uuid_len = (
|
||||
len(mm_uuids[modality])
|
||||
if isinstance(mm_uuids[modality], list)
|
||||
else 1
|
||||
)
|
||||
if uuid_len != data_len:
|
||||
raise ValueError(
|
||||
f"multi_modal_uuids for modality '{modality}' "
|
||||
"must have same length as data: got "
|
||||
f"{uuid_len} uuids vs "
|
||||
f"{data_len} items.")
|
||||
f"{data_len} items."
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"multi_modal_uuids for modality '{modality}' must "
|
||||
"be provided if multi_modal_data is provided.")
|
||||
"be provided if multi_modal_data is provided."
|
||||
)
|
||||
|
||||
# Handle explicit encoder/decoder prompts or singleton prompt
|
||||
if isinstance(prompt, dict) and "encoder_prompt" in prompt:
|
||||
@@ -208,8 +212,9 @@ class Processor:
|
||||
|
||||
# LoRA request passed in while LoRA is not enabled
|
||||
if not self.lora_config:
|
||||
raise ValueError(f"Got lora_request {lora_request} but LoRA is "
|
||||
"not enabled!")
|
||||
raise ValueError(
|
||||
f"Got lora_request {lora_request} but LoRA is not enabled!"
|
||||
)
|
||||
|
||||
if self.tokenizer is not None:
|
||||
logger.warning_once(
|
||||
@@ -217,7 +222,8 @@ class Processor:
|
||||
"tokenizers for different LoRAs. By default, vLLM uses base "
|
||||
"model's tokenizer. If you are using a LoRA "
|
||||
"with its own tokenizer, consider specifying `--tokenizer "
|
||||
"[lora_path]` to use the LoRA tokenizer.")
|
||||
"[lora_path]` to use the LoRA tokenizer."
|
||||
)
|
||||
|
||||
def _validate_structured_output(self, params: SamplingParams) -> None:
|
||||
if not params.structured_outputs or not self.structured_outputs_config:
|
||||
@@ -235,20 +241,23 @@ class Processor:
|
||||
# to a specific backend based on `auto` behavior in a previous
|
||||
# request. We remember that it was set as a result of `auto`
|
||||
# using the `_backend_was_auto` field set in the params.
|
||||
if (backend != _backend
|
||||
and not (backend == "auto"
|
||||
and params.structured_outputs._backend_was_auto)):
|
||||
if backend != _backend and not (
|
||||
backend == "auto" and params.structured_outputs._backend_was_auto
|
||||
):
|
||||
raise ValueError(
|
||||
"Request-level structured output backend selection is not "
|
||||
f"supported. The request specified '{_backend}', but vLLM "
|
||||
f"was initialised with '{backend}'. This error can be "
|
||||
"resolved by removing '_backend' from the request.")
|
||||
"resolved by removing '_backend' from the request."
|
||||
)
|
||||
else:
|
||||
params.structured_outputs._backend = backend
|
||||
|
||||
# Request content validation
|
||||
if (isinstance(params.structured_outputs.choice, list)
|
||||
and not params.structured_outputs.choice):
|
||||
if (
|
||||
isinstance(params.structured_outputs.choice, list)
|
||||
and not params.structured_outputs.choice
|
||||
):
|
||||
# It is invalid for choice to be an empty list
|
||||
raise ValueError(
|
||||
f"Choice '{params.structured_outputs.choice}' cannot be an empty list" # noqa: E501
|
||||
@@ -318,9 +327,7 @@ class Processor:
|
||||
mm_uuids: MultiModalUUIDDict = {}
|
||||
for modality, data in mm_data.items():
|
||||
n = len(data) if isinstance(data, list) else 1
|
||||
mm_uuids[modality] = [
|
||||
f"{request_id}-{modality}-{i}" for i in range(n)
|
||||
]
|
||||
mm_uuids[modality] = [f"{request_id}-{modality}-{i}" for i in range(n)]
|
||||
return mm_uuids
|
||||
|
||||
def process_inputs(
|
||||
@@ -339,10 +346,13 @@ class Processor:
|
||||
self._validate_params(params)
|
||||
|
||||
data_parallel_size = self.vllm_config.parallel_config.data_parallel_size
|
||||
if data_parallel_rank is not None and not (0 <= data_parallel_rank <
|
||||
data_parallel_size):
|
||||
raise ValueError(f"data_parallel_rank {data_parallel_rank} "
|
||||
f"is out of range [0, {data_parallel_size}).")
|
||||
if data_parallel_rank is not None and not (
|
||||
0 <= data_parallel_rank < data_parallel_size
|
||||
):
|
||||
raise ValueError(
|
||||
f"data_parallel_rank {data_parallel_rank} "
|
||||
f"is out of range [0, {data_parallel_size})."
|
||||
)
|
||||
|
||||
if arrival_time is None:
|
||||
arrival_time = time.time()
|
||||
@@ -355,9 +365,11 @@ class Processor:
|
||||
# reused across requests, therefore identifying multimodal data items
|
||||
# by their content is no longer necessary, and we create uuids with
|
||||
# request id-modality-index as multimodal hash overrides.
|
||||
if (self.model_config.multimodal_config and
|
||||
self.model_config.multimodal_config.mm_processor_cache_gb == 0
|
||||
and not self.cache_config.enable_prefix_caching):
|
||||
if (
|
||||
self.model_config.multimodal_config
|
||||
and self.model_config.multimodal_config.mm_processor_cache_gb == 0
|
||||
and not self.cache_config.enable_prefix_caching
|
||||
):
|
||||
mm_uuids = self._maybe_build_mm_uuids(request_id, prompt)
|
||||
else:
|
||||
# Otherwise, use user-provided uuids as multimodal hash overrides
|
||||
@@ -378,6 +390,7 @@ class Processor:
|
||||
mm_uuids=mm_uuids,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
current_platform.validate_request(
|
||||
prompt=prompt,
|
||||
params=params,
|
||||
@@ -393,10 +406,16 @@ class Processor:
|
||||
# discriminated unions of TypedDicts, because of how it handles
|
||||
# inheritance of TypedDict. If we explicitly extract the items we want
|
||||
# we can avoid type errors from using `dict.get` later in the method.
|
||||
prompt_token_ids = decoder_inputs[
|
||||
"prompt_token_ids"] if decoder_inputs["type"] != "embeds" else None
|
||||
prompt_embeds = decoder_inputs["prompt_embeds"] if decoder_inputs[
|
||||
"type"] == "embeds" else None
|
||||
prompt_token_ids = (
|
||||
decoder_inputs["prompt_token_ids"]
|
||||
if decoder_inputs["type"] != "embeds"
|
||||
else None
|
||||
)
|
||||
prompt_embeds = (
|
||||
decoder_inputs["prompt_embeds"]
|
||||
if decoder_inputs["type"] == "embeds"
|
||||
else None
|
||||
)
|
||||
|
||||
sampling_params = None
|
||||
pooling_params = None
|
||||
@@ -406,11 +425,12 @@ class Processor:
|
||||
# If unset max tokens, then generate up to the max_model_len.
|
||||
if sampling_params.max_tokens is None:
|
||||
seq_len = length_from_prompt_token_ids_or_embeds(
|
||||
prompt_token_ids, prompt_embeds)
|
||||
sampling_params.max_tokens = \
|
||||
self.model_config.max_model_len - seq_len
|
||||
prompt_token_ids, prompt_embeds
|
||||
)
|
||||
sampling_params.max_tokens = self.model_config.max_model_len - seq_len
|
||||
sampling_params.update_from_generation_config(
|
||||
self.generation_config_fields, eos_token_id)
|
||||
self.generation_config_fields, eos_token_id
|
||||
)
|
||||
if self.tokenizer is not None:
|
||||
sampling_params.update_from_tokenizer(self.tokenizer)
|
||||
else:
|
||||
@@ -436,7 +456,9 @@ class Processor:
|
||||
data=decoder_mm_inputs[modality][idx],
|
||||
modality=modality,
|
||||
identifier=decoder_mm_hashes[modality][idx],
|
||||
mm_position=decoder_mm_positions[modality][idx]))
|
||||
mm_position=decoder_mm_positions[modality][idx],
|
||||
)
|
||||
)
|
||||
|
||||
return EngineCoreRequest(
|
||||
request_id=request_id,
|
||||
@@ -454,8 +476,9 @@ class Processor:
|
||||
trace_headers=trace_headers,
|
||||
)
|
||||
|
||||
def _validate_model_inputs(self, encoder_inputs: Optional[SingletonInputs],
|
||||
decoder_inputs: SingletonInputs):
|
||||
def _validate_model_inputs(
|
||||
self, encoder_inputs: Optional[SingletonInputs], decoder_inputs: SingletonInputs
|
||||
):
|
||||
if encoder_inputs is not None:
|
||||
self._validate_model_input(encoder_inputs, prompt_type="encoder")
|
||||
|
||||
@@ -469,12 +492,17 @@ class Processor:
|
||||
):
|
||||
model_config = self.model_config
|
||||
|
||||
prompt_ids = None if prompt_inputs[
|
||||
"type"] == "embeds" else prompt_inputs["prompt_token_ids"]
|
||||
prompt_embeds = prompt_inputs["prompt_embeds"] if prompt_inputs[
|
||||
"type"] == "embeds" else None
|
||||
prompt_len = length_from_prompt_token_ids_or_embeds(
|
||||
prompt_ids, prompt_embeds)
|
||||
prompt_ids = (
|
||||
None
|
||||
if prompt_inputs["type"] == "embeds"
|
||||
else prompt_inputs["prompt_token_ids"]
|
||||
)
|
||||
prompt_embeds = (
|
||||
prompt_inputs["prompt_embeds"]
|
||||
if prompt_inputs["type"] == "embeds"
|
||||
else None
|
||||
)
|
||||
prompt_len = length_from_prompt_token_ids_or_embeds(prompt_ids, prompt_embeds)
|
||||
if not prompt_ids:
|
||||
if prompt_type == "encoder" and model_config.is_multimodal_model:
|
||||
pass # Mllama may have empty encoder inputs for text-only data
|
||||
@@ -499,10 +527,10 @@ class Processor:
|
||||
|
||||
# Here we take the max of the two to determine if a token id is
|
||||
# truly out-of-vocabulary.
|
||||
if max_input_id > max(tokenizer.max_token_id,
|
||||
self.model_config.get_vocab_size() - 1):
|
||||
raise ValueError(
|
||||
f"Token id {max_input_id} is out of vocabulary")
|
||||
if max_input_id > max(
|
||||
tokenizer.max_token_id, self.model_config.get_vocab_size() - 1
|
||||
):
|
||||
raise ValueError(f"Token id {max_input_id} is out of vocabulary")
|
||||
|
||||
max_prompt_len = self.model_config.max_model_len
|
||||
if prompt_len > max_prompt_len:
|
||||
@@ -522,16 +550,19 @@ class Processor:
|
||||
"Make sure that `max_model_len` is no smaller than the "
|
||||
"number of text tokens plus multimodal tokens. For image "
|
||||
"inputs, the number of image tokens depends on the number "
|
||||
"of images, and possibly their aspect ratios as well.")
|
||||
"of images, and possibly their aspect ratios as well."
|
||||
)
|
||||
else:
|
||||
suggestion = (
|
||||
"Make sure that `max_model_len` is no smaller than the "
|
||||
"number of text tokens.")
|
||||
"number of text tokens."
|
||||
)
|
||||
|
||||
raise ValueError(
|
||||
f"The {prompt_type} prompt (length {prompt_len}) is "
|
||||
f"longer than the maximum model length of {max_prompt_len}. "
|
||||
f"{suggestion}")
|
||||
f"{suggestion}"
|
||||
)
|
||||
|
||||
# TODO: Find out how many placeholder tokens are there so we can
|
||||
# check that chunked prefill does not truncate them
|
||||
|
||||
+244
-190
@@ -70,6 +70,7 @@ class EngineHandshakeMetadata:
|
||||
including addresses of the front-end ZMQ queues that they should
|
||||
connect to.
|
||||
"""
|
||||
|
||||
addresses: EngineZmqAddresses
|
||||
parallel_config: dict[str, Union[int, str, list[int]]]
|
||||
|
||||
@@ -103,8 +104,7 @@ class CoreEngineProcManager:
|
||||
}
|
||||
|
||||
if client_handshake_address:
|
||||
common_kwargs[
|
||||
"client_handshake_address"] = client_handshake_address
|
||||
common_kwargs["client_handshake_address"] = client_handshake_address
|
||||
|
||||
self.processes: list[BaseProcess] = []
|
||||
local_dp_ranks = []
|
||||
@@ -115,21 +115,27 @@ class CoreEngineProcManager:
|
||||
# Start EngineCore in background process.
|
||||
local_dp_ranks.append(local_index)
|
||||
self.processes.append(
|
||||
context.Process(target=target_fn,
|
||||
name=f"EngineCore_DP{global_index}",
|
||||
kwargs=common_kwargs | {
|
||||
"dp_rank": global_index,
|
||||
"local_dp_rank": local_index,
|
||||
}))
|
||||
context.Process(
|
||||
target=target_fn,
|
||||
name=f"EngineCore_DP{global_index}",
|
||||
kwargs=common_kwargs
|
||||
| {
|
||||
"dp_rank": global_index,
|
||||
"local_dp_rank": local_index,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
self._finalizer = weakref.finalize(self, shutdown, self.processes)
|
||||
|
||||
data_parallel = vllm_config.parallel_config.data_parallel_size > 1
|
||||
try:
|
||||
for proc, local_dp_rank in zip(self.processes, local_dp_ranks):
|
||||
with set_device_control_env_var(
|
||||
vllm_config, local_dp_rank) if (
|
||||
data_parallel) else contextlib.nullcontext():
|
||||
with (
|
||||
set_device_control_env_var(vllm_config, local_dp_rank)
|
||||
if (data_parallel)
|
||||
else contextlib.nullcontext()
|
||||
):
|
||||
proc.start()
|
||||
finally:
|
||||
# Kill other procs if not all are running.
|
||||
@@ -151,13 +157,15 @@ class CoreEngineProcManager:
|
||||
"""Returns dict of proc name -> exit code for any finished procs."""
|
||||
return {
|
||||
proc.name: proc.exitcode
|
||||
for proc in self.processes if proc.exitcode is not None
|
||||
for proc in self.processes
|
||||
if proc.exitcode is not None
|
||||
}
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def set_device_control_env_var(vllm_config: VllmConfig,
|
||||
local_dp_rank: int) -> Iterator[None]:
|
||||
def set_device_control_env_var(
|
||||
vllm_config: VllmConfig, local_dp_rank: int
|
||||
) -> Iterator[None]:
|
||||
"""
|
||||
Temporarily set CUDA_VISIBLE_DEVICES or equivalent
|
||||
for engine subprocess.
|
||||
@@ -166,12 +174,13 @@ def set_device_control_env_var(vllm_config: VllmConfig,
|
||||
evar = current_platform.device_control_env_var
|
||||
|
||||
value = get_device_indices(evar, local_dp_rank, world_size)
|
||||
with patch.dict(os.environ, values=((evar, value), )):
|
||||
with patch.dict(os.environ, values=((evar, value),)):
|
||||
yield
|
||||
|
||||
|
||||
def get_device_indices(device_control_env_var: str, local_dp_rank: int,
|
||||
world_size: int):
|
||||
def get_device_indices(
|
||||
device_control_env_var: str, local_dp_rank: int, world_size: int
|
||||
):
|
||||
"""
|
||||
Returns a comma-separated string of device indices for the specified
|
||||
data parallel rank.
|
||||
@@ -182,14 +191,16 @@ def get_device_indices(device_control_env_var: str, local_dp_rank: int,
|
||||
try:
|
||||
value = ",".join(
|
||||
str(current_platform.device_id_to_physical_device_id(i))
|
||||
for i in range(local_dp_rank * world_size, (local_dp_rank + 1) *
|
||||
world_size))
|
||||
for i in range(local_dp_rank * world_size, (local_dp_rank + 1) * world_size)
|
||||
)
|
||||
except IndexError as e:
|
||||
raise Exception(f"Error setting {device_control_env_var}: "
|
||||
f"local range: [{local_dp_rank * world_size}, "
|
||||
f"{(local_dp_rank + 1) * world_size}) "
|
||||
"base value: "
|
||||
f"\"{os.getenv(device_control_env_var)}\"") from e
|
||||
raise Exception(
|
||||
f"Error setting {device_control_env_var}: "
|
||||
f"local range: [{local_dp_rank * world_size}, "
|
||||
f"{(local_dp_rank + 1) * world_size}) "
|
||||
"base value: "
|
||||
f'"{os.getenv(device_control_env_var)}"'
|
||||
) from e
|
||||
return value
|
||||
|
||||
|
||||
@@ -215,8 +226,7 @@ class CoreEngineActorManager:
|
||||
|
||||
import ray
|
||||
from ray.runtime_env import RuntimeEnv
|
||||
from ray.util.scheduling_strategies import (
|
||||
PlacementGroupSchedulingStrategy)
|
||||
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
|
||||
|
||||
from vllm.v1.engine.core import DPEngineCoreActor
|
||||
|
||||
@@ -225,8 +235,7 @@ class CoreEngineActorManager:
|
||||
|
||||
env_vars_list = get_env_vars_to_copy(destination="DPEngineCoreActor")
|
||||
self.env_vars_dict = {
|
||||
name: os.environ[name]
|
||||
for name in env_vars_list if name in os.environ
|
||||
name: os.environ[name] for name in env_vars_list if name in os.environ
|
||||
}
|
||||
runtime_env = RuntimeEnv(env_vars=self.env_vars_dict)
|
||||
|
||||
@@ -234,37 +243,38 @@ class CoreEngineActorManager:
|
||||
self.executor_class = executor_class
|
||||
self.log_stats = log_stats
|
||||
dp_size = vllm_config.parallel_config.data_parallel_size
|
||||
local_engine_count = \
|
||||
vllm_config.parallel_config.data_parallel_size_local
|
||||
local_engine_count = vllm_config.parallel_config.data_parallel_size_local
|
||||
world_size = vllm_config.parallel_config.world_size
|
||||
|
||||
if ray.is_initialized():
|
||||
logger.info(
|
||||
"Ray is already initialized. Skipping Ray initialization.")
|
||||
logger.info("Ray is already initialized. Skipping Ray initialization.")
|
||||
else:
|
||||
ray.init()
|
||||
|
||||
if placement_groups is not None:
|
||||
assert local_dp_ranks is not None, (
|
||||
"local_dp_ranks must be provided if "
|
||||
"placement_groups is provided")
|
||||
"local_dp_ranks must be provided if placement_groups is provided"
|
||||
)
|
||||
assert len(placement_groups) == len(local_dp_ranks), (
|
||||
"placement_groups and local_dp_ranks must "
|
||||
"have the same length")
|
||||
"placement_groups and local_dp_ranks must have the same length"
|
||||
)
|
||||
logger.info("Using provided placement groups")
|
||||
# TODO(rui): validate passed-in placement groups
|
||||
self.created_placement_groups = []
|
||||
else:
|
||||
placement_groups, local_dp_ranks = \
|
||||
placement_groups, local_dp_ranks = (
|
||||
CoreEngineActorManager.create_dp_placement_groups(vllm_config)
|
||||
)
|
||||
self.created_placement_groups = placement_groups
|
||||
assert len(placement_groups) == dp_size, (
|
||||
"Number of placement groups must match data parallel size")
|
||||
"Number of placement groups must match data parallel size"
|
||||
)
|
||||
|
||||
self.placement_group_is_local = []
|
||||
refs = []
|
||||
for index, local_index, pg in zip(range(dp_size), local_dp_ranks,
|
||||
placement_groups):
|
||||
for index, local_index, pg in zip(
|
||||
range(dp_size), local_dp_ranks, placement_groups
|
||||
):
|
||||
dp_vllm_config = copy.deepcopy(vllm_config)
|
||||
dp_vllm_config.parallel_config.placement_group = pg
|
||||
local_client = index < local_engine_count
|
||||
@@ -275,24 +285,32 @@ class CoreEngineActorManager:
|
||||
# https://github.com/ray-project/ray/blob/master/python/ray/_private/accelerators/intel_gpu.py#L56 # noqa: E501
|
||||
if current_platform.is_xpu():
|
||||
device_evar = current_platform.device_control_env_var
|
||||
device_indices = get_device_indices(device_evar, local_index,
|
||||
world_size)
|
||||
device_indices = get_device_indices(
|
||||
device_evar, local_index, world_size
|
||||
)
|
||||
actor_env_vars = self.env_vars_dict.copy()
|
||||
actor_env_vars[device_evar] = device_indices
|
||||
runtime_env = RuntimeEnv(env_vars=actor_env_vars)
|
||||
|
||||
actor = ray.remote(DPEngineCoreActor).options(
|
||||
scheduling_strategy=PlacementGroupSchedulingStrategy(
|
||||
placement_group=pg,
|
||||
placement_group_bundle_index=world_size,
|
||||
),
|
||||
runtime_env=runtime_env).remote(vllm_config=dp_vllm_config,
|
||||
executor_class=executor_class,
|
||||
log_stats=log_stats,
|
||||
local_client=local_client,
|
||||
addresses=addresses,
|
||||
dp_rank=index,
|
||||
local_dp_rank=local_index)
|
||||
actor = (
|
||||
ray.remote(DPEngineCoreActor)
|
||||
.options(
|
||||
scheduling_strategy=PlacementGroupSchedulingStrategy(
|
||||
placement_group=pg,
|
||||
placement_group_bundle_index=world_size,
|
||||
),
|
||||
runtime_env=runtime_env,
|
||||
)
|
||||
.remote(
|
||||
vllm_config=dp_vllm_config,
|
||||
executor_class=executor_class,
|
||||
log_stats=log_stats,
|
||||
local_client=local_client,
|
||||
addresses=addresses,
|
||||
dp_rank=index,
|
||||
local_dp_rank=local_index,
|
||||
)
|
||||
)
|
||||
if local_client:
|
||||
self.local_engine_actors.append(actor)
|
||||
else:
|
||||
@@ -307,7 +325,7 @@ class CoreEngineActorManager:
|
||||
|
||||
@staticmethod
|
||||
def create_dp_placement_groups(
|
||||
vllm_config: VllmConfig
|
||||
vllm_config: VllmConfig,
|
||||
) -> tuple[list["PlacementGroup"], list[int]]:
|
||||
"""
|
||||
Create placement groups for data parallel.
|
||||
@@ -317,23 +335,23 @@ class CoreEngineActorManager:
|
||||
from ray._private.state import available_resources_per_node
|
||||
|
||||
logger.info("Creating placement groups for data parallel")
|
||||
dp_master_ip = \
|
||||
vllm_config.parallel_config.data_parallel_master_ip
|
||||
dp_master_ip = vllm_config.parallel_config.data_parallel_master_ip
|
||||
num_pg_to_create = vllm_config.parallel_config.data_parallel_size
|
||||
local_engine_count = \
|
||||
vllm_config.parallel_config.data_parallel_size_local
|
||||
local_engine_count = vllm_config.parallel_config.data_parallel_size_local
|
||||
|
||||
available_resources = available_resources_per_node()
|
||||
world_size = vllm_config.parallel_config.world_size
|
||||
placement_groups: list[PlacementGroup] = []
|
||||
local_dp_ranks: list[int] = []
|
||||
dp_master_ip_key = f'node:{dp_master_ip}'
|
||||
nodes = sorted(available_resources.values(),
|
||||
key=lambda x: dp_master_ip_key not in x)
|
||||
assert len(nodes) > 0, (
|
||||
"No nodes with resources found in Ray cluster.")
|
||||
dp_master_ip_key = f"node:{dp_master_ip}"
|
||||
nodes = sorted(
|
||||
available_resources.values(), key=lambda x: dp_master_ip_key not in x
|
||||
)
|
||||
assert len(nodes) > 0, "No nodes with resources found in Ray cluster."
|
||||
assert dp_master_ip_key in nodes[0], (
|
||||
"The DP master node (ip: %s) is missing or dead", dp_master_ip)
|
||||
"The DP master node (ip: %s) is missing or dead",
|
||||
dp_master_ip,
|
||||
)
|
||||
device_str = current_platform.ray_device_key
|
||||
for node_resources in nodes:
|
||||
if device_str not in node_resources:
|
||||
@@ -341,19 +359,16 @@ class CoreEngineActorManager:
|
||||
# For now, each DP rank can only be assigned to one node
|
||||
# TODO(rui): support allocating a single DP rank
|
||||
# to multiple nodes
|
||||
available_engine_count = int(
|
||||
node_resources[device_str]) // world_size
|
||||
available_engine_count = int(node_resources[device_str]) // world_size
|
||||
if dp_master_ip_key in node_resources:
|
||||
assert available_engine_count >= local_engine_count, (
|
||||
"Not enough resources to allocate DP ranks "
|
||||
f"on DP master node {dp_master_ip}")
|
||||
f"on DP master node {dp_master_ip}"
|
||||
)
|
||||
for i in range(local_engine_count):
|
||||
bundles = [{
|
||||
device_str: 1.0,
|
||||
"node:" + dp_master_ip: 0.001
|
||||
}] * world_size + [{
|
||||
"CPU": 1.0
|
||||
}]
|
||||
bundles = [
|
||||
{device_str: 1.0, "node:" + dp_master_ip: 0.001}
|
||||
] * world_size + [{"CPU": 1.0}]
|
||||
pg = ray.util.placement_group(
|
||||
name=f"dp_rank_{len(placement_groups)}",
|
||||
strategy="STRICT_PACK",
|
||||
@@ -379,7 +394,8 @@ class CoreEngineActorManager:
|
||||
"placement groups, only created "
|
||||
f"{len(placement_groups)} placement groups. "
|
||||
"Available resources: "
|
||||
f"{available_resources}")
|
||||
f"{available_resources}"
|
||||
)
|
||||
return placement_groups, local_dp_ranks
|
||||
|
||||
@staticmethod
|
||||
@@ -390,8 +406,10 @@ class CoreEngineActorManager:
|
||||
Add placement groups for new data parallel size.
|
||||
"""
|
||||
import ray
|
||||
from ray._private.state import (available_resources_per_node,
|
||||
total_resources_per_node)
|
||||
from ray._private.state import (
|
||||
available_resources_per_node,
|
||||
total_resources_per_node,
|
||||
)
|
||||
from ray.util.state import list_nodes
|
||||
|
||||
old_dp_size = old_vllm_config.parallel_config.data_parallel_size
|
||||
@@ -405,10 +423,10 @@ class CoreEngineActorManager:
|
||||
|
||||
nodes = list_nodes()
|
||||
nodes = sorted(nodes, key=lambda node: node.node_ip != dp_master_ip)
|
||||
assert nodes[0].node_ip == dp_master_ip, (
|
||||
"The first node must be the head node")
|
||||
assert nodes[0].node_ip == dp_master_ip, "The first node must be the head node"
|
||||
assert len(nodes) == 1 or nodes[1].node_ip != dp_master_ip, (
|
||||
"There can only be one head node")
|
||||
"There can only be one head node"
|
||||
)
|
||||
|
||||
available_resources = available_resources_per_node()
|
||||
total_resources = total_resources_per_node()
|
||||
@@ -446,12 +464,9 @@ class CoreEngineActorManager:
|
||||
|
||||
# Create bundles with node constraint for master node
|
||||
if node_ip == dp_master_ip:
|
||||
bundles = [{
|
||||
device_str: 1.0,
|
||||
"node:" + dp_master_ip: 0.001
|
||||
}] * world_size + [{
|
||||
"CPU": 1.0
|
||||
}]
|
||||
bundles = [
|
||||
{device_str: 1.0, "node:" + dp_master_ip: 0.001}
|
||||
] * world_size + [{"CPU": 1.0}]
|
||||
else:
|
||||
bundles = [{device_str: 1.0}] * world_size + [{"CPU": 1.0}]
|
||||
|
||||
@@ -470,69 +485,76 @@ class CoreEngineActorManager:
|
||||
|
||||
return placement_groups, local_dp_ranks
|
||||
|
||||
def scale_up_elastic_ep(self, cur_vllm_config: VllmConfig,
|
||||
new_data_parallel_size: int) -> None:
|
||||
def scale_up_elastic_ep(
|
||||
self, cur_vllm_config: VllmConfig, new_data_parallel_size: int
|
||||
) -> None:
|
||||
import copy
|
||||
|
||||
import ray
|
||||
from ray.runtime_env import RuntimeEnv
|
||||
from ray.util.scheduling_strategies import (
|
||||
PlacementGroupSchedulingStrategy)
|
||||
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
|
||||
|
||||
from vllm.v1.engine.core import DPEngineCoreActor
|
||||
|
||||
cur_data_parallel_size = len(self.local_engine_actors) + \
|
||||
len(self.remote_engine_actors)
|
||||
cur_data_parallel_size = len(self.local_engine_actors) + len(
|
||||
self.remote_engine_actors
|
||||
)
|
||||
|
||||
assert new_data_parallel_size > cur_data_parallel_size, (
|
||||
f"New data parallel size {new_data_parallel_size} must be greater "
|
||||
f"than current data parallel size {cur_data_parallel_size} "
|
||||
"for scale up")
|
||||
"for scale up"
|
||||
)
|
||||
|
||||
placement_groups, local_dp_ranks = \
|
||||
self.add_dp_placement_groups(
|
||||
cur_vllm_config, new_data_parallel_size)
|
||||
placement_groups, local_dp_ranks = self.add_dp_placement_groups(
|
||||
cur_vllm_config, new_data_parallel_size
|
||||
)
|
||||
|
||||
world_size = cur_vllm_config.parallel_config.world_size
|
||||
dp_master_ip = cur_vllm_config.parallel_config.data_parallel_master_ip
|
||||
new_local_engines = 0
|
||||
|
||||
runtime_env = RuntimeEnv(env_vars=self.env_vars_dict
|
||||
| {"VLLM_ELASTIC_EP_SCALE_UP_LAUNCH": "1"})
|
||||
for i, (pg,
|
||||
local_rank) in enumerate(zip(placement_groups,
|
||||
local_dp_ranks)):
|
||||
runtime_env = RuntimeEnv(
|
||||
env_vars=self.env_vars_dict | {"VLLM_ELASTIC_EP_SCALE_UP_LAUNCH": "1"}
|
||||
)
|
||||
for i, (pg, local_rank) in enumerate(zip(placement_groups, local_dp_ranks)):
|
||||
rank = cur_data_parallel_size + i
|
||||
dp_vllm_config = copy.deepcopy(cur_vllm_config)
|
||||
dp_vllm_config.parallel_config.data_parallel_size = \
|
||||
new_data_parallel_size
|
||||
dp_vllm_config.parallel_config.data_parallel_size = new_data_parallel_size
|
||||
dp_vllm_config.parallel_config.placement_group = pg
|
||||
|
||||
# Check if this placement group is on the head node
|
||||
local_client = any(
|
||||
bundle.get("node:" + dp_master_ip, 0) > 0
|
||||
for bundle in pg.bundle_specs)
|
||||
bundle.get("node:" + dp_master_ip, 0) > 0 for bundle in pg.bundle_specs
|
||||
)
|
||||
|
||||
if local_client:
|
||||
new_local_engines += 1
|
||||
# Update data_parallel_size_local
|
||||
dp_vllm_config.parallel_config.data_parallel_size_local = (
|
||||
cur_vllm_config.parallel_config.data_parallel_size_local +
|
||||
new_local_engines)
|
||||
cur_vllm_config.parallel_config.data_parallel_size_local
|
||||
+ new_local_engines
|
||||
)
|
||||
|
||||
actor = ray.remote(DPEngineCoreActor).options(
|
||||
scheduling_strategy=PlacementGroupSchedulingStrategy(
|
||||
placement_group=pg,
|
||||
placement_group_bundle_index=world_size,
|
||||
),
|
||||
runtime_env=runtime_env).remote(
|
||||
actor = (
|
||||
ray.remote(DPEngineCoreActor)
|
||||
.options(
|
||||
scheduling_strategy=PlacementGroupSchedulingStrategy(
|
||||
placement_group=pg,
|
||||
placement_group_bundle_index=world_size,
|
||||
),
|
||||
runtime_env=runtime_env,
|
||||
)
|
||||
.remote(
|
||||
vllm_config=dp_vllm_config,
|
||||
executor_class=self.executor_class,
|
||||
log_stats=self.log_stats,
|
||||
local_client=local_client,
|
||||
addresses=self.addresses,
|
||||
dp_rank=rank,
|
||||
local_dp_rank=local_rank)
|
||||
local_dp_rank=local_rank,
|
||||
)
|
||||
)
|
||||
|
||||
if local_client:
|
||||
self.local_engine_actors.append(actor)
|
||||
@@ -541,37 +563,47 @@ class CoreEngineActorManager:
|
||||
self.created_placement_groups.append(pg)
|
||||
self.placement_group_is_local.append(local_client)
|
||||
|
||||
ray.get([
|
||||
actor.wait_for_init.remote()
|
||||
for actor in (self.local_engine_actors[-new_local_engines:]
|
||||
if new_local_engines > 0 else []) +
|
||||
self.remote_engine_actors[-(len(placement_groups) -
|
||||
new_local_engines):]
|
||||
])
|
||||
ray.get(
|
||||
[
|
||||
actor.wait_for_init.remote()
|
||||
for actor in (
|
||||
self.local_engine_actors[-new_local_engines:]
|
||||
if new_local_engines > 0
|
||||
else []
|
||||
)
|
||||
+ self.remote_engine_actors[
|
||||
-(len(placement_groups) - new_local_engines) :
|
||||
]
|
||||
]
|
||||
)
|
||||
|
||||
actors = (self.local_engine_actors[-new_local_engines:]
|
||||
if new_local_engines > 0 else []) + \
|
||||
self.remote_engine_actors[-(len(placement_groups) -
|
||||
new_local_engines):]
|
||||
actors = (
|
||||
self.local_engine_actors[-new_local_engines:]
|
||||
if new_local_engines > 0
|
||||
else []
|
||||
) + self.remote_engine_actors[-(len(placement_groups) - new_local_engines) :]
|
||||
|
||||
for actor in actors:
|
||||
self.run_refs.append(actor.run.remote())
|
||||
|
||||
cur_vllm_config.parallel_config.data_parallel_size = \
|
||||
new_data_parallel_size
|
||||
cur_vllm_config.parallel_config.data_parallel_size = new_data_parallel_size
|
||||
# Update old_vllm_config with new data_parallel_size_local if any new
|
||||
# local engines were added
|
||||
if new_local_engines > 0:
|
||||
cur_vllm_config.parallel_config.data_parallel_size_local += \
|
||||
cur_vllm_config.parallel_config.data_parallel_size_local += (
|
||||
new_local_engines
|
||||
)
|
||||
|
||||
def scale_down_elastic_ep(self, cur_data_parallel_size: int,
|
||||
new_data_parallel_size: int) -> None:
|
||||
def scale_down_elastic_ep(
|
||||
self, cur_data_parallel_size: int, new_data_parallel_size: int
|
||||
) -> None:
|
||||
import ray
|
||||
|
||||
assert cur_data_parallel_size > new_data_parallel_size, (
|
||||
f"cur_data_parallel_size {cur_data_parallel_size} must be greater "
|
||||
f"than new_data_parallel_size {new_data_parallel_size} "
|
||||
"for scale down")
|
||||
"for scale down"
|
||||
)
|
||||
for _ in range(cur_data_parallel_size - new_data_parallel_size):
|
||||
pg = self.created_placement_groups.pop()
|
||||
is_local = self.placement_group_is_local.pop()
|
||||
@@ -586,6 +618,7 @@ class CoreEngineActorManager:
|
||||
|
||||
def close(self):
|
||||
import ray
|
||||
|
||||
for actor in self.local_engine_actors + self.remote_engine_actors:
|
||||
ray.kill(actor)
|
||||
for pg in self.created_placement_groups:
|
||||
@@ -598,11 +631,13 @@ def launch_core_engines(
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
num_api_servers: int = 1,
|
||||
) -> Iterator[tuple[
|
||||
) -> Iterator[
|
||||
tuple[
|
||||
Optional[Union[CoreEngineProcManager, CoreEngineActorManager]],
|
||||
Optional[DPCoordinator],
|
||||
EngineZmqAddresses,
|
||||
]]:
|
||||
]
|
||||
]:
|
||||
"""Launch engine and DP coordinator processes as needed."""
|
||||
|
||||
parallel_config = vllm_config.parallel_config
|
||||
@@ -611,8 +646,10 @@ def launch_core_engines(
|
||||
local_start_index = parallel_config.data_parallel_rank_local
|
||||
dp_rank = parallel_config.data_parallel_rank
|
||||
host = parallel_config.data_parallel_master_ip
|
||||
local_engines_only = (parallel_config.data_parallel_hybrid_lb
|
||||
or parallel_config.data_parallel_external_lb)
|
||||
local_engines_only = (
|
||||
parallel_config.data_parallel_hybrid_lb
|
||||
or parallel_config.data_parallel_external_lb
|
||||
)
|
||||
|
||||
# In offline mode there is an LLM instance per DP rank and
|
||||
# one core engine per LLM, see
|
||||
@@ -621,8 +658,9 @@ def launch_core_engines(
|
||||
|
||||
# client_local_only = True for cases where this front-end
|
||||
# sends requests only to colocated engines.
|
||||
client_local_only = (offline_mode or local_engines_only
|
||||
or (local_engine_count == dp_size))
|
||||
client_local_only = (
|
||||
offline_mode or local_engines_only or (local_engine_count == dp_size)
|
||||
)
|
||||
|
||||
# Set up input and output addresses.
|
||||
addresses = EngineZmqAddresses(
|
||||
@@ -644,12 +682,13 @@ def launch_core_engines(
|
||||
coordinator = DPCoordinator(parallel_config)
|
||||
|
||||
addresses.coordinator_input, addresses.coordinator_output = (
|
||||
coordinator.get_engine_socket_addresses())
|
||||
coordinator.get_engine_socket_addresses()
|
||||
)
|
||||
addresses.frontend_stats_publish_address = (
|
||||
coordinator.get_stats_publish_address())
|
||||
coordinator.get_stats_publish_address()
|
||||
)
|
||||
|
||||
logger.info("Started DP Coordinator process (PID: %d)",
|
||||
coordinator.proc.pid)
|
||||
logger.info("Started DP Coordinator process (PID: %d)", coordinator.proc.pid)
|
||||
else:
|
||||
coordinator = None
|
||||
|
||||
@@ -675,14 +714,14 @@ def launch_core_engines(
|
||||
# Note this also covers the case where we have zero local engines
|
||||
# and rank 0 is headless.
|
||||
engines_to_handshake = [
|
||||
CoreEngine(index=i, local=(i < local_engine_count))
|
||||
for i in range(dp_size)
|
||||
CoreEngine(index=i, local=(i < local_engine_count)) for i in range(dp_size)
|
||||
]
|
||||
else:
|
||||
# Rank > 0 handshakes with just the local cores it is managing.
|
||||
assert local_engines_only, (
|
||||
"Attempting to launch core_engines from dp_rank > 0, but "
|
||||
"found internal DPLB, which is incompatible.")
|
||||
"found internal DPLB, which is incompatible."
|
||||
)
|
||||
engines_to_handshake = [
|
||||
CoreEngine(index=i, local=True)
|
||||
for i in range(dp_rank, dp_rank + local_engine_count)
|
||||
@@ -695,7 +734,8 @@ def launch_core_engines(
|
||||
handshake_local_only = offline_mode or local_engine_count == dp_size
|
||||
|
||||
handshake_address = get_engine_client_zmq_addr(
|
||||
handshake_local_only, host, parallel_config.data_parallel_rpc_port)
|
||||
handshake_local_only, host, parallel_config.data_parallel_rpc_port
|
||||
)
|
||||
|
||||
if local_engines_only and dp_rank > 0:
|
||||
assert not handshake_local_only
|
||||
@@ -705,9 +745,9 @@ def launch_core_engines(
|
||||
local_handshake_address = handshake_address
|
||||
client_handshake_address = None
|
||||
|
||||
with zmq_socket_ctx(local_handshake_address, zmq.ROUTER,
|
||||
bind=True) as handshake_socket:
|
||||
|
||||
with zmq_socket_ctx(
|
||||
local_handshake_address, zmq.ROUTER, bind=True
|
||||
) as handshake_socket:
|
||||
from vllm.v1.engine.core import EngineCoreProc
|
||||
|
||||
# Start local engines.
|
||||
@@ -722,7 +762,8 @@ def launch_core_engines(
|
||||
local_client=True,
|
||||
local_engine_count=local_engine_count,
|
||||
start_index=dp_rank,
|
||||
local_start_index=local_start_index or 0)
|
||||
local_start_index=local_start_index or 0,
|
||||
)
|
||||
else:
|
||||
local_engine_manager = None
|
||||
|
||||
@@ -757,8 +798,10 @@ def wait_for_engine_startup(
|
||||
poller = zmq.Poller()
|
||||
poller.register(handshake_socket, zmq.POLLIN)
|
||||
|
||||
remote_should_be_headless = not parallel_config.data_parallel_hybrid_lb \
|
||||
remote_should_be_headless = (
|
||||
not parallel_config.data_parallel_hybrid_lb
|
||||
and not parallel_config.data_parallel_external_lb
|
||||
)
|
||||
|
||||
if proc_manager is not None:
|
||||
for sentinel in proc_manager.sentinels():
|
||||
@@ -770,67 +813,73 @@ def wait_for_engine_startup(
|
||||
if not events:
|
||||
if any(conn_pending):
|
||||
logger.debug(
|
||||
"Waiting for %d local, %d remote core engine proc(s) "
|
||||
"to connect.", *conn_pending)
|
||||
"Waiting for %d local, %d remote core engine proc(s) to connect.",
|
||||
*conn_pending,
|
||||
)
|
||||
if any(start_pending):
|
||||
logger.debug(
|
||||
"Waiting for %d local, %d remote core engine proc(s) "
|
||||
"to start.", *start_pending)
|
||||
"Waiting for %d local, %d remote core engine proc(s) to start.",
|
||||
*start_pending,
|
||||
)
|
||||
continue
|
||||
if len(events) > 1 or events[0][0] != handshake_socket:
|
||||
# One of the local core processes exited.
|
||||
finished = proc_manager.finished_procs() if proc_manager else {}
|
||||
if coord_process is not None and coord_process.exitcode is not None:
|
||||
finished[coord_process.name] = coord_process.exitcode
|
||||
raise RuntimeError("Engine core initialization failed. "
|
||||
"See root cause above. "
|
||||
f"Failed core proc(s): {finished}")
|
||||
raise RuntimeError(
|
||||
"Engine core initialization failed. "
|
||||
"See root cause above. "
|
||||
f"Failed core proc(s): {finished}"
|
||||
)
|
||||
|
||||
# Receive HELLO and READY messages from the input socket.
|
||||
eng_identity, ready_msg_bytes = handshake_socket.recv_multipart()
|
||||
eng_index = int.from_bytes(eng_identity, "little")
|
||||
engine = next((e for e in core_engines if e.identity == eng_identity),
|
||||
None)
|
||||
engine = next((e for e in core_engines if e.identity == eng_identity), None)
|
||||
if engine is None:
|
||||
raise RuntimeError(f"Message from engine with unexpected data "
|
||||
f"parallel rank: {eng_index}")
|
||||
raise RuntimeError(
|
||||
f"Message from engine with unexpected data parallel rank: {eng_index}"
|
||||
)
|
||||
msg = msgspec.msgpack.decode(ready_msg_bytes)
|
||||
status, local, headless = msg["status"], msg["local"], msg["headless"]
|
||||
if local != engine.local:
|
||||
raise RuntimeError(f"{status} message from "
|
||||
f"{'local' if local else 'remote'} "
|
||||
f"engine {eng_index}, expected it to be "
|
||||
f"{'local' if engine.local else 'remote'}")
|
||||
raise RuntimeError(
|
||||
f"{status} message from "
|
||||
f"{'local' if local else 'remote'} "
|
||||
f"engine {eng_index}, expected it to be "
|
||||
f"{'local' if engine.local else 'remote'}"
|
||||
)
|
||||
|
||||
# Remote engines must be headless iff we aren't in hybrid dp lb mode.
|
||||
if not local and headless != remote_should_be_headless:
|
||||
if headless:
|
||||
raise RuntimeError(f"Remote engine {eng_index} must not use "
|
||||
f"--headless in external or hybrid dp lb "
|
||||
f"mode")
|
||||
raise RuntimeError(
|
||||
f"Remote engine {eng_index} must not use "
|
||||
f"--headless in external or hybrid dp lb "
|
||||
f"mode"
|
||||
)
|
||||
else:
|
||||
raise RuntimeError(f"Remote engine {eng_index} must use "
|
||||
f"--headless unless in external or hybrid "
|
||||
f"dp lb mode")
|
||||
raise RuntimeError(
|
||||
f"Remote engine {eng_index} must use "
|
||||
f"--headless unless in external or hybrid "
|
||||
f"dp lb mode"
|
||||
)
|
||||
|
||||
if status == "HELLO" and engine.state == CoreEngineState.NEW:
|
||||
|
||||
# Send init message with DP config info.
|
||||
init_message = msgspec.msgpack.encode(
|
||||
EngineHandshakeMetadata(
|
||||
addresses=addresses,
|
||||
parallel_config={
|
||||
"data_parallel_master_ip":
|
||||
parallel_config.data_parallel_master_ip,
|
||||
"data_parallel_master_port":
|
||||
parallel_config.data_parallel_master_port,
|
||||
"_data_parallel_master_port_list":
|
||||
parallel_config._data_parallel_master_port_list,
|
||||
"data_parallel_size":
|
||||
parallel_config.data_parallel_size,
|
||||
}))
|
||||
handshake_socket.send_multipart((eng_identity, init_message),
|
||||
copy=False)
|
||||
"data_parallel_master_ip": parallel_config.data_parallel_master_ip,
|
||||
"data_parallel_master_port": parallel_config.data_parallel_master_port,
|
||||
"_data_parallel_master_port_list": parallel_config._data_parallel_master_port_list,
|
||||
"data_parallel_size": parallel_config.data_parallel_size,
|
||||
},
|
||||
)
|
||||
)
|
||||
handshake_socket.send_multipart((eng_identity, init_message), copy=False)
|
||||
conn_pending[0 if local else 1] -= 1
|
||||
start_pending[0 if local else 1] += 1
|
||||
engine.state = CoreEngineState.CONNECTED
|
||||
@@ -846,15 +895,20 @@ def wait_for_engine_startup(
|
||||
# one of the engine handshakes, and passed to the local
|
||||
# front-end process in the response from the other.
|
||||
if addresses.frontend_stats_publish_address is None:
|
||||
addresses.frontend_stats_publish_address = msg.get(
|
||||
"dp_stats_address")
|
||||
addresses.frontend_stats_publish_address = msg.get("dp_stats_address")
|
||||
|
||||
start_pending[0 if local else 1] -= 1
|
||||
engine.state = CoreEngineState.READY
|
||||
else:
|
||||
raise RuntimeError(f"Unexpected {status} message for "
|
||||
f"{'local' if local else 'remote'} engine "
|
||||
f"{eng_index} in {engine.state} state.")
|
||||
raise RuntimeError(
|
||||
f"Unexpected {status} message for "
|
||||
f"{'local' if local else 'remote'} engine "
|
||||
f"{eng_index} in {engine.state} state."
|
||||
)
|
||||
|
||||
logger.debug("%s from %s core engine process %s.", status,
|
||||
"local" if local else "remote", eng_index)
|
||||
logger.debug(
|
||||
"%s from %s core engine process %s.",
|
||||
status,
|
||||
"local" if local else "remote",
|
||||
eng_index,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user