[Refactor] Consolidate sequence normalization and enc-dec parsing (#33928)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -28,6 +28,8 @@ from vllm.outputs import STREAM_FINISHED, PoolingRequestOutput, RequestOutput
|
||||
from vllm.plugins.io_processors import get_io_processor
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.renderers import BaseRenderer, merge_kwargs
|
||||
from vllm.renderers.inputs import DictPrompt, TokPrompt
|
||||
from vllm.renderers.inputs.preprocess import extract_prompt_components
|
||||
from vllm.sampling_params import RequestOutputKind, SamplingParams
|
||||
from vllm.tasks import SupportedTask
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
@@ -42,7 +44,6 @@ from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
|
||||
from vllm.v1.engine.input_processor import InputProcessor
|
||||
from vllm.v1.engine.output_processor import OutputProcessor, RequestOutputCollector
|
||||
from vllm.v1.engine.parallel_sampling import ParentRequest
|
||||
from vllm.v1.engine.utils import get_prompt_text
|
||||
from vllm.v1.executor import Executor
|
||||
from vllm.v1.metrics.loggers import (
|
||||
StatLoggerFactory,
|
||||
@@ -284,7 +285,11 @@ class AsyncLLM(EngineClient):
|
||||
async def add_request(
|
||||
self,
|
||||
request_id: str,
|
||||
prompt: EngineCoreRequest | PromptType | AsyncGenerator[StreamingInput, None],
|
||||
prompt: EngineCoreRequest
|
||||
| PromptType
|
||||
| DictPrompt
|
||||
| TokPrompt
|
||||
| AsyncGenerator[StreamingInput, None],
|
||||
params: SamplingParams | PoolingParams,
|
||||
arrival_time: float | None = None,
|
||||
lora_request: LoRARequest | None = None,
|
||||
@@ -367,7 +372,7 @@ class AsyncLLM(EngineClient):
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
supported_tasks=await self.get_supported_tasks(),
|
||||
)
|
||||
prompt_text = get_prompt_text(prompt)
|
||||
prompt_text, _, _ = extract_prompt_components(self.model_config, prompt)
|
||||
|
||||
self.input_processor.assign_request_id(request)
|
||||
|
||||
@@ -484,7 +489,9 @@ class AsyncLLM(EngineClient):
|
||||
raise ValueError(
|
||||
"prompt_embeds not supported for streaming inputs"
|
||||
)
|
||||
prompt_text = get_prompt_text(input_chunk.prompt)
|
||||
prompt_text, _, _ = extract_prompt_components(
|
||||
self.model_config, input_chunk.prompt
|
||||
)
|
||||
await self._add_request(req, prompt_text, None, 0, queue)
|
||||
except (asyncio.CancelledError, GeneratorExit):
|
||||
cancelled = True
|
||||
@@ -528,7 +535,11 @@ class AsyncLLM(EngineClient):
|
||||
# re-multiplexed in the API server anyhow.
|
||||
async def generate(
|
||||
self,
|
||||
prompt: EngineCoreRequest | PromptType | AsyncGenerator[StreamingInput, None],
|
||||
prompt: EngineCoreRequest
|
||||
| PromptType
|
||||
| DictPrompt
|
||||
| TokPrompt
|
||||
| AsyncGenerator[StreamingInput, None],
|
||||
sampling_params: SamplingParams,
|
||||
request_id: str,
|
||||
*,
|
||||
@@ -769,7 +780,7 @@ class AsyncLLM(EngineClient):
|
||||
|
||||
async def encode(
|
||||
self,
|
||||
prompt: PromptType,
|
||||
prompt: PromptType | DictPrompt | TokPrompt,
|
||||
pooling_params: PoolingParams,
|
||||
request_id: str,
|
||||
lora_request: LoRARequest | None = None,
|
||||
|
||||
@@ -7,14 +7,13 @@ from typing import Any, Literal, cast
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.inputs import (
|
||||
from vllm.inputs.data import (
|
||||
ProcessorInputs,
|
||||
PromptType,
|
||||
SingletonInputs,
|
||||
SingletonPrompt,
|
||||
TextPrompt,
|
||||
)
|
||||
from vllm.inputs.parse import is_explicit_encoder_decoder_prompt, split_enc_dec_inputs
|
||||
from vllm.inputs.parse import split_enc_dec_inputs
|
||||
from vllm.inputs.preprocess import InputPreprocessor
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
@@ -30,6 +29,7 @@ from vllm.multimodal.processing.context import set_request_id
|
||||
from vllm.multimodal.utils import argsort_mm_positions
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.renderers import BaseRenderer
|
||||
from vllm.renderers.inputs import DictPrompt, TokPrompt
|
||||
from vllm.sampling_params import _SAMPLING_EPS, SamplingParams
|
||||
from vllm.tasks import POOLING_TASKS, SupportedTask
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
@@ -243,8 +243,8 @@ class InputProcessor:
|
||||
return mm_processor.info.parse_mm_data(mm_data)
|
||||
|
||||
def _validate_singleton_mm_uuids(self, prompt: SingletonPrompt) -> None:
|
||||
if isinstance(prompt, str):
|
||||
prompt = TextPrompt(prompt=prompt)
|
||||
if not isinstance(prompt, dict):
|
||||
return
|
||||
|
||||
mm_data = cast(MultiModalDataDict, prompt.get("multi_modal_data") or {})
|
||||
mm_uuids = cast(MultiModalUUIDDict, prompt.get("multi_modal_uuids") or {})
|
||||
@@ -297,7 +297,7 @@ class InputProcessor:
|
||||
f"multi_modal_uuids[{modality!r}] is missing."
|
||||
)
|
||||
|
||||
def _validate_mm_uuids(self, prompt: PromptType) -> None:
|
||||
def _validate_mm_uuids(self, prompt: PromptType | DictPrompt | TokPrompt) -> None:
|
||||
"""
|
||||
Validate that user-provided multi_modal_uuids align with
|
||||
multi_modal_data in the incoming request prompt(s).
|
||||
@@ -305,10 +305,10 @@ class InputProcessor:
|
||||
auto-hashed downstream.
|
||||
"""
|
||||
|
||||
if is_explicit_encoder_decoder_prompt(prompt):
|
||||
self._validate_singleton_mm_uuids(prompt["encoder_prompt"])
|
||||
if isinstance(prompt, dict) and "encoder_prompt" in prompt:
|
||||
self._validate_singleton_mm_uuids(prompt["encoder_prompt"]) # type: ignore[typeddict-item]
|
||||
|
||||
if (dec_prompt := prompt["decoder_prompt"]) is not None:
|
||||
if (dec_prompt := prompt["decoder_prompt"]) is not None: # type: ignore[typeddict-item]
|
||||
self._validate_singleton_mm_uuids(dec_prompt)
|
||||
else:
|
||||
self._validate_singleton_mm_uuids(prompt)
|
||||
@@ -449,21 +449,23 @@ class InputProcessor:
|
||||
def _extract_singleton_mm_data(
|
||||
self, prompt: SingletonPrompt
|
||||
) -> MultiModalDataDict | None:
|
||||
if isinstance(prompt, str):
|
||||
if not isinstance(prompt, dict):
|
||||
return None
|
||||
|
||||
return prompt.get("multi_modal_data") # type: ignore[return-value]
|
||||
return prompt.get("multi_modal_data")
|
||||
|
||||
def _extract_mm_data(self, prompt: PromptType) -> MultiModalDataDict | None:
|
||||
if is_explicit_encoder_decoder_prompt(prompt):
|
||||
return self._extract_singleton_mm_data(prompt["encoder_prompt"])
|
||||
def _extract_mm_data(
|
||||
self, prompt: PromptType | DictPrompt | TokPrompt
|
||||
) -> MultiModalDataDict | None:
|
||||
if isinstance(prompt, dict) and "encoder_prompt" in prompt:
|
||||
return self._extract_singleton_mm_data(prompt["encoder_prompt"]) # type: ignore[typeddict-item]
|
||||
else:
|
||||
return self._extract_singleton_mm_data(prompt)
|
||||
|
||||
def _maybe_build_mm_uuids(
|
||||
self,
|
||||
request_id: str,
|
||||
prompt: PromptType,
|
||||
prompt: PromptType | DictPrompt | TokPrompt,
|
||||
) -> MultiModalUUIDDict | None:
|
||||
"""Build per-item multimodal hash overrides when enabled. In this case,
|
||||
multimodal data items are identified by their request id, modality and
|
||||
@@ -519,7 +521,7 @@ class InputProcessor:
|
||||
def process_inputs(
|
||||
self,
|
||||
request_id: str,
|
||||
prompt: PromptType,
|
||||
prompt: PromptType | DictPrompt | TokPrompt,
|
||||
params: SamplingParams | PoolingParams,
|
||||
arrival_time: float | None = None,
|
||||
lora_request: LoRARequest | None = None,
|
||||
|
||||
@@ -22,6 +22,8 @@ from vllm.outputs import PoolingRequestOutput, RequestOutput
|
||||
from vllm.plugins.io_processors import get_io_processor
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.renderers import BaseRenderer
|
||||
from vllm.renderers.inputs import DictPrompt, TokPrompt
|
||||
from vllm.renderers.inputs.preprocess import extract_prompt_components
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.tasks import SupportedTask
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
@@ -32,7 +34,6 @@ from vllm.v1.engine.core_client import EngineCoreClient
|
||||
from vllm.v1.engine.input_processor import InputProcessor
|
||||
from vllm.v1.engine.output_processor import OutputProcessor
|
||||
from vllm.v1.engine.parallel_sampling import ParentRequest
|
||||
from vllm.v1.engine.utils import get_prompt_text
|
||||
from vllm.v1.executor import Executor
|
||||
from vllm.v1.metrics.loggers import StatLoggerFactory, StatLoggerManager
|
||||
from vllm.v1.metrics.reader import Metric, get_metrics_snapshot
|
||||
@@ -216,7 +217,7 @@ class LLMEngine:
|
||||
def add_request(
|
||||
self,
|
||||
request_id: str,
|
||||
prompt: EngineCoreRequest | PromptType,
|
||||
prompt: EngineCoreRequest | PromptType | DictPrompt | TokPrompt,
|
||||
params: SamplingParams | PoolingParams,
|
||||
arrival_time: float | None = None,
|
||||
lora_request: LoRARequest | None = None,
|
||||
@@ -251,7 +252,7 @@ class LLMEngine:
|
||||
priority,
|
||||
supported_tasks=self.get_supported_tasks(),
|
||||
)
|
||||
prompt_text = get_prompt_text(prompt)
|
||||
prompt_text, _, _ = extract_prompt_components(self.model_config, prompt)
|
||||
|
||||
self.input_processor.assign_request_id(request)
|
||||
|
||||
|
||||
@@ -17,8 +17,6 @@ import zmq
|
||||
|
||||
from vllm import envs
|
||||
from vllm.config import CacheConfig, ParallelConfig, VllmConfig
|
||||
from vllm.inputs import PromptType
|
||||
from vllm.inputs.parse import get_prompt_components
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.ray.ray_env import get_env_vars_to_copy
|
||||
@@ -226,10 +224,6 @@ def get_device_indices(
|
||||
return value
|
||||
|
||||
|
||||
def get_prompt_text(prompt: PromptType) -> str | None:
|
||||
return get_prompt_components(prompt)[0]
|
||||
|
||||
|
||||
class CoreEngineActorManager:
|
||||
"""
|
||||
Utility class to handle creation, readiness, and shutdown
|
||||
|
||||
Reference in New Issue
Block a user