[Refactor] Consolidate sequence normalization and enc-dec parsing (#33928)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -28,6 +28,8 @@ from vllm.outputs import STREAM_FINISHED, PoolingRequestOutput, RequestOutput
|
||||
from vllm.plugins.io_processors import get_io_processor
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.renderers import BaseRenderer, merge_kwargs
|
||||
from vllm.renderers.inputs import DictPrompt, TokPrompt
|
||||
from vllm.renderers.inputs.preprocess import extract_prompt_components
|
||||
from vllm.sampling_params import RequestOutputKind, SamplingParams
|
||||
from vllm.tasks import SupportedTask
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
@@ -42,7 +44,6 @@ from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
|
||||
from vllm.v1.engine.input_processor import InputProcessor
|
||||
from vllm.v1.engine.output_processor import OutputProcessor, RequestOutputCollector
|
||||
from vllm.v1.engine.parallel_sampling import ParentRequest
|
||||
from vllm.v1.engine.utils import get_prompt_text
|
||||
from vllm.v1.executor import Executor
|
||||
from vllm.v1.metrics.loggers import (
|
||||
StatLoggerFactory,
|
||||
@@ -284,7 +285,11 @@ class AsyncLLM(EngineClient):
|
||||
async def add_request(
|
||||
self,
|
||||
request_id: str,
|
||||
prompt: EngineCoreRequest | PromptType | AsyncGenerator[StreamingInput, None],
|
||||
prompt: EngineCoreRequest
|
||||
| PromptType
|
||||
| DictPrompt
|
||||
| TokPrompt
|
||||
| AsyncGenerator[StreamingInput, None],
|
||||
params: SamplingParams | PoolingParams,
|
||||
arrival_time: float | None = None,
|
||||
lora_request: LoRARequest | None = None,
|
||||
@@ -367,7 +372,7 @@ class AsyncLLM(EngineClient):
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
supported_tasks=await self.get_supported_tasks(),
|
||||
)
|
||||
prompt_text = get_prompt_text(prompt)
|
||||
prompt_text, _, _ = extract_prompt_components(self.model_config, prompt)
|
||||
|
||||
self.input_processor.assign_request_id(request)
|
||||
|
||||
@@ -484,7 +489,9 @@ class AsyncLLM(EngineClient):
|
||||
raise ValueError(
|
||||
"prompt_embeds not supported for streaming inputs"
|
||||
)
|
||||
prompt_text = get_prompt_text(input_chunk.prompt)
|
||||
prompt_text, _, _ = extract_prompt_components(
|
||||
self.model_config, input_chunk.prompt
|
||||
)
|
||||
await self._add_request(req, prompt_text, None, 0, queue)
|
||||
except (asyncio.CancelledError, GeneratorExit):
|
||||
cancelled = True
|
||||
@@ -528,7 +535,11 @@ class AsyncLLM(EngineClient):
|
||||
# re-multiplexed in the API server anyhow.
|
||||
async def generate(
|
||||
self,
|
||||
prompt: EngineCoreRequest | PromptType | AsyncGenerator[StreamingInput, None],
|
||||
prompt: EngineCoreRequest
|
||||
| PromptType
|
||||
| DictPrompt
|
||||
| TokPrompt
|
||||
| AsyncGenerator[StreamingInput, None],
|
||||
sampling_params: SamplingParams,
|
||||
request_id: str,
|
||||
*,
|
||||
@@ -769,7 +780,7 @@ class AsyncLLM(EngineClient):
|
||||
|
||||
async def encode(
|
||||
self,
|
||||
prompt: PromptType,
|
||||
prompt: PromptType | DictPrompt | TokPrompt,
|
||||
pooling_params: PoolingParams,
|
||||
request_id: str,
|
||||
lora_request: LoRARequest | None = None,
|
||||
|
||||
Reference in New Issue
Block a user