[Frontend] Cleanup serving engine (#33103)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -17,8 +17,10 @@ from starlette.background import BackgroundTask, BackgroundTasks
|
||||
|
||||
from vllm import envs
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.inputs import EmbedsPrompt, TokensPrompt
|
||||
from vllm.logger import current_formatter_type, init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils import length_from_prompt_token_ids_or_embeds
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -32,11 +34,15 @@ if TYPE_CHECKING:
|
||||
StreamOptions,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
|
||||
from vllm.entrypoints.openai.responses.protocol import (
|
||||
ResponsesRequest,
|
||||
)
|
||||
else:
|
||||
ChatCompletionRequest = object
|
||||
CompletionRequest = object
|
||||
StreamOptions = object
|
||||
LoRAModulePath = object
|
||||
ResponsesRequest = object
|
||||
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -211,11 +217,26 @@ def _validate_truncation_size(
|
||||
|
||||
def get_max_tokens(
|
||||
max_model_len: int,
|
||||
request: "ChatCompletionRequest | CompletionRequest",
|
||||
input_length: int,
|
||||
request: "CompletionRequest | ChatCompletionRequest | ResponsesRequest",
|
||||
prompt: TokensPrompt | EmbedsPrompt,
|
||||
default_sampling_params: dict,
|
||||
) -> int:
|
||||
max_tokens = getattr(request, "max_completion_tokens", None) or request.max_tokens
|
||||
# NOTE: Avoid isinstance() for better efficiency
|
||||
max_tokens: int | None = None
|
||||
if max_tokens is None:
|
||||
# ChatCompletionRequest
|
||||
max_tokens = getattr(request, "max_completion_tokens", None)
|
||||
if max_tokens is None:
|
||||
# ResponsesRequest
|
||||
max_tokens = getattr(request, "max_output_tokens", None)
|
||||
if max_tokens is None:
|
||||
# CompletionRequest (also a fallback for ChatCompletionRequest)
|
||||
max_tokens = getattr(request, "max_tokens", None)
|
||||
|
||||
input_length = length_from_prompt_token_ids_or_embeds(
|
||||
prompt.get("prompt_token_ids"), # type: ignore[arg-type]
|
||||
prompt.get("prompt_embeds"), # type: ignore[arg-type]
|
||||
)
|
||||
default_max_tokens = max_model_len - input_length
|
||||
max_output_tokens = current_platform.get_max_output_tokens(input_length)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user