[Platform] Add custom default max tokens (#18557)
Signed-off-by: Gabriel Marinho <gmarinho@ibm.com>
This commit is contained in:
@@ -5,13 +5,17 @@ import argparse
|
||||
import asyncio
|
||||
import functools
|
||||
import os
|
||||
from typing import Any, Optional
|
||||
import sys
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from fastapi import Request
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
from starlette.background import BackgroundTask, BackgroundTasks
|
||||
|
||||
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
|
||||
CompletionRequest)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -181,7 +185,6 @@ def _validate_truncation_size(
|
||||
|
||||
def show_filtered_argument_or_group_from_help(parser: argparse.ArgumentParser,
|
||||
subcommand_name: list[str]):
|
||||
import sys
|
||||
|
||||
# Only handle --help=<keyword> for the current subcommand.
|
||||
# Since subparser_init() runs for all subcommands during CLI setup,
|
||||
@@ -242,3 +245,18 @@ def show_filtered_argument_or_group_from_help(parser: argparse.ArgumentParser,
|
||||
print(f"\nNo group or parameter matching '{search_keyword}'")
|
||||
print("Tip: use `--help=listgroup` to view all groups.")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def get_max_tokens(max_model_len: int, request: Union[ChatCompletionRequest,
|
||||
CompletionRequest],
|
||||
input_length: int, default_sampling_params: dict) -> int:
|
||||
|
||||
max_tokens = getattr(request, "max_completion_tokens",
|
||||
None) or request.max_tokens
|
||||
default_max_tokens = max_model_len - input_length
|
||||
max_output_tokens = current_platform.get_max_output_tokens(input_length)
|
||||
|
||||
return min(val
|
||||
for val in (default_max_tokens, max_tokens, max_output_tokens,
|
||||
default_sampling_params.get("max_tokens"))
|
||||
if val is not None)
|
||||
|
||||
Reference in New Issue
Block a user