Add option to completion API to truncate prompt tokens (#3144)

This commit is contained in:
Thomas Parnell
2024-04-05 10:15:42 -07:00
committed by GitHub
parent cfaf49a167
commit 1d7c940d74
4 changed files with 41 additions and 8 deletions
+3 -1
View File
@@ -4,7 +4,7 @@ import time
from typing import Dict, List, Literal, Optional, Union
import torch
from pydantic import BaseModel, Field, model_validator
from pydantic import BaseModel, Field, conint, model_validator
from vllm.sampling_params import SamplingParams
from vllm.utils import random_uuid
@@ -229,6 +229,7 @@ class CompletionRequest(BaseModel):
min_tokens: Optional[int] = 0
skip_special_tokens: Optional[bool] = True
spaces_between_special_tokens: Optional[bool] = True
truncate_prompt_tokens: Optional[conint(ge=1)] = None
# doc: end-completion-sampling-params
# doc: begin-completion-extra-params
@@ -309,6 +310,7 @@ class CompletionRequest(BaseModel):
include_stop_str_in_output=self.include_stop_str_in_output,
length_penalty=self.length_penalty,
logits_processors=logits_processors,
truncate_prompt_tokens=self.truncate_prompt_tokens,
)
@model_validator(mode="before")