Re-enable the 80 char line width limit (#3305)
This commit is contained in:
@@ -1,7 +1,8 @@
|
||||
import asyncio
|
||||
import time
|
||||
from fastapi import Request
|
||||
from typing import AsyncGenerator, AsyncIterator, Callable, List, Optional, Dict, Tuple
|
||||
from typing import (AsyncGenerator, AsyncIterator, Callable, List, Optional,
|
||||
Dict, Tuple)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils import random_uuid
|
||||
from vllm.engine.async_llm_engine import AsyncLLMEngine
|
||||
@@ -16,7 +17,8 @@ from vllm.entrypoints.openai.protocol import (
|
||||
)
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.entrypoints.openai.serving_engine import OpenAIServing, LoRA
|
||||
from vllm.model_executor.guided_decoding import get_guided_decoding_logits_processor
|
||||
from vllm.model_executor.guided_decoding import (
|
||||
get_guided_decoding_logits_processor)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -44,9 +46,8 @@ def parse_prompt_format(prompt) -> Tuple[bool, list]:
|
||||
prompt_is_tokens = True
|
||||
prompts = prompt # case 4: array of token arrays
|
||||
else:
|
||||
raise ValueError(
|
||||
"prompt must be a string, array of strings, array of tokens, or array of token arrays"
|
||||
)
|
||||
raise ValueError("prompt must be a string, array of strings, "
|
||||
"array of tokens, or array of token arrays")
|
||||
return prompt_is_tokens, prompts
|
||||
|
||||
|
||||
@@ -156,7 +157,8 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
int, RequestOutput]] = merge_async_iterators(*generators)
|
||||
|
||||
# Similar to the OpenAI API, when n != best_of, we do not stream the
|
||||
# results. In addition, we do not stream the results when use beam search.
|
||||
# results. In addition, we do not stream the results when use
|
||||
# beam search.
|
||||
stream = (request.stream
|
||||
and (request.best_of is None or request.n == request.best_of)
|
||||
and not request.use_beam_search)
|
||||
@@ -223,7 +225,8 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
|
||||
for output in res.outputs:
|
||||
i = output.index + prompt_idx * request.n
|
||||
# TODO(simon): optimize the performance by avoiding full text O(n^2) sending.
|
||||
# TODO(simon): optimize the performance by avoiding full
|
||||
# text O(n^2) sending.
|
||||
|
||||
if request.echo and request.max_tokens == 0:
|
||||
# only return the prompt
|
||||
@@ -231,11 +234,12 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
delta_token_ids = res.prompt_token_ids
|
||||
top_logprobs = res.prompt_logprobs
|
||||
has_echoed[i] = True
|
||||
elif request.echo and request.max_tokens > 0 and not has_echoed[
|
||||
i]:
|
||||
elif (request.echo and request.max_tokens > 0
|
||||
and not has_echoed[i]):
|
||||
# echo the prompt and first token
|
||||
delta_text = res.prompt + output.text
|
||||
delta_token_ids = res.prompt_token_ids + output.token_ids
|
||||
delta_token_ids = (res.prompt_token_ids +
|
||||
output.token_ids)
|
||||
top_logprobs = res.prompt_logprobs + (output.logprobs
|
||||
or [])
|
||||
has_echoed[i] = True
|
||||
@@ -248,7 +252,9 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
i]:] if output.logprobs else None
|
||||
|
||||
if request.logprobs is not None:
|
||||
assert top_logprobs is not None, "top_logprobs must be provided when logprobs is requested"
|
||||
assert top_logprobs is not None, (
|
||||
"top_logprobs must be provided when logprobs "
|
||||
"is requested")
|
||||
logprobs = self._create_logprobs(
|
||||
token_ids=delta_token_ids,
|
||||
top_logprobs=top_logprobs,
|
||||
|
||||
Reference in New Issue
Block a user