Add docstrings for LLMServer and related classes and examples (#142)

This commit is contained in:
Zhuohan Li
2023-06-07 18:25:20 +08:00
committed by GitHub
parent e38074b1e6
commit 4298374265
10 changed files with 212 additions and 18 deletions
+6 -5
View File
@@ -1,14 +1,15 @@
import openai
# Modify OpenAI's API key and API base to use CacheFlow's API server.
openai.api_key = "EMPTY"
openai.api_base = "http://localhost:8000/v1"
model = "facebook/opt-125m"
# list models
# Test list models API
models = openai.Model.list()
print(models)
# create a completion
print("Models:", models)
# Test completion API
stream = True
completion = openai.Completion.create(
model=model, prompt="A robot may not injure a human being", echo=False, n=2,
@@ -19,4 +20,4 @@ if stream:
for c in completion:
print(c)
else:
print("completion:", completion)
print("Completion result:", completion)
+1 -1
View File
@@ -19,7 +19,7 @@ def main(args: argparse.Namespace):
SamplingParams(n=3, best_of=3, use_beam_search=True, temperature=0.0)),
]
# Run the server.
# Run the server by calling `server.step()` manually.
request_id = 0
while True:
# To test iteration-level scheduling, we add one request at each step.