Add docstrings for LLMServer and related classes and examples (#142)
This commit is contained in:
@@ -1,14 +1,15 @@
|
||||
import openai
|
||||
|
||||
# Modify OpenAI's API key and API base to use CacheFlow's API server.
|
||||
openai.api_key = "EMPTY"
|
||||
openai.api_base = "http://localhost:8000/v1"
|
||||
model = "facebook/opt-125m"
|
||||
|
||||
# list models
|
||||
# Test list models API
|
||||
models = openai.Model.list()
|
||||
print(models)
|
||||
|
||||
# create a completion
|
||||
print("Models:", models)
|
||||
|
||||
# Test completion API
|
||||
stream = True
|
||||
completion = openai.Completion.create(
|
||||
model=model, prompt="A robot may not injure a human being", echo=False, n=2,
|
||||
@@ -19,4 +20,4 @@ if stream:
|
||||
for c in completion:
|
||||
print(c)
|
||||
else:
|
||||
print("completion:", completion)
|
||||
print("Completion result:", completion)
|
||||
|
||||
@@ -19,7 +19,7 @@ def main(args: argparse.Namespace):
|
||||
SamplingParams(n=3, best_of=3, use_beam_search=True, temperature=0.0)),
|
||||
]
|
||||
|
||||
# Run the server.
|
||||
# Run the server by calling `server.step()` manually.
|
||||
request_id = 0
|
||||
while True:
|
||||
# To test iteration-level scheduling, we add one request at each step.
|
||||
|
||||
Reference in New Issue
Block a user