[P/D] Support CPU Transfer in NixlConnector (#18293)
Signed-off-by: Juncheng Gu <juncgu@gmail.com> Signed-off-by: Richard Liu <ricliu@google.com> Co-authored-by: Richard Liu <39319471+richardsliu@users.noreply.github.com> Co-authored-by: Richard Liu <ricliu@google.com>
This commit is contained in:
co-authored by
Richard Liu
Richard Liu
parent
1e9ea8e69d
commit
6066284914
@@ -0,0 +1,162 @@
|
||||
#!/bin/bash
|
||||
set -xe
|
||||
|
||||
# Hosts / ports
|
||||
PREFILL_HOST=${PREFILL_HOST:-"localhost"}
|
||||
PREFILL_PORT=${PREFILL_PORT:-8100}
|
||||
PREFILL_NIXL_SIDE_PORT=${PREFILL_NIXL_SIDE_PORT:-5577}
|
||||
DECODE_HOST=${DECODE_HOST:-"localhost"}
|
||||
DECODE_PORT=${DECODE_PORT:-8200}
|
||||
PROXY_HOST=${PROXY_HOST:-"localhost"}
|
||||
PROXY_PORT=${PROXY_PORT:-8192}
|
||||
BASELINE_HOST=${BASELINE_HOST:-"localhost"}
|
||||
BASELINE_PORT=${BASELINE_PORT:-9290}
|
||||
|
||||
|
||||
# Model to run.
|
||||
MODEL_NAME=${MODEL_NAME:-"meta-llama/Llama-3.2-3B-Instruct"}
|
||||
MAX_MODEL_LEN=${MAX_MODEL_LEN:-1024}
|
||||
BLOCK_SIZE=${BLOCK_SIZE:-32}
|
||||
|
||||
|
||||
# execution env
|
||||
GIT_ROOT=$(git rev-parse --show-toplevel)
|
||||
EXP_ROOT="${GIT_ROOT}/tests/v1/kv_connector/nixl_integration"
|
||||
CONDA_PATH=${CONDA_PATH:-"/home/${USER}/anaconda3"}
|
||||
CONDA_ENV_NAME=${CONDA_ENV_NAME:-"nixl"}
|
||||
|
||||
OUTPUT_FILE=${OUTPUT_FILE:-"${EXP_ROOT}/.tpu_accuracy_test_outputs.txt"}
|
||||
|
||||
# Trap the SIGINT signal (triggered by Ctrl+C)
|
||||
trap 'kill $(jobs -pr)' SIGINT SIGTERM EXIT
|
||||
|
||||
|
||||
# Waits for vLLM server to start.
|
||||
wait_for_server() {
|
||||
local host=$1
|
||||
local port=$2
|
||||
timeout 1200 bash -c "
|
||||
until curl -s ${host}:${port}/v1/completions > /dev/null; do
|
||||
sleep 1
|
||||
done" && return 0 || return 1
|
||||
}
|
||||
|
||||
# Cleanup function
|
||||
cleanup() {
|
||||
echo "Caught Ctrl+C, cleaning up..."
|
||||
# Cleanup commands
|
||||
pgrep python | xargs kill -9 || true
|
||||
# pkill -f python || true
|
||||
echo "Cleanup complete. Exiting."
|
||||
}
|
||||
|
||||
launch_baseline() {
|
||||
BASELINE_BASE_CMD="source ${CONDA_PATH}/bin/activate ${CONDA_ENV_NAME};
|
||||
VLLM_LOGGING_LEVEL=DEBUG \
|
||||
VLLM_USE_V1=1 \
|
||||
PJRT_DEVICE=TPU \
|
||||
VLLM_WORKER_MULTIPROC_METHOD=spawn \
|
||||
VLLM_ENABLE_V1_MULTIPROCESSING=0 vllm serve $MODEL_NAME \
|
||||
--host ${BASELINE_HOST} \
|
||||
--port ${BASELINE_PORT} \
|
||||
--max-model-len ${MAX_MODEL_LEN}\
|
||||
--seed 42 \
|
||||
--block-size ${BLOCK_SIZE} \
|
||||
--gpu-memory-utilization 0.5 \
|
||||
--disable-log-requests \
|
||||
--enforce-eager"
|
||||
echo ${BASELINE_BASE_CMD}
|
||||
ssh -tt ${BASELINE_HOST} "${BASELINE_BASE_CMD}" &
|
||||
}
|
||||
|
||||
launch_pd() {
|
||||
PREFILL_BASE_CMD="source ${CONDA_PATH}/bin/activate ${CONDA_ENV_NAME};
|
||||
UCX_TLS=tcp \
|
||||
VLLM_MULTIPROC_EXECUTE_MODEL_TIMEOUT_S=200 \
|
||||
VLLM_LOGGING_LEVEL=DEBUG \
|
||||
VLLM_USE_V1=1 \
|
||||
VLLM_NIXL_SIDE_CHANNEL_HOST=${PREFILL_HOST} \
|
||||
VLLM_NIXL_SIDE_CHANNEL_PORT=${PREFILL_NIXL_SIDE_PORT} \
|
||||
PJRT_DEVICE=TPU \
|
||||
VLLM_WORKER_MULTIPROC_METHOD=spawn \
|
||||
VLLM_ENABLE_V1_MULTIPROCESSING=0 vllm serve $MODEL_NAME \
|
||||
--host ${PREFILL_HOST} \
|
||||
--port ${PREFILL_PORT} \
|
||||
--max-model-len ${MAX_MODEL_LEN}\
|
||||
--seed 42 \
|
||||
--block-size ${BLOCK_SIZE} \
|
||||
--enforce-eager \
|
||||
--gpu-memory-utilization 0.5 \
|
||||
--disable-log-requests \
|
||||
--kv-transfer-config '{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_buffer_device\":\"cpu\"}'"
|
||||
|
||||
|
||||
DECODE_BASE_CMD="source ${CONDA_PATH}/bin/activate ${CONDA_ENV_NAME};
|
||||
UCX_TLS=tcp \
|
||||
VLLM_MULTIPROC_EXECUTE_MODEL_TIMEOUT_S=200 \
|
||||
VLLM_LOGGING_LEVEL=DEBUG \
|
||||
VLLM_USE_V1=1 \
|
||||
PJRT_DEVICE=TPU \
|
||||
VLLM_WORKER_MULTIPROC_METHOD=spawn \
|
||||
VLLM_ENABLE_V1_MULTIPROCESSING=0 vllm serve $MODEL_NAME \
|
||||
--host ${DECODE_HOST} \
|
||||
--port ${DECODE_PORT} \
|
||||
--max-model-len ${MAX_MODEL_LEN}\
|
||||
--seed 42 \
|
||||
--block-size ${BLOCK_SIZE} \
|
||||
--enforce-eager \
|
||||
--gpu-memory-utilization 0.5 \
|
||||
--disable-log-requests \
|
||||
--kv-transfer-config '{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_buffer_device\":\"cpu\"}'"
|
||||
|
||||
echo ${PREFILL_BASE_CMD}
|
||||
echo ${DECODE_BASE_CMD}
|
||||
sleep 2
|
||||
|
||||
# execute on hosts
|
||||
ssh -tt ${PREFILL_HOST} "${PREFILL_BASE_CMD}" &
|
||||
ssh -tt ${DECODE_HOST} "${DECODE_BASE_CMD}" &
|
||||
sleep 1
|
||||
wait_for_server ${PREFILL_HOST} ${PREFILL_PORT}
|
||||
sleep 1
|
||||
wait_for_server ${DECODE_HOST} ${DECODE_PORT}
|
||||
sleep 1
|
||||
}
|
||||
|
||||
launch_pd_proxy(){
|
||||
PROXY_BASE_CMD="source ${CONDA_PATH}/bin/activate ${CONDA_ENV_NAME};
|
||||
python3 ${EXP_ROOT}/toy_proxy_server.py \
|
||||
--prefiller-host ${PREFILL_HOST} --prefiller-port ${PREFILL_PORT} \
|
||||
--decoder-host ${DECODE_HOST} --decoder-port ${DECODE_PORT} \
|
||||
--host=${PROXY_HOST} --port ${PROXY_PORT}"
|
||||
echo ${PROXY_BASE_CMD}
|
||||
ssh -tt ${PROXY_HOST} "${PROXY_BASE_CMD}" &
|
||||
}
|
||||
|
||||
run_tests(){
|
||||
local service_url=$1
|
||||
local mode=$2
|
||||
python3 ${EXP_ROOT}/test_disagg_accuracy.py --service_url=${service_url} --model_name=${MODEL_NAME} --mode=${mode} --file_name=${OUTPUT_FILE}
|
||||
}
|
||||
|
||||
|
||||
# run non-disagg. baseline & save outputs
|
||||
launch_baseline
|
||||
sleep 2
|
||||
wait_for_server ${BASELINE_HOST} ${BASELINE_PORT}
|
||||
run_tests "http://${BASELINE_HOST}:${BASELINE_PORT}" "baseline"
|
||||
cleanup
|
||||
sleep 10
|
||||
|
||||
|
||||
# run disagg. & do exact-match with the outputs from baseline
|
||||
launch_pd
|
||||
launch_pd_proxy
|
||||
sleep 10
|
||||
run_tests "http://${PROXY_HOST}:${PROXY_PORT}" "disagg"
|
||||
echo "-----P/D success----"
|
||||
|
||||
rm ${OUTPUT_FILE}
|
||||
cleanup
|
||||
|
||||
exit 0
|
||||
@@ -0,0 +1,128 @@
|
||||
#!/bin/bash
|
||||
set -xe
|
||||
|
||||
# Hosts / ports
|
||||
PREFILL_HOST=${PREFILL_HOST:-"localhost"}
|
||||
PREFILL_PORT=${PREFILL_PORT:-8100}
|
||||
PREFILL_NIXL_SIDE_PORT=${PREFILL_NIXL_SIDE_PORT:-5577}
|
||||
DECODE_HOST=${DECODE_HOST:-"localhost"}
|
||||
DECODE_PORT=${DECODE_PORT:-8200}
|
||||
PROXY_HOST=${PROXY_HOST:-"localhost"}
|
||||
PROXY_PORT=${PROXY_PORT:-8192}
|
||||
BASELINE_HOST=${BASELINE_HOST:-"localhost"}
|
||||
BASELINE_PORT=${BASELINE_PORT:-9290}
|
||||
|
||||
|
||||
# Model to run.
|
||||
MODEL_NAME=${MODEL_NAME:-"meta-llama/Llama-3.2-3B-Instruct"}
|
||||
MAX_MODEL_LEN=${MAX_MODEL_LEN:-1024}
|
||||
BLOCK_SIZE=${BLOCK_SIZE:-32}
|
||||
|
||||
|
||||
# execution env
|
||||
GIT_ROOT=$(git rev-parse --show-toplevel)
|
||||
EXP_ROOT="${GIT_ROOT}/tests/v1/kv_connector/nixl_integration"
|
||||
CONDA_PATH=${CONDA_PATH:-"/home/${USER}/anaconda3"}
|
||||
CONDA_ENV_NAME=${CONDA_ENV_NAME:-"nixl"}
|
||||
|
||||
OUTPUT_FILE=${OUTPUT_FILE:-"${EXP_ROOT}/.tpu_accuracy_test_outputs.txt"}
|
||||
|
||||
# Trap the SIGINT signal (triggered by Ctrl+C)
|
||||
trap 'kill $(jobs -pr)' SIGINT SIGTERM EXIT
|
||||
|
||||
# Waits for vLLM server to start.
|
||||
wait_for_server() {
|
||||
local host=$1
|
||||
local port=$2
|
||||
timeout 1200 bash -c "
|
||||
until curl -s ${host}:${port}/v1/completions > /dev/null; do
|
||||
sleep 1
|
||||
done" && return 0 || return 1
|
||||
}
|
||||
|
||||
# Cleanup function
|
||||
cleanup() {
|
||||
echo "Caught Ctrl+C, cleaning up..."
|
||||
# Cleanup commands
|
||||
pgrep python | xargs kill -9 || true
|
||||
# pkill -f python || true
|
||||
echo "Cleanup complete. Exiting."
|
||||
}
|
||||
|
||||
|
||||
launch_pd() {
|
||||
PREFILL_BASE_CMD="source ${CONDA_PATH}/bin/activate ${CONDA_ENV_NAME};
|
||||
UCX_TLS=tcp \
|
||||
VLLM_MULTIPROC_EXECUTE_MODEL_TIMEOUT_S=200 \
|
||||
VLLM_LOGGING_LEVEL=DEBUG \
|
||||
VLLM_USE_V1=1 \
|
||||
VLLM_NIXL_SIDE_CHANNEL_HOST=${PREFILL_HOST} \
|
||||
VLLM_NIXL_SIDE_CHANNEL_PORT=${PREFILL_NIXL_SIDE_PORT} \
|
||||
PJRT_DEVICE=TPU \
|
||||
VLLM_WORKER_MULTIPROC_METHOD=spawn \
|
||||
VLLM_ENABLE_V1_MULTIPROCESSING=0 vllm serve $MODEL_NAME \
|
||||
--host ${PREFILL_HOST} \
|
||||
--port ${PREFILL_PORT} \
|
||||
--max-model-len ${MAX_MODEL_LEN}\
|
||||
--seed 42 \
|
||||
--block-size ${BLOCK_SIZE} \
|
||||
--enforce-eager \
|
||||
--gpu-memory-utilization 0.5 \
|
||||
--disable-log-requests \
|
||||
--kv-transfer-config '{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_buffer_device\":\"cpu\"}'"
|
||||
|
||||
|
||||
DECODE_BASE_CMD="source ${CONDA_PATH}/bin/activate ${CONDA_ENV_NAME};
|
||||
UCX_TLS=tcp \
|
||||
VLLM_MULTIPROC_EXECUTE_MODEL_TIMEOUT_S=200 \
|
||||
VLLM_LOGGING_LEVEL=DEBUG \
|
||||
VLLM_USE_V1=1 \
|
||||
PJRT_DEVICE=TPU \
|
||||
VLLM_WORKER_MULTIPROC_METHOD=spawn \
|
||||
VLLM_ENABLE_V1_MULTIPROCESSING=0 vllm serve $MODEL_NAME \
|
||||
--host ${DECODE_HOST} \
|
||||
--port ${DECODE_PORT} \
|
||||
--max-model-len ${MAX_MODEL_LEN}\
|
||||
--seed 42 \
|
||||
--block-size ${BLOCK_SIZE} \
|
||||
--enforce-eager \
|
||||
--gpu-memory-utilization 0.5 \
|
||||
--disable-log-requests \
|
||||
--kv-transfer-config '{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_buffer_device\":\"cpu\"}'"
|
||||
|
||||
echo ${PREFILL_BASE_CMD}
|
||||
echo ${DECODE_BASE_CMD}
|
||||
sleep 2
|
||||
|
||||
# execute on hosts
|
||||
ssh -tt ${PREFILL_HOST} "${PREFILL_BASE_CMD}" &
|
||||
ssh -tt ${DECODE_HOST} "${DECODE_BASE_CMD}" &
|
||||
sleep 1
|
||||
wait_for_server ${PREFILL_HOST} ${PREFILL_PORT}
|
||||
sleep 1
|
||||
wait_for_server ${DECODE_HOST} ${DECODE_PORT}
|
||||
sleep 1
|
||||
}
|
||||
|
||||
launch_pd_proxy(){
|
||||
PROXY_BASE_CMD="source ${CONDA_PATH}/bin/activate ${CONDA_ENV_NAME};
|
||||
python3 ${EXP_ROOT}/toy_proxy_server.py \
|
||||
--prefiller-host ${PREFILL_HOST} --prefiller-port ${PREFILL_PORT} \
|
||||
--decoder-host ${DECODE_HOST} --decoder-port ${DECODE_PORT} \
|
||||
--host=${PROXY_HOST} --port ${PROXY_PORT}"
|
||||
echo ${PROXY_BASE_CMD}
|
||||
ssh -tt ${PROXY_HOST} "${PROXY_BASE_CMD}" &
|
||||
}
|
||||
|
||||
|
||||
# run disagg. & do exact-match with the outputs from baseline
|
||||
launch_pd
|
||||
launch_pd_proxy
|
||||
sleep 10
|
||||
|
||||
PREFILL_HOST=${PREFILL_HOST} \
|
||||
PREFILL_PORT=${PREFILL_PORT} \
|
||||
DECODE_HOST=${DECODE_HOST} \
|
||||
DECODE_PORT=${DECODE_PORT} \
|
||||
PROXY_HOST=${PROXY_HOST} \
|
||||
PROXY_PORT=${PROXY_PORT} python -m pytest -s -v ${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/test_edge_cases.py
|
||||
@@ -0,0 +1,162 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
|
||||
import openai
|
||||
import requests
|
||||
|
||||
MAX_OUTPUT_LEN = 30
|
||||
|
||||
SAMPLE_PROMPTS = (
|
||||
"Red Hat is the best company in the world to work for because it works on "
|
||||
"open source software, which means that all the contributions are "
|
||||
"delivered to the community. As a result, when working on projects like "
|
||||
"vLLM we are able to meet many amazing people from various organizations "
|
||||
"like AMD, Google, NVIDIA, ",
|
||||
"We hold these truths to be self-evident, that all men are created equal, "
|
||||
"that they are endowed by their Creator with certain unalienable Rights, "
|
||||
"that among these are Life, Liberty and the pursuit of Happiness.--That "
|
||||
"to secure these rights, Governments are instituted among Men, deriving "
|
||||
"their just powers from the consent of the governed, ",
|
||||
)
|
||||
|
||||
|
||||
def check_vllm_server(url: str, timeout=5, retries=3) -> bool:
|
||||
"""
|
||||
Checks if the vLLM server is ready by sending a GET request to the
|
||||
/health endpoint.
|
||||
|
||||
Args:
|
||||
url (str): The base URL of the vLLM server.
|
||||
timeout (int): Timeout in seconds for the request.
|
||||
retries (int): Number of retries if the server is not ready.
|
||||
|
||||
Returns:
|
||||
bool: True if the server is ready, False otherwise.
|
||||
"""
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
response = requests.get(url, timeout=timeout)
|
||||
if response.status_code == 200:
|
||||
return True
|
||||
else:
|
||||
print(f"Attempt {attempt + 1}: Server returned status code "
|
||||
"{response.status_code}")
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f"Attempt {attempt + 1}: Error connecting to server: {e}")
|
||||
time.sleep(1) # Wait before retrying
|
||||
return False
|
||||
|
||||
|
||||
def run_simple_prompt(base_url: str, model_name: str,
|
||||
input_prompt: str) -> str:
|
||||
client = openai.OpenAI(api_key="EMPTY", base_url=base_url)
|
||||
completion = client.completions.create(model=model_name,
|
||||
prompt=input_prompt,
|
||||
max_tokens=MAX_OUTPUT_LEN,
|
||||
temperature=0.0,
|
||||
seed=42)
|
||||
|
||||
# print("-" * 50)
|
||||
# print(f"Completion results for {model_name}:")
|
||||
# print(completion)
|
||||
# print("-" * 50)
|
||||
return completion.choices[0].text
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
This script demonstrates how to accept two optional string arguments
|
||||
("service_url" and "file_name") from the command line, each with a
|
||||
default value of an empty string, using the argparse module.
|
||||
"""
|
||||
parser = argparse.ArgumentParser(description="vLLM client script")
|
||||
|
||||
parser.add_argument(
|
||||
"--service_url", # Name of the first argument
|
||||
type=str,
|
||||
required=True,
|
||||
help="The vLLM service URL.")
|
||||
|
||||
parser.add_argument(
|
||||
"--model_name", # Name of the first argument
|
||||
type=str,
|
||||
required=True,
|
||||
help="model_name",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--mode", # Name of the second argument
|
||||
type=str,
|
||||
default="baseline",
|
||||
help="mode: baseline==non-disagg, or disagg",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--file_name", # Name of the second argument
|
||||
type=str,
|
||||
default=".vllm_output.txt",
|
||||
help="the file that saves the output tokens ",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
for arg in vars(args):
|
||||
print(f"{arg}: {getattr(args, arg)}")
|
||||
|
||||
if args.mode == "baseline":
|
||||
# non-disagg
|
||||
health_check_url = f"{args.service_url}/health"
|
||||
else:
|
||||
# disagg proxy
|
||||
health_check_url = f"{args.service_url}/healthcheck"
|
||||
if not os.path.exists(args.file_name):
|
||||
raise ValueError(
|
||||
f"In disagg mode, the output file {args.file_name} from "
|
||||
"non-disagg. baseline does not exist.")
|
||||
|
||||
service_url = f"{args.service_url}/v1"
|
||||
|
||||
if not check_vllm_server(health_check_url):
|
||||
raise RuntimeError(
|
||||
f"vllm server: {args.service_url} is not ready yet!")
|
||||
|
||||
output_strs = dict()
|
||||
for prompt in SAMPLE_PROMPTS:
|
||||
output_str = run_simple_prompt(base_url=service_url,
|
||||
model_name=args.model_name,
|
||||
input_prompt=prompt)
|
||||
print(f"Prompt: {prompt}, output: {output_str}")
|
||||
output_strs[prompt] = output_str
|
||||
|
||||
if args.mode == "baseline":
|
||||
# baseline: save outputs
|
||||
try:
|
||||
with open(args.file_name, 'w') as json_file:
|
||||
json.dump(output_strs, json_file, indent=4)
|
||||
except OSError as e:
|
||||
print(f"Error writing to file: {e}")
|
||||
raise
|
||||
else:
|
||||
# disagg. verify outputs
|
||||
baseline_outputs = None
|
||||
try:
|
||||
with open(args.file_name) as json_file:
|
||||
baseline_outputs = json.load(json_file)
|
||||
except OSError as e:
|
||||
print(f"Error writing to file: {e}")
|
||||
raise
|
||||
assert isinstance(baseline_outputs, dict)
|
||||
assert len(baseline_outputs) == len(output_strs)
|
||||
for prompt, output in baseline_outputs.items():
|
||||
assert prompt in output_strs, f"{prompt} not included"
|
||||
assert output == output_strs[prompt], (
|
||||
f"baseline_output: {output} != PD output: {output_strs[prompt]}"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -4,8 +4,11 @@ import os
|
||||
|
||||
import openai
|
||||
|
||||
PREFILL_HOST = os.getenv("PREFILL_HOST", "localhost")
|
||||
PREFILL_PORT = os.getenv("PREFILL_PORT", None)
|
||||
DECODE_HOST = os.getenv("DECODE_HOST", "localhost")
|
||||
DECODE_PORT = os.getenv("DECODE_PORT", None)
|
||||
PROXY_HOST = os.getenv("PROXY_HOST", "localhost")
|
||||
PROXY_PORT = os.getenv("PROXY_PORT", None)
|
||||
|
||||
if PREFILL_PORT is None or DECODE_PORT is None or PROXY_PORT is None:
|
||||
@@ -21,15 +24,15 @@ def test_edge_cases():
|
||||
# Set the OpenAI API key and base URL
|
||||
decode_client = openai.OpenAI(
|
||||
api_key="MY_KEY",
|
||||
base_url=f"http://localhost:{DECODE_PORT}/v1",
|
||||
base_url=f"http://{DECODE_HOST}:{DECODE_PORT}/v1",
|
||||
)
|
||||
prefill_client = openai.OpenAI(
|
||||
api_key="MY_KEY",
|
||||
base_url=f"http://localhost:{PREFILL_PORT}/v1",
|
||||
base_url=f"http://{PREFILL_HOST}:{PREFILL_PORT}/v1",
|
||||
)
|
||||
proxy_client = openai.OpenAI(
|
||||
api_key="MY_KEY",
|
||||
base_url=f"http://localhost:{PROXY_PORT}/v1",
|
||||
base_url=f"http://{PROXY_HOST}:{PROXY_PORT}/v1",
|
||||
)
|
||||
|
||||
# Get the list of models
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
import argparse
|
||||
import itertools
|
||||
import logging
|
||||
import os
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
@@ -11,9 +12,8 @@ import httpx
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(logging.DEBUG)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
|
||||
Reference in New Issue
Block a user