[Feature][EPD] Support dynamic EPD (EC-connector) proxy (#54176)
Signed-off-by: Tianyu Guo <guoty@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
OpenAI Codex
parent
1596fa5f88
commit
70fc7bd695
@@ -152,17 +152,33 @@ wait_for_server "$PREFILL_DECODE_PORT"
|
||||
###############################################################################
|
||||
# Proxy
|
||||
###############################################################################
|
||||
PROXY_ARGS=()
|
||||
if [[ "${DYNAMIC_REGISTRATION:-0}" == "1" ]]; then
|
||||
: "${ADMIN_API_KEY:?Set ADMIN_API_KEY for dynamic registration}"
|
||||
export ADMIN_API_KEY
|
||||
PROXY_ARGS=(--dynamic-registration)
|
||||
else
|
||||
PROXY_ARGS=(
|
||||
--encode-servers-urls "http://localhost:$ENCODE_PORT"
|
||||
--decode-servers-urls "http://localhost:$PREFILL_DECODE_PORT"
|
||||
)
|
||||
fi
|
||||
python disagg_epd_proxy.py \
|
||||
--host "0.0.0.0" \
|
||||
--port "$PROXY_PORT" \
|
||||
--encode-servers-urls "http://localhost:$ENCODE_PORT" \
|
||||
--prefill-servers-urls "disable" \
|
||||
--decode-servers-urls "http://localhost:$PREFILL_DECODE_PORT" \
|
||||
"${PROXY_ARGS[@]}" \
|
||||
>"${PROXY_LOG}" 2>&1 &
|
||||
|
||||
PIDS+=($!)
|
||||
|
||||
wait_for_server "$PROXY_PORT"
|
||||
if [[ "${DYNAMIC_REGISTRATION:-0}" == "1" ]]; then
|
||||
for endpoint in "encode:$ENCODE_PORT" "prefill_decode:$PREFILL_DECODE_PORT"; do
|
||||
curl --fail-with-body "http://127.0.0.1:$PROXY_PORT/instances" \
|
||||
-H "X-API-Key: $ADMIN_API_KEY" -H 'Content-Type: application/json' \
|
||||
-d "{\"role\":\"${endpoint%%:*}\",\"url\":\"http://127.0.0.1:${endpoint#*:}\"}"
|
||||
done
|
||||
fi
|
||||
echo "All services are up!"
|
||||
|
||||
###############################################################################
|
||||
@@ -201,4 +217,4 @@ curl http://127.0.0.1:"${PROXY_PORT}"/v1/chat/completions \
|
||||
|
||||
# cleanup
|
||||
echo "cleanup..."
|
||||
cleanup
|
||||
cleanup
|
||||
|
||||
Reference in New Issue
Block a user