diff --git a/start.sh b/start.sh index 756e3d7..1955ee6 100755 --- a/start.sh +++ b/start.sh @@ -209,6 +209,10 @@ export HF_HUB_OFFLINE=1 HF_DATASETS_OFFLINE=1 export MERLIN_FULL_K=1 MERLIN_QSA_SORT=1 MERLIN_TIE_DET=1 MERLIN_TIE_RECENT=1 MERLIN_TOPK_SORTED_EMIT=1 export VLLM_E2_GUARD_MODE="${GUARD:-warn}" export VLLM_E2_SIDE_CACHE_LAYOUT=1 VLLM_USE_V2_MODEL_RUNNER=1 VLLM_E2_MAMBA_RETAIN_CHECKPOINT=1 +# Accept any name in the model field of a request, like llama.cpp. Upstream's own switch, on by +# default here: /v1/models and the model field of every reply still report the served name. +# Set VLLM_SKIP_MODEL_NAME_VALIDATION=0 to reject a name that is not served. +export VLLM_SKIP_MODEL_NAME_VALIDATION="${VLLM_SKIP_MODEL_NAME_VALIDATION:-1}" if [ "${COUNTERS:-0}" = 1 ]; then export MERLIN_TOPK_EXACT_COUNTERS_LOG=30 VLLM_QSA_KV_CLIP_COUNT=300; fi [ -f "$VLLM_QSA_KV_SCALES" ] || { echo "scales sidecar missing: $VLLM_QSA_KV_SCALES (FP8 KV would run at scale 1.0). Set SCALES= to the sidecar of this model."; exit 1; }