Initial commit

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-15 01:01:52 +07:00
commit 777e910dd0
124 changed files with 76242 additions and 0 deletions
+35
View File
@@ -0,0 +1,35 @@
# Build products the serving tree needs. ./build.sh verifies every path in this list
# after the compile, and hashes them into vllm/.build-products.sha256.
# Lines starting with '#' are skipped.
vllm/_C_stable_libtorch.abi3.so
vllm/_moe_C_stable_libtorch.abi3.so
vllm/_version.py
vllm/cumem_allocator.abi3.so
vllm/fs_io_C.abi3.so
vllm/spinloop.abi3.so
vllm/third_party/flashmla/flash_mla_interface.py
vllm/third_party/fmha_sm100/
vllm/third_party/tml_fa4/
vllm/third_party/triton_kernels/
vllm/vllm_flash_attn/_vllm_fa2_C.abi3.so
vllm/vllm_flash_attn/_vllm_fa3_C.abi3.so
vllm/vllm_flash_attn/cute/
vllm/vllm_flash_attn/layers/
vllm/vllm_flash_attn/ops/
# Built only when the matching toolchain is present. build.sh warns when cargo is missing,
# and prints the exact command to add it. vllm-rs is the optional Rust frontend
# (envs.py: VLLM_RUST_FRONTEND_PATH); the tool parsers in the shipped serve profile are Python.
#vllm/vllm-rs
#vllm/_rust_tool_parser.abi3.so
# Built only for the architectures that have kernels. cmake/external_projects/*.cmake gate these
# per arch: flashmla 9.0a/10.0/10.7, deepselect 10.0, flashkda 9.0a/10.0/12.0, qutlass 10.0/12.0,
# deep_gemm (optional vllm._deep_gemm_C, the DeepSeek-V4 path). arch 8.6 (RTX 3090) builds none of
# them, and the Qwen3.8-Flash-Next serve path does not import them.
#vllm/_deepselect_C.abi3.so
#vllm/_flashkda_C.abi3.so
#vllm/_flashmla_C.abi3.so
#vllm/_flashmla_extension_C.abi3.so
#vllm/_qutlass_C.abi3.so
#vllm/third_party/deep_gemm/
+70
View File
@@ -0,0 +1,70 @@
#!/usr/bin/env bash
# Generates the CUDA driver link stub: $1 = a CUDA C header, $2 = the .so to write.
#
# vllm links its C++ kernels against CUDA::cuda_driver, the CUDA driver library
# (libcuda.so.1). The driver package provides it; the CUDA toolkit wheel does
# not, and neither does a build container without a GPU. NVIDIA ships the
# link-time stub for it in the runfile installer, not in the pip wheel, so this
# script builds an equivalent one from the header: one exported symbol per
# driver entry point, no bodies, 87 KB.
#
# Link time only. The soname is libcuda.so.1, so the dynamic linker opens the
# real driver from the host at run time and can never pick the stub.
#
# Writes nothing when the file already has every entry point, so a re-run does
# not invalidate the compiled-ops hash recorded by build.sh.
set -euo pipefail
H="${1:?usage: $0 <cuda.h> <out.so>}"
OUT="${2:?usage: $0 <cuda.h> <out.so>}"
[ -r "$H" ] || { echo "no readable CUDA header at $H"; exit 1; }
decls(){
# Calls written as "CUresult CUDAAPI cuFoo(...)", split over lines.
awk '/CUresult[ \t]+CUDAAPI/{cap=1;buf=""} cap{buf=buf" "$0; if(buf~/;/){
if(match(buf,/CUresult[ \t]+CUDAAPI[ \t]+[A-Za-z_][A-Za-z0-9_]*/)){
s=substr(buf,RSTART,RLENGTH); sub(/.*CUDAAPI[ \t]+/,"",s); print s} cap=0}}' "$H"
# Versioned names hidden behind a macro: code writes cuFoo, the preprocessor
# rewrites it to cuFoo_v2, so the object file asks the linker for cuFoo_v2.
sed -nE 's/^#define[ \t]+(cu[A-Za-z0-9_]+)[ \t]+(cu[A-Za-z0-9_]+_v[0-9]+)([ \t]|$).*/\2/p' "$H"
}
want="$(decls | sort -u)"
n="$(printf '%s\n' "$want" | sed '/^$/d' | wc -l)"
[ "$n" -gt 100 ] || { echo "only $n driver entry points found in $H"; exit 1; }
if [ -e "$OUT" ]; then
have="$(nm -D --defined-only "$OUT" 2>/dev/null | awk '$2=="T"{print $3}' | sort -u)"
miss="$(LC_ALL=C comm -23 <(printf '%s\n' "$want") <(printf '%s\n' "$have") | sed '/^$/d')"
if [ -z "$miss" ]; then
echo "driver stub is current: $(basename "$OUT"), $n entry points"
exit 0
fi
echo "driver stub is missing $(printf '%s\n' "$miss" | wc -l) entry points, regenerating"
fi
mkdir -p "$(dirname "$OUT")"
src="$(mktemp --suffix=.c)"
cleanup(){ rm -f "$src"; }
trap cleanup EXIT
: > "$src"
printf '%s\n' "$want" | while read -r f; do
[ -n "$f" ] && printf 'unsigned int %s(void){ return 0; }\n' "$f" >> "$src"
done
cc -shared -fPIC -O0 -o "$OUT" "$src" -Wl,-soname,libcuda.so.1
have="$(nm -D --defined-only "$OUT" | awk '$2=="T"{print $3}' | sort -u)"
miss="$(LC_ALL=C comm -23 <(printf '%s\n' "$want") <(printf '%s\n' "$have") | sed '/^$/d')"
[ -z "$miss" ] || { echo "the stub is still missing:"; printf '%s\n' "$miss"; exit 1; }
echo "driver stub generated: $OUT ($(printf '%s\n' "$want" | wc -l) entry points, soname libcuda.so.1)"
+93
View File
@@ -0,0 +1,93 @@
#!/usr/bin/env bash
# One entry point for the bare-metal systemd service: install, start, stop, status, logs,
# update, uninstall. Run from anywhere; the script uses the repo it lives in.
#
# scripts/install-as-service.sh <command>
#
# install write /etc/systemd/system/vllm.service for THIS repo directory,
# write /etc/default/vllm (checkpoint path, model name) when missing,
# install the logrotate policy, daemon-reload, enable
# start|stop|restart|status|logs systemctl for the service
# update ./build.sh in this repo, then restart the service
# uninstall disable + remove the unit, the EnvironmentFile and the logrotate file
#
# Needs root (the log goes to /var/log, the unit goes to /etc/systemd/system).
# The service runs the repo where it lives: no copy, no /opt, no second build tree.
# Env: CKPT MODEL_NAME PORT override what the EnvironmentFile sets.
set -euo pipefail
HERE="$(cd "$(dirname "$0")/.." && pwd)"
UNIT=vllm
UNIT_PATH=/etc/systemd/system/$UNIT.service
DEFAULT=/etc/default/$UNIT
LOGROTATE=/etc/logrotate.d/$UNIT
LOG="${LOG:-/var/log/vllm.log}"
[ "$(id -u)" = 0 ] || { echo "run as root: sudo $0 $*"; exit 1; }
command -v systemctl >/dev/null || { echo "no systemd on this host"; exit 1; }
cmd="${1:-}"; shift || true
case "$cmd" in
install)
ckpt="${CKPT:-$(sed -n 's/^MODEL_DIR=//p' "$HERE/.env" 2>/dev/null | head -1)}"
ckpt="${ckpt:-/data/models/halt95/Swift1.5-Qwen3.8-Flash-Next-W4A16-Merlin}"
[ -f "$HERE/start.sh" ] || { echo "not the repo root: $HERE"; exit 1; }
# 1. the unit: the shipped template, with /opt/vllm replaced by this directory
sed "s|/opt/vllm|$HERE|g" "$HERE/systemd/$UNIT.service" > "$UNIT_PATH"
echo "==> unit: $UNIT_PATH (WorkingDirectory=$HERE)"
# 2. the EnvironmentFile: create once, never overwrite an edited one
if [ ! -f "$DEFAULT" ]; then
{ echo "# Read by $UNIT.service. Uncomment and edit, then: systemctl restart $UNIT"
echo "CKPT=$ckpt"
echo "MODEL_NAME=Scarlet"
echo "# LOG=/var/log/vllm.log"
echo "# CACHE_ROOT=$HERE/.vllm-cache"
echo "# CUDA_VISIBLE_DEVICES=0,1,2,3"
echo "# VLLM_LOG_STATS_INTERVAL=1"
} > "$DEFAULT"
echo "==> wrote $DEFAULT: set CKPT to the checkpoint directory"
else
echo "==> kept $DEFAULT (it exists; CKPT=$(. "$DEFAULT" 2>/dev/null && echo "${CKPT:-unset}" || echo unset))"
fi
# 3. log rotation for $LOG (same policy as the shipped systemd/vllm.logrotate)
sed "s|/var/log/vllm.log|$LOG|g" "$HERE/systemd/$UNIT.logrotate" > "$LOGROTATE"
chmod 644 "$LOGROTATE"
echo "==> logrotate: $LOGROTATE"
systemctl daemon-reload
systemctl enable "$UNIT"
ck="$( . "$DEFAULT" 2>/dev/null && echo "${CKPT:-}" )"
[ -n "$ck" ] && [ -d "$ck" ] || echo "==> next: edit CKPT in $DEFAULT (the checkpoint directory), then sudo $0 start"
echo "==> installed and enabled. Start with: sudo $0 start"
;;
start) systemctl start "$UNIT"; echo "==> started; a cold start takes ~6 min: sudo $0 logs" ;;
stop) systemctl stop "$UNIT" ;;
restart) systemctl restart "$UNIT" ;;
status)
systemctl status "$UNIT" --no-pager -l | head -12 || :
echo "==> engine log: tail -f $LOG (healthy line: GPU KV cache size)"
;;
logs) exec tail -n 100 -f "$LOG" ;;
update)
echo "==> build, then restart"
( cd "$HERE" && ./build.sh )
systemctl restart "$UNIT"
echo "==> update done"
;;
uninstall)
printf 'remove the unit, the EnvironmentFile and the logrotate file? [y/N] '
read -r a
case "$a" in
[yY]*) systemctl disable --now "$UNIT" 2>/dev/null || :
rm -f "$UNIT_PATH" "$DEFAULT" "$LOGROTATE"
systemctl daemon-reload
echo "==> removed (the repo, .venv, .vllm-cache and $LOG are kept)" ;;
*) echo "cancelled" ;;
esac
;;
*)
sed -n '3,18p' "$0"
exit 1
;;
esac
+96
View File
@@ -0,0 +1,96 @@
#!/usr/bin/env python3
"""Add the FP8 PLE config key the serving tree needs to a Qwen3.8-Flash-Next-W4A16-Merlin checkpoint.
The serving tree keys the FP8 PLE (n-gram embedding) table on a checkpoint config field instead of the
VLLM_PLE_FP8_GLOBAL_SCALE=1 environment variable the earlier lane used. The field is read from the model's
*text* config (`vllm_config.model_config.hf_text_config`), so it has to live inside the `"text_config"`
object of config.json, not at the top level. Checkpoints published without that key serve at the env-var
scale, which is the wrong scale for this tree. This script inserts
the key as a targeted text edit (the rest of the file is byte-preserved), re-parses the result to prove the
key landed where the engine reads it, writes a new file and renames it into place -- so a hard-linked
config.json is un-linked rather than mutated in place -- and keeps a backup.
Weights are untouched: every shard can stay a hard link of the original checkpoint.
python3 scripts/make-e1-config.py /path/to/Qwen3.8-Flash-Next-W4A16-Merlin
"""
import json, os, re, shutil, sys
KEY, VAL = "ple_embedding_dtype", "float8_e4m3fn"
d = sys.argv[1] if len(sys.argv) > 1 else sys.exit(__doc__)
p = os.path.join(d, "config.json")
if not os.path.isfile(p) or not os.path.isfile(os.path.join(d, "model.safetensors.index.json")):
sys.exit(f"{d}: not a checkpoint directory (config.json + model.safetensors.index.json expected)")
raw = open(p, "rb").read()
cfg = json.loads(raw)
text = cfg.get("text_config")
if not isinstance(text, dict):
sys.exit("config.json has no text_config object; this is not the Flash-Next checkpoint this script is for")
if KEY in text and text[KEY] != VAL:
sys.exit(f"text_config.{KEY}={text[KEY]!r} already present; refusing to overwrite")
stale_top = KEY in cfg # left by the pre-fix version of this script; the engine never read it
if text.get(KEY) == VAL and not stale_top:
print("already set"); sys.exit(0)
if os.stat(p).st_nlink > 1:
print(f"note: config.json has {os.stat(p).st_nlink} hard links; writing a new file so the other links stay unchanged")
s = raw.decode("utf-8")
if stale_top:
# remove the top-level copy (the pre-fix script wrote it right after the opening brace): the first
# occurrence of the key line that sits before "text_config" is the top-level one
tc = s.index('"text_config"')
mm = None
for cand in re.finditer(r'\r?\n[ \t]*"' + KEY + r'"\s*:\s*"[^"]*",?', s):
if cand.start() < tc: mm = cand; break
if mm is None: sys.exit(f"top-level {KEY} present but not in the expected position; remove it by hand")
s = s[:mm.start()] + s[mm.end():]
if KEY in json.loads(s): sys.exit("internal error: top-level key still present after removal; nothing written")
print(f"note: stale top-level {KEY} (from the pre-fix script) removed")
m = re.search(r'"text_config"\s*:\s*\{', s)
if not m or s.count('"text_config"') != 1:
sys.exit("could not locate a unique \"text_config\": { in config.json")
i = m.end()
nl = "\r\n" if "\r\n" in s[:400] else "\n"
# indentation of the first key inside text_config (the text after the opening brace up to the first quote)
after = s[i:i + 64]
indent = after[: len(after) - len(after.lstrip())].replace("\r", "").replace("\n", "") or " "
new = s if text.get(KEY) == VAL else s[:i] + nl + indent + f'"{KEY}": "{VAL}",' + s[i:]
check = json.loads(new) # must still parse ...
if check["text_config"].get(KEY) != VAL: # ... and the key must be where the engine reads it
sys.exit("internal error: key did not land in text_config; nothing written")
other = {k: v for k, v in check.items() if k != "text_config"}
if other != {k: v for k, v in cfg.items() if k not in ("text_config", KEY)} or \
{k: v for k, v in check["text_config"].items() if k != KEY} != {k: v for k, v in text.items() if k != KEY}:
sys.exit("internal error: something other than the one key changed; nothing written")
shutil.copy2(p, p + ".bak-pre-v2")
tmp = p + ".tmp-v2"
with open(tmp, "wb") as f: f.write(new.encode("utf-8"))
os.replace(tmp, p)
print(f"config.json: text_config.{KEY} = {VAL} in place, no other key (backup config.json.bak-pre-v2)")
+46
View File
@@ -0,0 +1,46 @@
#!/usr/bin/env python3
"""Reproduce the UntypedStorage failure from boot #1.
Mirrors the engine: CUDA initialized, then safetensors_weights_iterator over the
heretic2 shards. Logs the exact tensor name that raises.
"""
import glob
import sys
import torch
device = sys.argv[1] if len(sys.argv) > 1 else "cuda:0"
torch.cuda.set_device(int(device.split(":")[1]))
torch.zeros(1, device=device)
from safetensors import safe_open
from vllm.model_executor.model_loader.weight_utils import (
safetensors_weights_iterator,
)
files = sorted(glob.glob("/models/model/*.safetensors"))
orig = safe_open.get_tensor
def patched(self, name):
try:
return orig(self, name)
except Exception as exc:
print(f"RAISE {self.__class__.__name__} name={name} err={exc}", flush=True)
raise
safe_open.get_tensor = patched
n = 0
for name, t in safetensors_weights_iterator(files, False):
n += 1
if n % 40000 == 0:
print(n, flush=True)
print("OK tensors:", n)
+196
View File
@@ -0,0 +1,196 @@
agent-detector==2.0.0
aiohappyeyeballs==2.7.1
aiohttp==3.14.3
aiosignal==1.4.0
annotated-doc==0.0.5
annotated-types==0.8.0
anthropic==1.7.0
anyio==4.15.1
apache-tvm-ffi==0.1.11
astor==0.8.1
attrs==26.1.0
blake3==1.0.9
cachetools==7.2.0
cbor2==6.1.4
certifi==2026.7.22
cffi==2.1.1
charset-normalizer==3.5.1
click==8.5.0
cloudpickle==3.1.2
compressed-tensors==0.17.0
cryptography==50.0.1
cuda-bindings==13.4.2
cuda-core==1.2.0
cuda-pathfinder==1.8.2
cuda-python==13.4.1
cuda-tile==1.6.0
cuda-toolkit==13.0.3.0
depyf==0.20.0
detect-installer==0.2.1
dill==0.4.1
dnspython==2.8.0
docstring_parser==0.18.0
einops==0.8.2
email-validator==2.3.0
fastapi==0.136.3
fastapi-cli==0.0.32
fastapi-cloud-cli==0.26.0
fastar==0.12.0
fastsafetensors==0.4.0
filelock==4.0.1
flashinfer-python==0.6.18.post1
frozenlist==1.8.0
fsspec==2026.9.0
googleapis-common-protos==1.75.3
grpcio==1.84.0
h11==0.16.0
hf-xet==1.6.0
httpcore==1.0.9
httpcore2==2.13.0
httptools==0.8.0
httpx==0.28.1
httpx2==2.13.0
huggingface_hub==1.32.0
humming-kernels==0.1.12
idna==3.20
ijson==3.5.1
instanttensor==0.2.0
interegular==0.3.3
Jinja2==3.1.6
jiter==0.17.0
jmespath==1.1.0
jsonschema==4.26.0
jsonschema-specifications==2025.9.1
lark==1.2.2
llguidance==1.7.6
llvmlite==0.47.0
lm-format-enforcer==0.11.3
loguru==0.7.3
markdown-it-py==4.2.0
MarkupSafe==3.0.3
mcp==2.2.0
mcp-types==2.2.0
mdurl==0.1.2
mistral_common==1.11.7
ml_dtypes==0.6.0
model-hosting-container-standards==0.1.16
mpmath==1.3.0
msgspec==0.21.1
multidict==6.9.1
nccl4py==0.5.0
networkx==3.7
ninja==1.13.2
numba==0.65.0
numpy==2.4.6
nvidia-cublas==13.1.1.3
nvidia-cuda-cccl==13.0.85
nvidia-cuda-crt==13.0.88
nvidia-cuda-cupti==13.0.85
nvidia-cuda-nvcc==13.0.88
nvidia-cuda-nvdisasm==13.4.92
nvidia-cuda-nvrtc==13.0.88
nvidia-cuda-runtime==13.0.96
nvidia-cudnn-cu13==9.20.0.48
nvidia-cudnn-frontend==1.29.0
nvidia-cufft==12.0.0.61
nvidia-cufile==1.15.1.6
nvidia-curand==10.4.0.35
nvidia-cusolver==12.0.4.66
nvidia-cusparse==12.6.3.3
nvidia-cusparselt-cu13==0.8.1
nvidia-cutlass-dsl==4.7.1
nvidia-cutlass-dsl-libs-base==4.7.1
nvidia-cutlass-dsl-libs-core==4.7.1
nvidia-cutlass-dsl-libs-cu12==4.7.1
nvidia-cutlass-dsl-libs-cu13==4.7.1
nvidia-ml-py==13.610.43
nvidia-nccl-cu13==2.29.7
nvidia-nvjitlink==13.4.92
nvidia-nvshmem-cu13==3.4.5
nvidia-nvtx==13.0.85
nvidia-nvvm==13.0.88
nvtx==0.2.15
openai==3.17.0
openai-harmony==0.0.8
opencv-python-headless==5.0.0.93
opentelemetry-api==1.44.0
opentelemetry-exporter-otlp==1.44.0
opentelemetry-exporter-otlp-proto-common==1.44.0
opentelemetry-exporter-otlp-proto-grpc==1.44.0
opentelemetry-exporter-otlp-proto-http==1.44.0
opentelemetry-proto==1.44.0
opentelemetry-sdk==1.44.0
opentelemetry-semantic-conventions==0.65b0
opentelemetry-semantic-conventions-ai==0.5.1
outlines_core==0.2.14
packaging==26.3
partial-json-parser==0.2.1.1.post7
pillow==12.3.0
prometheus-fastapi-instrumentator==8.1.0
prometheus_client==0.26.0
propcache==0.5.4
protobuf==7.36.2
psutil==7.2.2
py-cpuinfo==9.0.0
pybase64==1.5.0
pycountry==26.2.16
pycparser==3.0
pydantic==2.13.5
pydantic-extra-types==2.11.1
pydantic-settings==2.15.0
pydantic_core==2.46.5
Pygments==2.21.0
PyJWT==2.14.0
pynvvideocodec==2.0.4
python-dotenv==1.2.3
python-json-logger==4.2.0
python-multipart==0.0.32
PyYAML==6.0.3
pyzmq==27.2.0
quack-kernels==0.6.5
referencing==0.37.0
regex==2026.9.10
requests==2.34.2
rich==15.0.0
rich-toolkit==0.20.5
rignore==0.8.1
rpds-py==2026.6.3
safetensors==0.8.0
sentencepiece==0.2.2
sentry-sdk==2.69.2
setproctitle==1.3.7
setuptools==80.10.2
shellingham==1.5.4
six==1.17.0
sniffio==1.3.1
sse-starlette==3.4.11
starlette==1.6.0
supervisor==4.3.0
sympy==1.14.0
tabulate==0.10.0
tiktoken==0.14.0
tilelang==0.1.12
tokenizers==0.23.2
tokenspeed-mla==0.1.8
tokenspeed-triton==3.8.10.post20260920
torch==2.13.0
torch_c_dlpack_ext==0.1.5
torchaudio==2.11.0
torchcodec==0.16.0
torchvision==0.28.0
tqdm==4.70.1
transformers==5.18.0
triton==3.7.1
truststore==0.10.4
typer==0.27.2
typing-inspection==0.4.4
typing_extensions==4.16.0
urllib3==2.8.0
uvicorn==0.53.0
uvloop==0.22.1
watchfiles==1.3.0
websockets==17.1
wheel==0.48.0
xgrammar==0.2.7
yarl==1.25.1
z3-solver==4.15.4.0
+36
View File
@@ -0,0 +1,36 @@
#!/usr/bin/env python3
"""Diagnose heretic2 boot Failure B.
When PLE_ST_DEBUG_GETTENSOR=1, logs every safetensors tensor read to
/tmp/stdbg-<pid>.log so the last line before an UntypedStorage ValueError
names the tensor that fails. Installed via PYTHONPATH as sitecustomize.
"""
import os
import sys
if os.environ.get("PLE_ST_DEBUG_GETTENSOR") == "1":
try:
from safetensors import safe_open
_orig = safe_open.get_tensor
_path = f"/stdbg/stdbg-{os.getpid()}.log"
_fh = open(_path, "a", buffering=1 << 16)
def _patched(self, name):
try:
t = _orig(self, name)
except Exception as exc:
_fh.write(f"RAISE {name} {type(exc).__name__} {exc}\n")
_fh.flush()
raise
_fh.write(f"OK {name} {tuple(t.shape)} {t.dtype}\n")
return t
safe_open.get_tensor = _patched
except Exception as exc: # pragma: no cover
sys.stderr.write(f"STDBG patch failed {exc}\n")
+1310
View File
File diff suppressed because it is too large Load Diff