[Feat][XPU] VLLM_BATCH_INVARIANT support for Dense/MoE models (#55881)

Signed-off-by: Tony Lin <tony.lin@intel.com>
Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
Tony Lin
2026-09-22 11:06:54 -04:00
committed by GitHub
co-authored by Flora Feng
parent 1ea7c63f4a
commit e4340e41c9
10 changed files with 619 additions and 8 deletions
+243
View File
@@ -0,0 +1,243 @@
#!/usr/bin/env bash
set -euo pipefail
repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../../.." && pwd)"
model="ibm-research/PowerMoE-3b"
port=8000
num_decode_tokens=128
server_pid=
server_log=
# XPU MoE kernels may not safely handle the -1 routes emitted for padding.
export VLLM_MOE_SKIP_PADDING=0
cd "$repo_root"
cleanup_server() {
local attempt
if [[ -n "$server_pid" ]] && kill -0 "$server_pid" 2>/dev/null; then
kill "$server_pid" 2>/dev/null || true
for ((attempt = 0; attempt < 30; attempt++)); do
if ! kill -0 "$server_pid" 2>/dev/null; then
break
fi
sleep 1
done
if kill -0 "$server_pid" 2>/dev/null; then
echo "vLLM did not stop after 30 seconds; sending SIGKILL" >&2
kill -KILL "$server_pid" 2>/dev/null || true
fi
wait "$server_pid" 2>/dev/null || true
fi
server_pid=
}
on_exit() {
local status=$?
cleanup_server
if [[ "$status" -ne 0 && -n "$server_log" && -f "$server_log" ]]; then
if command -v buildkite-agent >/dev/null 2>&1; then
buildkite-agent artifact upload "$server_log" ||
echo "Failed to upload $server_log as a Buildkite artifact" >&2
fi
fi
[[ -z "$server_log" ]] || rm -f "$server_log"
exit "$status"
}
trap on_exit EXIT
wait_for_server() {
for _ in $(seq 1 600); do
if curl -sf "http://127.0.0.1:$port/health" >/dev/null; then
return
fi
if ! kill -0 "$server_pid" 2>/dev/null; then
echo "vLLM exited before becoming healthy; see the server log above." >&2
return 1
fi
sleep 1
done
echo "Timed out waiting for vLLM server health check" >&2
return 1
}
request_outputs() {
local prompts_json=$1
local expected_choices=$2
local response
response=$(printf \
'{"model":"%s","prompt":%s,"temperature":0.7,"top_p":0.95,"seed":0,"max_tokens":%s,"logprobs":64,"ignore_eos":true,"return_token_ids":true}' \
"$model" "$prompts_json" "$num_decode_tokens" |
curl --fail --silent --show-error \
--max-time 1200 \
"http://127.0.0.1:$port/v1/completions" \
-H "Content-Type: application/json" \
--data-binary @-)
local -a token_id_arrays logprob_arrays
mapfile -t token_id_arrays < <(
grep -o '"token_ids":\[[^]]*\]' <<<"$response" |
sed 's/^"token_ids":\[//; s/\]$//'
)
mapfile -t logprob_arrays < <(
grep -o '"token_logprobs":\[[^]]*\]' <<<"$response" |
sed 's/^"token_logprobs":\[//; s/\]$//'
)
if [[ "${#token_id_arrays[@]}" -ne "$expected_choices" ]] ||
[[ "${#logprob_arrays[@]}" -ne "$expected_choices" ]]; then
echo "Expected $expected_choices choices with token IDs and logits" >&2
return 1
fi
local index
local array
local value
local -a values
for index in "${!logprob_arrays[@]}"; do
array="${logprob_arrays[index]}"
IFS=, read -r -a values <<<"$array"
if [[ "${#values[@]}" -ne "$num_decode_tokens" ]]; then
echo "Completion returned an unexpected number of decode steps" >&2
return 1
fi
IFS=, read -r -a values <<<"${token_id_arrays[index]}"
if [[ "${#values[@]}" -ne "$num_decode_tokens" ]]; then
echo "Completion returned an unexpected number of generated tokens" >&2
return 1
fi
for value in "${values[@]}"; do
if [[ ! "$value" =~ ^[0-9]+$ ]]; then
echo "Completion returned an invalid token ID" >&2
return 1
fi
done
IFS=, read -r -a values <<<"$array"
for value in "${values[@]}"; do
if [[ ! "$value" =~ ^-?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?$ ]]; then
echo "Completion returned non-finite raw logits" >&2
return 1
fi
done
printf '%s|%s\n' "${token_id_arrays[index]}" "${logprob_arrays[index]}"
done
}
make_prompt() {
local length=$1
local offset=${2:-0}
local i
local prompt="Use the following system-design context to answer the question. Scenario $offset:"
for ((i = 0; i < length; i++)); do
prompt+=" context"
done
prompt+="\\n\\nQuestion: How should batch-invariant inference behave when this same request is decoded alongside unrelated requests?\\nAnswer:"
printf '"%s"' "$prompt"
}
first_difference() {
local actual=$1
local expected=$2
local position=$3
local actual_tokens=${actual%%|*}
local expected_tokens=${expected%%|*}
local actual_logits=${actual#*|}
local expected_logits=${expected#*|}
local -a actual_token_values
local -a expected_token_values
local -a actual_logit_values
local -a expected_logit_values
local step
IFS=, read -r -a actual_token_values <<<"$actual_tokens"
IFS=, read -r -a expected_token_values <<<"$expected_tokens"
IFS=, read -r -a actual_logit_values <<<"$actual_logits"
IFS=, read -r -a expected_logit_values <<<"$expected_logits"
for ((step = 0; step < num_decode_tokens; step++)); do
if [[ "${actual_token_values[step]}" != "${expected_token_values[step]}" ]]; then
printf 'batch position %s: step=%s, baseline token=%s, actual token=%s' \
"$position" \
"$step" \
"${expected_token_values[step]}" \
"${actual_token_values[step]}"
return
fi
if [[ "${actual_logit_values[step]}" != "${expected_logit_values[step]}" ]]; then
printf 'batch position %s: step=%s, token=%s, baseline logit=%s, actual logit=%s' \
"$position" \
"$step" \
"${expected_token_values[step]}" \
"${expected_logit_values[step]}" \
"${actual_logit_values[step]}"
return
fi
done
}
run_e2e() {
local needle
local batch
local baseline_output
local batch_response
local difference
local position
local -a fillers
local -a batch_outputs
local -a differences=()
echo "Testing PowerMoE-3b: TP=2, VLLM_BATCH_INVARIANT=1, VLLM_MOE_SKIP_PADDING=0"
server_log=$(mktemp "${TMPDIR:-/tmp}/xpu-batch-invariance.XXXXXX.log")
VLLM_BATCH_INVARIANT=1 vllm serve "$model" \
--tensor-parallel-size 2 \
--max-model-len 1024 \
--max-num-seqs 16 \
--max-num-batched-tokens 512 \
--enable-chunked-prefill \
--no-enable-prefix-caching \
--enforce-eager \
--seed 0 \
--max-logprobs 64 \
--logprobs-mode raw_logits \
--port "$port" > >(tee "$server_log") 2>&1 &
server_pid=$!
wait_for_server
needle=$(make_prompt 257)
local filler_lengths=(7 31 63 95 127 159 191 223 255 287 319 383 447)
for position in "${!filler_lengths[@]}"; do
fillers+=( "$(make_prompt "${filler_lengths[position]}" "$((position + 1))")" )
done
batch="[$needle,${fillers[0]},${fillers[1]},${fillers[2]},${fillers[3]},${fillers[4]},${fillers[5]},$needle,${fillers[6]},${fillers[7]},${fillers[8]},${fillers[9]},${fillers[10]},${fillers[11]},${fillers[12]},$needle]"
echo "Requesting BS=1 baseline ($num_decode_tokens decode tokens)..."
baseline_output=$(request_outputs "$needle" 1)
echo "Received BS=1 baseline."
echo "Requesting BS=16 batch ($((16 * num_decode_tokens)) decode tokens)..."
batch_response=$(request_outputs "$batch" 16)
mapfile -t batch_outputs <<<"$batch_response"
echo "Received BS=16 batch; comparing generated tokens and logits..."
for position in 0 7 15; do
difference=$(first_difference "${batch_outputs[position]}" "$baseline_output" "$position")
if [[ -n "$difference" ]]; then
differences+=("$difference")
fi
done
if [[ "${#differences[@]}" -ne 0 ]]; then
echo "Batch-invariant PowerMoE-3b output changed: ${differences[*]}" >&2
return 1
fi
echo "PowerMoE-3b batch invariance check passed"
cleanup_server
rm -f "$server_log"
server_log=
}
run_e2e