[Feat][XPU] VLLM_BATCH_INVARIANT support for Dense/MoE models (#55881)

Signed-off-by: Tony Lin <tony.lin@intel.com>
Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
Tony Lin
2026-09-22 11:06:54 -04:00
committed by GitHub
co-authored by Flora Feng
parent 1ea7c63f4a
commit e4340e41c9
10 changed files with 619 additions and 8 deletions
+29 -1
View File
@@ -2,6 +2,34 @@ group: Miscellaneous Intel
depends_on:
- image-build-xpu
steps:
- label: XPU Batch Invariance
key: xpu-batch-invariance
timeout_in_minutes: 60
num_devices: 2
device: intel_gpu
agent_tags:
label: production
gpu: 2+
mem: 24+
no_plugin: true
working_dir: "."
env:
REGISTRY: "public.ecr.aws/q9t5s3a7"
REPO: "vllm-ci-test-repo"
VLLM_TEST_DEVICE: "xpu"
source_file_dependencies:
- vllm/
- tests/v1/determinism/test_xpu_batch_invariant_ut.py
- .buildkite/scripts/hardware_ci/run-xpu-batch-invariance.sh
- .buildkite/intel_jobs/misc_intel.yaml
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'export VLLM_BATCH_INVARIANT=1 &&
cd tests &&
pytest -v -s v1/determinism/test_xpu_batch_invariant_ut.py &&
bash ../.buildkite/scripts/hardware_ci/run-xpu-batch-invariance.sh'
- label: V1 Core + KV + Metrics
key: v1-core-kv-metrics-intel
timeout_in_minutes: 45
@@ -290,4 +318,4 @@ steps:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'cd tests &&
pytest -v -s compile/passes/test_qk_norm_rope_fusion.py'
pytest -v -s compile/passes/test_qk_norm_rope_fusion.py'
+243
View File
@@ -0,0 +1,243 @@
#!/usr/bin/env bash
set -euo pipefail
repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../../.." && pwd)"
model="ibm-research/PowerMoE-3b"
port=8000
num_decode_tokens=128
server_pid=
server_log=
# XPU MoE kernels may not safely handle the -1 routes emitted for padding.
export VLLM_MOE_SKIP_PADDING=0
cd "$repo_root"
cleanup_server() {
local attempt
if [[ -n "$server_pid" ]] && kill -0 "$server_pid" 2>/dev/null; then
kill "$server_pid" 2>/dev/null || true
for ((attempt = 0; attempt < 30; attempt++)); do
if ! kill -0 "$server_pid" 2>/dev/null; then
break
fi
sleep 1
done
if kill -0 "$server_pid" 2>/dev/null; then
echo "vLLM did not stop after 30 seconds; sending SIGKILL" >&2
kill -KILL "$server_pid" 2>/dev/null || true
fi
wait "$server_pid" 2>/dev/null || true
fi
server_pid=
}
on_exit() {
local status=$?
cleanup_server
if [[ "$status" -ne 0 && -n "$server_log" && -f "$server_log" ]]; then
if command -v buildkite-agent >/dev/null 2>&1; then
buildkite-agent artifact upload "$server_log" ||
echo "Failed to upload $server_log as a Buildkite artifact" >&2
fi
fi
[[ -z "$server_log" ]] || rm -f "$server_log"
exit "$status"
}
trap on_exit EXIT
wait_for_server() {
for _ in $(seq 1 600); do
if curl -sf "http://127.0.0.1:$port/health" >/dev/null; then
return
fi
if ! kill -0 "$server_pid" 2>/dev/null; then
echo "vLLM exited before becoming healthy; see the server log above." >&2
return 1
fi
sleep 1
done
echo "Timed out waiting for vLLM server health check" >&2
return 1
}
request_outputs() {
local prompts_json=$1
local expected_choices=$2
local response
response=$(printf \
'{"model":"%s","prompt":%s,"temperature":0.7,"top_p":0.95,"seed":0,"max_tokens":%s,"logprobs":64,"ignore_eos":true,"return_token_ids":true}' \
"$model" "$prompts_json" "$num_decode_tokens" |
curl --fail --silent --show-error \
--max-time 1200 \
"http://127.0.0.1:$port/v1/completions" \
-H "Content-Type: application/json" \
--data-binary @-)
local -a token_id_arrays logprob_arrays
mapfile -t token_id_arrays < <(
grep -o '"token_ids":\[[^]]*\]' <<<"$response" |
sed 's/^"token_ids":\[//; s/\]$//'
)
mapfile -t logprob_arrays < <(
grep -o '"token_logprobs":\[[^]]*\]' <<<"$response" |
sed 's/^"token_logprobs":\[//; s/\]$//'
)
if [[ "${#token_id_arrays[@]}" -ne "$expected_choices" ]] ||
[[ "${#logprob_arrays[@]}" -ne "$expected_choices" ]]; then
echo "Expected $expected_choices choices with token IDs and logits" >&2
return 1
fi
local index
local array
local value
local -a values
for index in "${!logprob_arrays[@]}"; do
array="${logprob_arrays[index]}"
IFS=, read -r -a values <<<"$array"
if [[ "${#values[@]}" -ne "$num_decode_tokens" ]]; then
echo "Completion returned an unexpected number of decode steps" >&2
return 1
fi
IFS=, read -r -a values <<<"${token_id_arrays[index]}"
if [[ "${#values[@]}" -ne "$num_decode_tokens" ]]; then
echo "Completion returned an unexpected number of generated tokens" >&2
return 1
fi
for value in "${values[@]}"; do
if [[ ! "$value" =~ ^[0-9]+$ ]]; then
echo "Completion returned an invalid token ID" >&2
return 1
fi
done
IFS=, read -r -a values <<<"$array"
for value in "${values[@]}"; do
if [[ ! "$value" =~ ^-?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?$ ]]; then
echo "Completion returned non-finite raw logits" >&2
return 1
fi
done
printf '%s|%s\n' "${token_id_arrays[index]}" "${logprob_arrays[index]}"
done
}
make_prompt() {
local length=$1
local offset=${2:-0}
local i
local prompt="Use the following system-design context to answer the question. Scenario $offset:"
for ((i = 0; i < length; i++)); do
prompt+=" context"
done
prompt+="\\n\\nQuestion: How should batch-invariant inference behave when this same request is decoded alongside unrelated requests?\\nAnswer:"
printf '"%s"' "$prompt"
}
first_difference() {
local actual=$1
local expected=$2
local position=$3
local actual_tokens=${actual%%|*}
local expected_tokens=${expected%%|*}
local actual_logits=${actual#*|}
local expected_logits=${expected#*|}
local -a actual_token_values
local -a expected_token_values
local -a actual_logit_values
local -a expected_logit_values
local step
IFS=, read -r -a actual_token_values <<<"$actual_tokens"
IFS=, read -r -a expected_token_values <<<"$expected_tokens"
IFS=, read -r -a actual_logit_values <<<"$actual_logits"
IFS=, read -r -a expected_logit_values <<<"$expected_logits"
for ((step = 0; step < num_decode_tokens; step++)); do
if [[ "${actual_token_values[step]}" != "${expected_token_values[step]}" ]]; then
printf 'batch position %s: step=%s, baseline token=%s, actual token=%s' \
"$position" \
"$step" \
"${expected_token_values[step]}" \
"${actual_token_values[step]}"
return
fi
if [[ "${actual_logit_values[step]}" != "${expected_logit_values[step]}" ]]; then
printf 'batch position %s: step=%s, token=%s, baseline logit=%s, actual logit=%s' \
"$position" \
"$step" \
"${expected_token_values[step]}" \
"${expected_logit_values[step]}" \
"${actual_logit_values[step]}"
return
fi
done
}
run_e2e() {
local needle
local batch
local baseline_output
local batch_response
local difference
local position
local -a fillers
local -a batch_outputs
local -a differences=()
echo "Testing PowerMoE-3b: TP=2, VLLM_BATCH_INVARIANT=1, VLLM_MOE_SKIP_PADDING=0"
server_log=$(mktemp "${TMPDIR:-/tmp}/xpu-batch-invariance.XXXXXX.log")
VLLM_BATCH_INVARIANT=1 vllm serve "$model" \
--tensor-parallel-size 2 \
--max-model-len 1024 \
--max-num-seqs 16 \
--max-num-batched-tokens 512 \
--enable-chunked-prefill \
--no-enable-prefix-caching \
--enforce-eager \
--seed 0 \
--max-logprobs 64 \
--logprobs-mode raw_logits \
--port "$port" > >(tee "$server_log") 2>&1 &
server_pid=$!
wait_for_server
needle=$(make_prompt 257)
local filler_lengths=(7 31 63 95 127 159 191 223 255 287 319 383 447)
for position in "${!filler_lengths[@]}"; do
fillers+=( "$(make_prompt "${filler_lengths[position]}" "$((position + 1))")" )
done
batch="[$needle,${fillers[0]},${fillers[1]},${fillers[2]},${fillers[3]},${fillers[4]},${fillers[5]},$needle,${fillers[6]},${fillers[7]},${fillers[8]},${fillers[9]},${fillers[10]},${fillers[11]},${fillers[12]},$needle]"
echo "Requesting BS=1 baseline ($num_decode_tokens decode tokens)..."
baseline_output=$(request_outputs "$needle" 1)
echo "Received BS=1 baseline."
echo "Requesting BS=16 batch ($((16 * num_decode_tokens)) decode tokens)..."
batch_response=$(request_outputs "$batch" 16)
mapfile -t batch_outputs <<<"$batch_response"
echo "Received BS=16 batch; comparing generated tokens and logits..."
for position in 0 7 15; do
difference=$(first_difference "${batch_outputs[position]}" "$baseline_output" "$position")
if [[ -n "$difference" ]]; then
differences+=("$difference")
fi
done
if [[ "${#differences[@]}" -ne 0 ]]; then
echo "Batch-invariant PowerMoE-3b output changed: ${differences[*]}" >&2
return 1
fi
echo "PowerMoE-3b batch invariance check passed"
cleanup_server
rm -f "$server_log"
server_log=
}
run_e2e
@@ -0,0 +1,228 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""XPU-only batch-invariance tests for norm, collectives, and sampler behavior."""
import pytest
import torch
from vllm import envs
from vllm.config import CacheConfig, LoadConfig, ModelConfig, VllmConfig
from vllm.distributed.device_communicators.xpu_communicator import XpuCommunicator
from vllm.platforms import current_platform
pytestmark = pytest.mark.skipif(
not current_platform.is_xpu(), reason="XPU batch-invariance tests"
)
@pytest.fixture
def tiny_qwen3_moe_path(tmp_path):
"""Create a local Qwen3-MoE config for dummy-loading tests."""
from transformers import Qwen3MoeConfig
config = Qwen3MoeConfig(
architectures=["Qwen3MoeForCausalLM"],
vocab_size=256,
hidden_size=512,
intermediate_size=1024,
moe_intermediate_size=512,
num_hidden_layers=2,
num_attention_heads=4,
num_key_value_heads=2,
head_dim=128,
num_experts=4,
num_experts_per_tok=2,
max_position_embeddings=512,
bos_token_id=1,
eos_token_id=2,
tie_word_embeddings=False,
)
config.save_pretrained(tmp_path)
return tmp_path
@pytest.mark.parametrize("source", ["argument", "checkpoint"])
def test_rejects_weight_quantization(tiny_qwen3_moe_path, source):
"""Reject both explicit and checkpoint-inferred quantization before loading."""
from transformers import Qwen3MoeConfig
if source == "checkpoint":
config = Qwen3MoeConfig.from_pretrained(tiny_qwen3_moe_path)
config.quantization_config = {
"quant_method": "fp8",
"activation_scheme": "dynamic",
}
config.save_pretrained(tiny_qwen3_moe_path)
model_config = ModelConfig(
model=str(tiny_qwen3_moe_path),
skip_tokenizer_init=True,
dtype="bfloat16",
quantization="fp8" if source == "argument" else None,
)
assert model_config.quantization == "fp8"
with pytest.raises(ValueError, match="supports only unquantized models"):
VllmConfig(
model_config=model_config,
load_config=LoadConfig(load_format="dummy"),
)
@pytest.mark.parametrize("cache_dtype", ["fp8", "int8_per_token_head", "nvfp4"])
def test_rejects_quantized_kv_cache(cache_dtype):
with pytest.raises(ValueError, match="does not support quantized KV caches"):
VllmConfig(cache_config=CacheConfig(cache_dtype=cache_dtype))
@pytest.mark.parametrize("cache_dtype", ["auto", "float16", "bfloat16"])
def test_accepts_unquantized_kv_cache(cache_dtype):
config = VllmConfig(cache_config=CacheConfig(cache_dtype=cache_dtype))
assert config.cache_config.cache_dtype == cache_dtype
def test_quantized_kv_cache_allowed_without_batch_invariance(monkeypatch):
monkeypatch.setattr(envs, "VLLM_BATCH_INVARIANT", False)
config = VllmConfig(cache_config=CacheConfig(cache_dtype="fp8"))
assert config.cache_config.cache_dtype == "fp8"
# These dimensions and seeds locally reproduce non-invariant norm mismatches.
# They make the invariant equality assertion exercise a discriminating case.
@pytest.mark.parametrize(
(
"norm_name",
"batch_size",
"hidden_size",
"position",
"weight_seed",
"input_seed",
"input_pattern",
),
[
("rms", 1024, 16384, 512, 7, 42, "normal"),
("gemma", 257, 8192, 255, 1, 2, "normal"),
("rms", 5, 8192, 3, 7, 46, "scaled"),
("gemma", 5, 8192, 3, 1, 46, "scaled"),
],
)
def test_residual_norm_preserves_batch_invariance(
norm_name,
batch_size,
hidden_size,
position,
weight_seed,
input_seed,
input_pattern,
monkeypatch,
default_vllm_config,
):
"""Residual norm output must not change when its row changes batch position."""
from vllm.model_executor.determinism.batch_invariant import init_batch_invariance
from vllm.model_executor.layers.layernorm import GemmaRMSNorm, RMSNorm
monkeypatch.setattr(envs, "VLLM_BATCH_INVARIANT", True)
init_batch_invariance()
norm_cls = RMSNorm if norm_name == "rms" else GemmaRMSNorm
norm = norm_cls(hidden_size, eps=1e-6).to(device="xpu", dtype=torch.bfloat16)
with torch.no_grad():
torch.manual_seed(weight_seed)
norm.weight.uniform_(-0.5, 0.5)
if input_pattern == "scaled":
# Heterogeneous columns expose native mean's batch-dependent rounding.
generator = torch.Generator(device="cpu").manual_seed(input_seed)
scale = torch.randn(hidden_size, generator=generator).exp()
x = (torch.randn(batch_size, hidden_size, generator=generator) * scale).to(
device="xpu", dtype=torch.bfloat16
)
residual = (
torch.randn(batch_size, hidden_size, generator=generator) * scale
).to(device="xpu", dtype=torch.bfloat16)
else:
torch.manual_seed(input_seed)
x = torch.randn(batch_size, hidden_size, device="xpu", dtype=torch.bfloat16)
residual = torch.randn_like(x)
single_output, single_residual = norm(
x[position : position + 1].clone(),
residual[position : position + 1].clone(),
)
batch_output, batch_residual = norm(x.clone(), residual.clone())
assert torch.equal(single_output, batch_output[position : position + 1])
assert torch.equal(single_residual, batch_residual[position : position + 1])
@pytest.mark.parametrize(
"operation", ["all_reduce", "reduce_scatter", "reduce_scatterv"]
)
@pytest.mark.parametrize("rank", range(4))
def test_collectives_preserve_rank_order_and_partition(monkeypatch, operation, rank):
"""Cancellation exposes reordered sums; mock ranks require no extra XPUs."""
from vllm.distributed.device_communicators import xpu_communicator
inputs = [
torch.full((8, 3), value, dtype=torch.float32, device="xpu")
for value in (1e20, -1e20, 3.0, 0.0)
]
inputs[2].add_(torch.arange(24, device="xpu").reshape(8, 3))
def all_gather_single(output, input_, group):
torch.testing.assert_close(input_, inputs[rank].reshape(-1), rtol=0, atol=0)
output.copy_(torch.stack(inputs).reshape(-1))
monkeypatch.setattr(xpu_communicator.dist, "all_gather_single", all_gather_single)
communicator = XpuCommunicator.__new__(XpuCommunicator)
communicator.world_size = 4
communicator.rank_in_group = rank
communicator.device_group = object()
expected = inputs[0].clone()
for value in inputs[1:]:
expected.add_(value)
assert torch.equal(expected, inputs[2])
if operation == "all_reduce":
actual = communicator.all_reduce(inputs[rank])
elif operation == "reduce_scatter":
actual = communicator.reduce_scatter(inputs[rank], dim=0)
expected = expected.chunk(4)[rank]
else:
sizes = [1, 3, 2, 2]
actual = communicator.reduce_scatterv(inputs[rank], dim=0, sizes=sizes)
expected = expected.split(sizes)[rank]
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
# This batch size, vocabulary size, and positions locally reproduce an
# unseeded non-invariant XPU sampler mismatch.
@pytest.mark.parametrize("position", [0, 128, 255])
def test_seeded_sampler_preserves_full_distribution_and_rng(position, monkeypatch):
"""Moving a request must preserve its distribution and per-request RNG."""
from vllm.triton_utils import HAS_TRITON
from vllm.v1.sample.ops.topk_topp_sampler import TopKTopPSampler
assert HAS_TRITON, "XPU sampling must exercise the Triton top-k/top-p path"
monkeypatch.setattr(envs, "VLLM_XPU_USE_SAMPLER_KERNEL", True)
sampler = TopKTopPSampler(logprobs_mode="processed_logprobs")
torch.manual_seed(42)
logits = torch.randn(256, 8192, device="xpu", dtype=torch.float32)
single_rng = torch.Generator(device="xpu").manual_seed(123)
batch_rng = torch.Generator(device="xpu").manual_seed(123)
k = torch.full((256,), 32, dtype=torch.int32, device="xpu")
p = torch.full((256,), 0.9, device="xpu")
for _ in range(3):
single_token, single_logprobs = sampler(
logits[position : position + 1].clone(),
{0: single_rng},
k[position : position + 1],
p[position : position + 1],
)
batch_tokens, batch_logprobs = sampler(
logits.clone(), {position: batch_rng}, k, p
)
assert single_logprobs is not None and batch_logprobs is not None
torch.testing.assert_close(
single_logprobs[0], batch_logprobs[position], rtol=0, atol=0
)
assert single_token.item() == batch_tokens[position].item()
assert torch.equal(single_rng.get_state(), batch_rng.get_state())
@@ -6,6 +6,7 @@ import torch
import torch.distributed as dist
from torch.distributed import ProcessGroup
import vllm.envs as envs
from vllm.logger import init_logger
from .base_device_communicator import DeviceCommunicatorBase
@@ -45,7 +46,25 @@ class XpuCommunicator(DeviceCommunicatorBase):
self.all2all_manager = AgRsAll2AllManager(self.cpu_group)
logger.info("Using AgRs manager on XPU device.")
def _fixed_rank_sum(self, input_: torch.Tensor) -> torch.Tensor:
flat_input = input_.reshape(-1)
gathered = torch.empty(
(self.world_size, flat_input.numel()),
dtype=input_.dtype,
device=input_.device,
)
dist.all_gather_single(gathered.view(-1), flat_input, group=self.device_group)
output = gathered[0].clone()
for rank in range(1, self.world_size):
output.add_(gathered[rank])
return output.view(input_.shape)
def all_reduce(self, input_: torch.Tensor) -> torch.Tensor:
if envs.VLLM_BATCH_INVARIANT:
# XCCL changes its reduction order with message size. Gather first
# and accumulate in rank order, independent of the token batch.
return self._fixed_rank_sum(input_)
output = input_.clone()
dist.all_reduce(output, group=self.device_group)
return output
@@ -69,7 +88,12 @@ class XpuCommunicator(DeviceCommunicatorBase):
output_shape, dtype=input_tensor.dtype, device=input_tensor.device
)
dist.reduce_scatter_tensor(output, input_tensor, group=self.device_group)
if envs.VLLM_BATCH_INVARIANT:
reduced = self._fixed_rank_sum(input_tensor)
start = self.rank_in_group * chunk_size
output.copy_(reduced.narrow(0, start, chunk_size))
else:
dist.reduce_scatter_tensor(output, input_tensor, group=self.device_group)
# Reshape before returning
return output.movedim(0, dim).contiguous()
@@ -99,7 +123,15 @@ class XpuCommunicator(DeviceCommunicatorBase):
output = torch.empty(
output_shape, dtype=input_tensor.dtype, device=input_tensor.device
)
if sizes is None or sizes.count(sizes[0]) == len(sizes):
if envs.VLLM_BATCH_INVARIANT:
reduced = self._fixed_rank_sum(input_tensor)
start = (
self.rank_in_group * chunk_size
if sizes is None
else sum(sizes[: self.rank_in_group])
)
output.copy_(reduced.narrow(0, start, chunk_size))
elif sizes is None or sizes.count(sizes[0]) == len(sizes):
dist.reduce_scatter_tensor(output, input_tensor, group=self.device_group)
elif torch.xpu.device_count() < self.world_size:
# uneven reduce_scatter is not supported when the
@@ -191,7 +223,7 @@ class XpuCommunicator(DeviceCommunicatorBase):
(self.world_size,) + input_size, dtype=input_.dtype, device=input_.device
)
# All-gather.
dist.all_gather_into_tensor(output_tensor, input_, group=self.device_group)
dist.all_gather_single(output_tensor, input_, group=self.device_group)
if self.rank_in_group == dst:
# Reshape
output_tensor = output_tensor.movedim(0, dim)
@@ -1104,7 +1104,10 @@ def enable_batch_invariant_mode():
_fp16_block_size_n = 128
# Softmax, log_softmax, mean, and bmm are already batch-invariant on XPU
# Native mean can change reduction order with the row count on XPU too.
_batch_invariant_LIB.impl("aten::mean.dim", mean_batch_invariant, key)
# Softmax, log_softmax, and bmm are already batch-invariant on XPU
# (oneDNN backend produces bitwise-identical results regardless of batch
# context). Only register these overrides on CUDA where they are needed.
if not current_platform.is_xpu():
@@ -1113,7 +1116,6 @@ def enable_batch_invariant_mode():
)
_batch_invariant_LIB.impl("aten::softmax", softmax_batch_invariant, key)
_batch_invariant_LIB.impl("aten::_softmax", softmax_batch_invariant, key)
_batch_invariant_LIB.impl("aten::mean.dim", mean_batch_invariant, key)
# torch 2.12+ registers a built-in Triton bmm kernel for CUDA
# (torch._native.ops.bmm_outer_product), so we need allow_override
# to replace it at the dispatcher level.
@@ -94,7 +94,13 @@ def _get_priority_backends(moe_config: FusedMoEConfig) -> list[UnquantizedMoeBac
_move_to_back(_AVAILABLE_BACKENDS, UnquantizedMoeBackend.FLASHINFER_CUTLASS)
elif current_platform.is_xpu():
_AVAILABLE_BACKENDS = [UnquantizedMoeBackend.XPU]
# XPUExperts does not support batch invariance; TRITON is kept as a
# fallback so VLLM_BATCH_INVARIANT=1 still resolves to a valid
# (batch-invariant) backend on XPU.
_AVAILABLE_BACKENDS = [
UnquantizedMoeBackend.XPU,
UnquantizedMoeBackend.TRITON,
]
elif current_platform.is_cpu():
_AVAILABLE_BACKENDS = [UnquantizedMoeBackend.CPU]
return _AVAILABLE_BACKENDS
+17
View File
@@ -119,6 +119,14 @@ class RMSNorm(CustomOp):
x: torch.Tensor,
residual: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
if envs.VLLM_BATCH_INVARIANT and residual is not None:
assert self.variance_size_override is None, (
"Batch invariance is not supported for variance_size_override"
)
weight = self.weight.data if self.pass_weight_add else None
return ir.ops.fused_add_rms_norm.impls["native"].impl_fn(
x, residual, weight, self.variance_epsilon
)
return self.forward_cuda(x, residual)
def extra_repr(self) -> str:
@@ -171,6 +179,15 @@ class GemmaRMSNorm(CustomOp):
x: torch.Tensor,
residual: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
if envs.VLLM_BATCH_INVARIANT:
if residual is None:
return self.forward_cuda(x, residual)
return ir.ops.fused_add_rms_norm.impls["native"].impl_fn(
x,
residual,
self.weight.float() + 1.0,
self.variance_epsilon,
)
import vllm._xpu_ops # noqa: F401 registers torch.ops.vllm.xpu_gemma_rms_norm
# Fall back to the native path if the fused gemma kernels are not
+40
View File
@@ -162,6 +162,25 @@ class XPUPlatform(Platform):
if selected_backend == AttentionBackendEnum.TRITON_ATTN:
logger.info_once("Using Triton backend.")
return AttentionBackendEnum.TRITON_ATTN.get_path()
elif attn_selector_config.use_batch_invariant:
# Flash Attention on XPU has not been validated for batch
# invariance. Honor an explicit Flash Attention request;
# otherwise fall back to Triton Attention, which implements
# batch-invariant kernels.
if selected_backend == AttentionBackendEnum.FLASH_ATTN:
logger.warning_once(
"Using Flash Attention on XPU with batch invariance "
"enabled because it was explicitly requested. This "
"backend has not been validated for batch invariance "
"on XPU and may produce non-deterministic results "
"across batch sizes."
)
return AttentionBackendEnum.FLASH_ATTN.get_path()
logger.info_once(
"VLLM_BATCH_INVARIANT is enabled. Using Triton Attention "
"backend on XPU, which implements batch-invariant kernels."
)
return AttentionBackendEnum.TRITON_ATTN.get_path()
elif attn_selector_config.use_mm_prefix:
# Flash Attention on XPU has no FA4 kernel, so it cannot apply the
# multimodal prefix-LM bidirectional mask. Honor an explicit Flash
@@ -283,6 +302,27 @@ class XPUPlatform(Platform):
# lazy import to avoid circular import
from vllm.config import CUDAGraphMode
if envs.VLLM_BATCH_INVARIANT:
model_config = vllm_config.model_config
if model_config is not None and (
model_config.quantization is not None
or vllm_config.quant_config is not None
):
raise ValueError(
"XPU batch invariance currently supports only unquantized "
f"models; got quantization={model_config.quantization!r}. "
"Use an unquantized model or disable VLLM_BATCH_INVARIANT."
)
cache_dtype = vllm_config.cache_config.cache_dtype
if cache_dtype not in ("auto", "float16", "bfloat16"):
raise ValueError(
"XPU batch invariance currently does not support quantized "
f"KV caches; got kv_cache_dtype={cache_dtype!r}. "
"Use an unquantized KV cache dtype or disable "
"VLLM_BATCH_INVARIANT."
)
compilation_config = vllm_config.compilation_config
if compilation_config.compile_sizes is None:
compilation_config.compile_sizes = []
+10
View File
@@ -526,6 +526,16 @@ class SamplingParams(
self._verify_args()
if (
envs.VLLM_BATCH_INVARIANT
and self.temperature >= _SAMPLING_EPS
and self.seed is None
):
logger.warning_once(
"Random sampling without an explicit seed may not be batch "
"invariant. Set seed in SamplingParams or use temperature=0."
)
if self.temperature < _SAMPLING_EPS:
# Zero temperature means greedy sampling.
self.top_p = 1.0
+6 -1
View File
@@ -224,9 +224,14 @@ class TopKTopPSampler(nn.Module):
else:
self.forward = self.forward_cpu
elif current_platform.is_xpu():
if xpu_sampler_supported():
if xpu_sampler_supported() and not envs.VLLM_BATCH_INVARIANT:
self.forward = self.forward_xpu
else:
if envs.VLLM_BATCH_INVARIANT:
logger.info_once(
"VLLM_BATCH_INVARIANT is enabled. Using the "
"PyTorch-native sampler on XPU."
)
self.forward = self.forward_native
elif (
logprobs_mode not in PROCESSED_LOGPROBS_MODES