[XPU] enable XPU GRAPH by default (#51600)
Signed-off-by: zhenwei-intel <zhenwei.liu@intel.com>
This commit is contained in:
@@ -63,7 +63,7 @@ steps:
|
|||||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||||
'cd tests &&
|
'cd tests &&
|
||||||
pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py &&
|
pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py &&
|
||||||
VLLM_XPU_ENABLE_XPU_GRAPH=1 pytest -v -s test_config.py'
|
pytest -v -s test_config.py'
|
||||||
|
|
||||||
- label: V1 e2e (2 GPUs)
|
- label: V1 e2e (2 GPUs)
|
||||||
key: v1-e2e-2-gpus
|
key: v1-e2e-2-gpus
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ steps:
|
|||||||
key: xpu-example-test
|
key: xpu-example-test
|
||||||
depends_on:
|
depends_on:
|
||||||
- image-build-xpu
|
- image-build-xpu
|
||||||
timeout_in_minutes: 50
|
timeout_in_minutes: 70
|
||||||
device: intel_gpu
|
device: intel_gpu
|
||||||
agent_tags:
|
agent_tags:
|
||||||
label: production
|
label: production
|
||||||
@@ -47,7 +47,7 @@ steps:
|
|||||||
python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 --enable-expert-parallel &&
|
python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 --enable-expert-parallel &&
|
||||||
python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --max-model-len 8192 &&
|
python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --max-model-len 8192 &&
|
||||||
VLLM_XPU_FUSED_MOE_USE_REF=1 python3 examples/basic/offline_inference/generate.py --model Qwen/Qwen3-30B-A3B-Instruct-2507-FP8 --enforce-eager -tp 2 --max-model-len 8192 &&
|
VLLM_XPU_FUSED_MOE_USE_REF=1 python3 examples/basic/offline_inference/generate.py --model Qwen/Qwen3-30B-A3B-Instruct-2507-FP8 --enforce-eager -tp 2 --max-model-len 8192 &&
|
||||||
python3 examples/basic/offline_inference/generate.py --model INCModel/Qwen3-30B-A3B-Instruct-2507-MXFP4-LLMC --enforce-eager -tp 2 --max-model-len 8192
|
python3 examples/basic/offline_inference/generate.py --model INCModel/Qwen3-30B-A3B-Instruct-2507-MXFP4-LLMC -tp 2 --max-model-len 8192
|
||||||
'
|
'
|
||||||
- label: "XPU W8A8 FP8 Linear Examples"
|
- label: "XPU W8A8 FP8 Linear Examples"
|
||||||
key: xpu-w8a8-fp8-linear-examples
|
key: xpu-w8a8-fp8-linear-examples
|
||||||
@@ -196,16 +196,20 @@ steps:
|
|||||||
VLLM_TEST_DEVICE: "xpu"
|
VLLM_TEST_DEVICE: "xpu"
|
||||||
source_file_dependencies:
|
source_file_dependencies:
|
||||||
- vllm/compilation/
|
- vllm/compilation/
|
||||||
|
- vllm/config/compilation.py
|
||||||
- vllm/platforms/xpu.py
|
- vllm/platforms/xpu.py
|
||||||
|
- vllm/v1/cudagraph_dispatcher.py
|
||||||
|
- vllm/v1/worker/encoder_cudagraph.py
|
||||||
|
- vllm/v1/worker/encoder_cudagraph_defs.py
|
||||||
- vllm/v1/worker/xpu_model_runner.py
|
- vllm/v1/worker/xpu_model_runner.py
|
||||||
- vllm/v1/worker/xpu_worker.py
|
- vllm/v1/worker/xpu_worker.py
|
||||||
|
- tests/v1/cudagraph
|
||||||
- .buildkite/intel_jobs/test-intel.yaml
|
- .buildkite/intel_jobs/test-intel.yaml
|
||||||
commands:
|
commands:
|
||||||
- >-
|
- >-
|
||||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||||
'export VLLM_XPU_ENABLE_XPU_GRAPH=1 &&
|
'cd tests &&
|
||||||
python3 examples/basic/offline_inference/generate.py --model Qwen/Qwen3-0.6B &&
|
pytest -v -s v1/cudagraph/test_cudagraph_dispatch.py &&
|
||||||
python3 examples/basic/offline_inference/generate.py --model Qwen/Qwen3-0.6B --kv-cache-dtype fp8 &&
|
pytest -v -s v1/cudagraph/test_cudagraph_mode.py &&
|
||||||
python3 examples/basic/offline_inference/generate.py --model INCModel/Qwen3-30B-A3B-Instruct-2507-MXFP4-CT-AutoRound --max-model-len 4096'
|
pytest -v -s v1/cudagraph/test_breakable_cudagraph.py &&
|
||||||
# Temporarily disabled (multi-GPU/MXFP8 not yet supported with XPU Graph):
|
pytest -v -s v1/cudagraph/test_encoder_cudagraph.py'
|
||||||
# python3 examples/basic/offline_inference/generate.py --model INCModel/Qwen3-30B-A3B-Instruct-2507-MXFP8-CT-AutoRound -tp 2 --max-model-len 4096
|
|
||||||
|
|||||||
@@ -65,8 +65,6 @@ case "${test_suite}" in
|
|||||||
pytest -v -s quantization/test_compressed_tensors.py::test_compressed_tensors_fp8
|
pytest -v -s quantization/test_compressed_tensors.py::test_compressed_tensors_fp8
|
||||||
;;
|
;;
|
||||||
graph)
|
graph)
|
||||||
export VLLM_XPU_ENABLE_XPU_GRAPH=1
|
|
||||||
|
|
||||||
python3 examples/basic/offline_inference/generate.py --model Qwen/Qwen3-0.6B
|
python3 examples/basic/offline_inference/generate.py --model Qwen/Qwen3-0.6B
|
||||||
python3 examples/basic/offline_inference/generate.py --model Qwen/Qwen3-0.6B --kv-cache-dtype fp8
|
python3 examples/basic/offline_inference/generate.py --model Qwen/Qwen3-0.6B --kv-cache-dtype fp8
|
||||||
python3 examples/basic/offline_inference/generate.py --model INCModel/Qwen3-30B-A3B-Instruct-2507-MXFP4-CT-AutoRound --max-model-len 4096
|
python3 examples/basic/offline_inference/generate.py --model INCModel/Qwen3-30B-A3B-Instruct-2507-MXFP4-CT-AutoRound --max-model-len 4096
|
||||||
|
|||||||
@@ -146,6 +146,21 @@ MODEL_RUNNER_KWARGS: dict[str, dict[str, Any]] = {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# These checkpoints fail during XPU graph capture, so run them eagerly there.
|
||||||
|
XPU_EAGER_ONLY_MODELS = {
|
||||||
|
"OPEA/Qwen2.5-0.5B-Instruct-int4-sym-inc",
|
||||||
|
"Intel/Qwen2-0.5B-Instruct-int4-sym-AutoRound",
|
||||||
|
"Intel/Qwen3-8B-w2g64-for-ut",
|
||||||
|
"INCModel/Qwen3-30B-A3B-12L-W4A16-test",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _runner_kwargs(model: str) -> dict[str, Any]:
|
||||||
|
kwargs = dict(MODEL_RUNNER_KWARGS.get(model, {}))
|
||||||
|
if current_platform.is_xpu() and model in XPU_EAGER_ONLY_MODELS:
|
||||||
|
kwargs["enforce_eager"] = True
|
||||||
|
return kwargs
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skipif(
|
@pytest.mark.skipif(
|
||||||
not (
|
not (
|
||||||
@@ -157,7 +172,7 @@ MODEL_RUNNER_KWARGS: dict[str, dict[str, Any]] = {
|
|||||||
)
|
)
|
||||||
@pytest.mark.parametrize("model", MODELS + QWEN3_AUTOROUND_MODELS)
|
@pytest.mark.parametrize("model", MODELS + QWEN3_AUTOROUND_MODELS)
|
||||||
def test_auto_round_model(vllm_runner, model):
|
def test_auto_round_model(vllm_runner, model):
|
||||||
with vllm_runner(model, **MODEL_RUNNER_KWARGS.get(model, {})) as llm:
|
with vllm_runner(model, **_runner_kwargs(model)) as llm:
|
||||||
output = llm.generate_greedy(["The capital of France is"], max_tokens=8)
|
output = llm.generate_greedy(["The capital of France is"], max_tokens=8)
|
||||||
|
|
||||||
assert output
|
assert output
|
||||||
|
|||||||
@@ -317,7 +317,6 @@ if TYPE_CHECKING:
|
|||||||
VLLM_ELASTIC_EP_SCALE_UP_LAUNCH: bool = False
|
VLLM_ELASTIC_EP_SCALE_UP_LAUNCH: bool = False
|
||||||
VLLM_ELASTIC_EP_DRAIN_REQUESTS: bool = False
|
VLLM_ELASTIC_EP_DRAIN_REQUESTS: bool = False
|
||||||
VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: bool = True
|
VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: bool = True
|
||||||
VLLM_XPU_ENABLE_XPU_GRAPH: bool = False
|
|
||||||
VLLM_XPU_FORCE_N_CONTIG_WEIGHT: bool = False
|
VLLM_XPU_FORCE_N_CONTIG_WEIGHT: bool = False
|
||||||
VLLM_XPU_USE_SAMPLER_KERNEL: bool = True
|
VLLM_XPU_USE_SAMPLER_KERNEL: bool = True
|
||||||
VLLM_XPU_INC_WNA16_BACKEND: Literal["auto", "ark", "w4a16", "w4a8"] = "auto"
|
VLLM_XPU_INC_WNA16_BACKEND: Literal["auto", "ark", "w4a16", "w4a8"] = "auto"
|
||||||
@@ -2139,10 +2138,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
|||||||
"VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS": lambda: bool(
|
"VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS": lambda: bool(
|
||||||
int(os.getenv("VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS", "1"))
|
int(os.getenv("VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS", "1"))
|
||||||
),
|
),
|
||||||
# Whether enable XPU graph on Intel GPU
|
|
||||||
"VLLM_XPU_ENABLE_XPU_GRAPH": lambda: bool(
|
|
||||||
int(os.getenv("VLLM_XPU_ENABLE_XPU_GRAPH", "0"))
|
|
||||||
),
|
|
||||||
# Force N-contiguous weight layout for all XPU unquantized linears.
|
# Force N-contiguous weight layout for all XPU unquantized linears.
|
||||||
"VLLM_XPU_FORCE_N_CONTIG_WEIGHT": lambda: bool(
|
"VLLM_XPU_FORCE_N_CONTIG_WEIGHT": lambda: bool(
|
||||||
int(os.getenv("VLLM_XPU_FORCE_N_CONTIG_WEIGHT", "0"))
|
int(os.getenv("VLLM_XPU_FORCE_N_CONTIG_WEIGHT", "0"))
|
||||||
|
|||||||
@@ -336,16 +336,15 @@ class XPUPlatform(Platform):
|
|||||||
"XPU Graph is not supported in the current PyTorch version, "
|
"XPU Graph is not supported in the current PyTorch version, "
|
||||||
"disabling cudagraph_mode."
|
"disabling cudagraph_mode."
|
||||||
)
|
)
|
||||||
elif not envs.VLLM_XPU_ENABLE_XPU_GRAPH:
|
|
||||||
|
if (
|
||||||
|
vllm_config.model_config is not None
|
||||||
|
and vllm_config.model_config.enable_sleep_mode
|
||||||
|
and compilation_config.cudagraph_mode != CUDAGraphMode.NONE
|
||||||
|
):
|
||||||
compilation_config.cudagraph_mode = CUDAGraphMode.NONE
|
compilation_config.cudagraph_mode = CUDAGraphMode.NONE
|
||||||
logger.warning_once(
|
logger.warning_once(
|
||||||
"XPU Graph is disabled by environment variable, "
|
"XPU Graph is not compatible with sleep mode, disabling cudagraph_mode."
|
||||||
"please set VLLM_XPU_ENABLE_XPU_GRAPH=1 to enable it."
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
logger.warning_once(
|
|
||||||
"XPU Graph support is experimental and currently only supports "
|
|
||||||
"single-GPU execution."
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# Disable fusion passes not yet supported on XPU.
|
# Disable fusion passes not yet supported on XPU.
|
||||||
|
|||||||
@@ -616,12 +616,10 @@ class Worker(WorkerBase):
|
|||||||
# torch.accelerator.get_memory_info (reliable on ROCm, as used by
|
# torch.accelerator.get_memory_info (reliable on ROCm, as used by
|
||||||
# the AMD-CI mem tests), and graph_pool_handle resolves to the same
|
# the AMD-CI mem tests), and graph_pool_handle resolves to the same
|
||||||
# torch.cuda handle the live capture path already uses on ROCm.
|
# torch.cuda handle the live capture path already uses on ROCm.
|
||||||
# XPU stays excluded (see #39977).
|
|
||||||
cudagraph_memory_estimate = 0
|
cudagraph_memory_estimate = 0
|
||||||
if (
|
if (
|
||||||
current_platform.is_cuda_alike()
|
current_platform.is_cuda_alike() or current_platform.is_xpu()
|
||||||
and self.vllm_config.compilation_config.cudagraph_mode != CUDAGraphMode.NONE
|
) and self.vllm_config.compilation_config.cudagraph_mode != CUDAGraphMode.NONE:
|
||||||
):
|
|
||||||
cudagraph_memory_estimate = self.model_runner.profile_cudagraph_memory()
|
cudagraph_memory_estimate = self.model_runner.profile_cudagraph_memory()
|
||||||
|
|
||||||
# Respect the opt-in flag as originally designed.
|
# Respect the opt-in flag as originally designed.
|
||||||
|
|||||||
Reference in New Issue
Block a user