From 4708292d48f3f15978a6ad3befe5f0052bc86491 Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Mon, 29 Jun 2026 12:30:57 -0400 Subject: [PATCH] Bump flashinfer version to 0.6.13 (#46683) Signed-off-by: wzhao18 Co-authored-by: Jee Jee Li --- docker/Dockerfile | 2 +- docker/Dockerfile.nightly_torch | 4 ++-- docker/versions.json | 2 +- requirements/cuda.txt | 4 ++-- tests/evals/gsm8k/test_gsm8k_correctness.py | 11 +++++++++-- vllm/model_executor/warmup/kernel_warmup.py | 21 ++++++++++++++------- 6 files changed, 29 insertions(+), 15 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index c86795586c..945cb14bcb 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -793,7 +793,7 @@ RUN --mount=type=cache,target=/opt/uv/cache \ # Install FlashInfer JIT cache (requires CUDA-version-specific index URL) # https://docs.flashinfer.ai/installation.html # From versions.json: .flashinfer.version -ARG FLASHINFER_VERSION=0.6.12 +ARG FLASHINFER_VERSION=0.6.13 RUN --mount=type=cache,target=/opt/uv/cache \ uv pip install --system flashinfer-jit-cache==${FLASHINFER_VERSION} \ --index-url https://flashinfer.ai/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') diff --git a/docker/Dockerfile.nightly_torch b/docker/Dockerfile.nightly_torch index 0f2ec9f3a2..cc706f59ae 100644 --- a/docker/Dockerfile.nightly_torch +++ b/docker/Dockerfile.nightly_torch @@ -257,13 +257,13 @@ RUN pip install setuptools==75.6.0 packaging==23.2 ninja==1.11.1.3 build==1.2.2. # build flashinfer for torch nightly from source around 10 mins -# release version: v0.6.12 +# release version: v0.6.13 # todo(elainewy): cache flashinfer build result for faster build ENV CCACHE_DIR=/root/.cache/ccache RUN --mount=type=cache,target=/root/.cache/ccache \ --mount=type=cache,target=/root/.cache/uv \ echo "git clone flashinfer..." \ - && git clone --depth 1 --branch v0.6.12 --recursive https://github.com/flashinfer-ai/flashinfer.git \ + && git clone --depth 1 --branch v0.6.13 --recursive https://github.com/flashinfer-ai/flashinfer.git \ && cd flashinfer \ && git submodule update --init --recursive \ && echo "finish git clone flashinfer..." \ diff --git a/docker/versions.json b/docker/versions.json index 3145cfcc53..cfe1b1654f 100644 --- a/docker/versions.json +++ b/docker/versions.json @@ -68,7 +68,7 @@ "default": "true" }, "FLASHINFER_VERSION": { - "default": "0.6.12" + "default": "0.6.13" }, "GDRCOPY_CUDA_VERSION": { "default": "12.8" diff --git a/requirements/cuda.txt b/requirements/cuda.txt index 124dae4846..edaf9d2dd6 100644 --- a/requirements/cuda.txt +++ b/requirements/cuda.txt @@ -10,8 +10,8 @@ torchaudio==2.11.0 torchvision==0.26.0 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version PyNvVideoCodec==2.0.4 # FlashInfer should be updated together with the Dockerfile -flashinfer-python==0.6.12 -flashinfer-cubin==0.6.12 +flashinfer-python==0.6.13 +flashinfer-cubin==0.6.13 apache-tvm-ffi==0.1.9 tilelang==0.1.9 nvidia-cudnn-frontend>=1.19.1 diff --git a/tests/evals/gsm8k/test_gsm8k_correctness.py b/tests/evals/gsm8k/test_gsm8k_correctness.py index d14f41843b..f796f910bb 100644 --- a/tests/evals/gsm8k/test_gsm8k_correctness.py +++ b/tests/evals/gsm8k/test_gsm8k_correctness.py @@ -33,6 +33,8 @@ QUARK_MXFP4_TORCH_COMPATIBLE = find_spec("quark") is not None and ( else True ) +DEFAULT_STARTUP_MAX_WAIT_SECONDS = 1200 + def run_gsm8k_eval(eval_config: dict, server_url: str) -> dict: """Run GSM8K evaluation using our isolated script.""" @@ -127,7 +129,11 @@ def test_gsm8k_correctness(config_filename): ] ) - env_dict = eval_config.get("env", None) + startup_max_wait_seconds = eval_config.get( + "startup_max_wait_seconds", DEFAULT_STARTUP_MAX_WAIT_SECONDS + ) + env_dict = dict(eval_config.get("env") or {}) + env_dict["VLLM_ENGINE_READY_TIMEOUT_S"] = str(int(startup_max_wait_seconds)) print(f"Starting GSM8K evaluation for model: {eval_config['model_name']}") print(f"Expected metric threshold: {eval_config['accuracy_threshold']}") @@ -139,6 +145,7 @@ def test_gsm8k_correctness(config_filename): "rocm_request_timeout_seconds", request_timeout_seconds ) print(f"Request timeout: {request_timeout_seconds}s") + print(f"Startup max wait: {startup_max_wait_seconds}s") print(f"Server args: {' '.join(server_args)}") print(f"Environment variables: {env_dict}") @@ -147,7 +154,7 @@ def test_gsm8k_correctness(config_filename): eval_config["model_name"], server_args, env_dict=env_dict, - max_wait_seconds=eval_config.get("startup_max_wait_seconds", 600), + max_wait_seconds=startup_max_wait_seconds, ) as remote_server: server_url = remote_server.url_for("v1") print(f"Server started at: {server_url}") diff --git a/vllm/model_executor/warmup/kernel_warmup.py b/vllm/model_executor/warmup/kernel_warmup.py index 7edbff4d4a..d9b71b2a0c 100644 --- a/vllm/model_executor/warmup/kernel_warmup.py +++ b/vllm/model_executor/warmup/kernel_warmup.py @@ -113,12 +113,6 @@ def kernel_warmup(worker: "Worker"): ) -# TODO: remove once FlashInfer upstream fixes the persistent file cache -# to resolve collisions like `use_8x4_sf_layout=True/False`, which causes -# invalid tactics to be chosen -_FLASHINFER_USE_PERSISTENT_CACHE = False - - def flashinfer_autotune(runner: "GPUModelRunner") -> None: """ Autotune FlashInfer operations. @@ -135,7 +129,20 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None: import vllm.utils.flashinfer as fi_utils from vllm.distributed.parallel_state import get_world_group - if not _FLASHINFER_USE_PERSISTENT_CACHE: + use_persistent_cache = True + + deepep_a2a_backends = { + "deepep_high_throughput", + "deepep_low_latency", + "deepep_v2", + } + if runner.vllm_config.parallel_config.all2all_backend in deepep_a2a_backends: + # DeepEP dispatch/combine can timeout when only rank 0 + # performs autotune and falls behind other ranks. + # Thus we skip persistent cache in this case. + use_persistent_cache = False + + if not use_persistent_cache: with torch.inference_mode(), fi_utils.autotune(): runner._dummy_run( num_tokens=runner.scheduler_config.max_num_batched_tokens,