From 7ab29234890b29b005e46b53037309e597425095 Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Sun, 30 Aug 2026 19:01:05 -0400 Subject: [PATCH] [Flashinfer] Upgrade Flashinfer version to 0.6.18 (#54313) Signed-off-by: wzhao18 Signed-off-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Co-authored-by: Jee Jee Li Co-authored-by: Yongye Zhu Co-authored-by: Jiangyun Zhu --- docker/Dockerfile | 2 +- docker/versions.json | 2 +- requirements/cuda.txt | 4 ++-- .../configs/gpt-oss-20b-flashinfer-mxfp4-bf16-cutlass.yaml | 2 +- vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py | 2 ++ 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 90f299c26a..aba7b17279 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -943,7 +943,7 @@ RUN --mount=type=cache,target=/opt/uv/cache \ # Install FlashInfer JIT cache (requires CUDA-version-specific index URL) # https://docs.flashinfer.ai/installation.html # From versions.json: .flashinfer.version -ARG FLASHINFER_VERSION=0.6.17 +ARG FLASHINFER_VERSION=0.6.18 RUN --mount=type=cache,target=/opt/uv/cache \ if [ "${INSTALL_RUBIN_PRERELEASE}" != "true" ]; then \ uv pip install --system flashinfer-jit-cache==${FLASHINFER_VERSION} \ diff --git a/docker/versions.json b/docker/versions.json index 85ca558fdf..ed12f03700 100644 --- a/docker/versions.json +++ b/docker/versions.json @@ -71,7 +71,7 @@ "default": "true" }, "FLASHINFER_VERSION": { - "default": "0.6.17" + "default": "0.6.18" }, "GDRCOPY_CUDA_VERSION": { "default": "12.8" diff --git a/requirements/cuda.txt b/requirements/cuda.txt index a2c0364d1f..ffcff13c82 100644 --- a/requirements/cuda.txt +++ b/requirements/cuda.txt @@ -14,8 +14,8 @@ PyNvVideoCodec==2.0.4 # flashinfer-cubin is not on PyPI since 0.6.14; setup.py excludes it from # install_requires so the published wheel does not carry an unresolvable pin --extra-index-url https://flashinfer.ai/whl/ -flashinfer-python==0.6.17 -flashinfer-cubin==0.6.17 +flashinfer-python==0.6.18 +flashinfer-cubin==0.6.18 apache-tvm-ffi==0.1.11 tilelang==0.1.12 nvidia-cudnn-frontend>=1.19.1 diff --git a/tests/evals/gpt_oss/configs/gpt-oss-20b-flashinfer-mxfp4-bf16-cutlass.yaml b/tests/evals/gpt_oss/configs/gpt-oss-20b-flashinfer-mxfp4-bf16-cutlass.yaml index 992cb3dfa4..1366968f11 100644 --- a/tests/evals/gpt_oss/configs/gpt-oss-20b-flashinfer-mxfp4-bf16-cutlass.yaml +++ b/tests/evals/gpt_oss/configs/gpt-oss-20b-flashinfer-mxfp4-bf16-cutlass.yaml @@ -3,4 +3,4 @@ model_name: "openai/gpt-oss-20b" metric_threshold: 0.568 reasoning_effort: "low" -server_args: "--tensor-parallel-size 2 --moe-backend flashinfer_cutlass" +server_args: "--tensor-parallel-size 2 --moe-backend flashinfer_cutlass --gpu-memory-utilization 0.85" diff --git a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py index 743752b296..84ea15d1a6 100644 --- a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py +++ b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py @@ -197,6 +197,8 @@ def fi_chunk_gated_delta_rule( fi_state = initial_state.to(torch.float32) fi_g = g.to(torch.float32) fi_beta = beta.to(torch.float32) + if cu_seqlens is not None: + cu_seqlens = cu_seqlens.to(torch.int64) result = chunk_gated_delta_rule_fi( q=q, k=k,