[Flashinfer] Upgrade Flashinfer version to 0.6.18 (#54313)
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Signed-off-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
This commit is contained in:
co-authored by
Jee Jee Li
Yongye Zhu
Jiangyun Zhu
parent
7a100bb617
commit
7ab2923489
+1
-1
@@ -943,7 +943,7 @@ RUN --mount=type=cache,target=/opt/uv/cache \
|
||||
# Install FlashInfer JIT cache (requires CUDA-version-specific index URL)
|
||||
# https://docs.flashinfer.ai/installation.html
|
||||
# From versions.json: .flashinfer.version
|
||||
ARG FLASHINFER_VERSION=0.6.17
|
||||
ARG FLASHINFER_VERSION=0.6.18
|
||||
RUN --mount=type=cache,target=/opt/uv/cache \
|
||||
if [ "${INSTALL_RUBIN_PRERELEASE}" != "true" ]; then \
|
||||
uv pip install --system flashinfer-jit-cache==${FLASHINFER_VERSION} \
|
||||
|
||||
@@ -71,7 +71,7 @@
|
||||
"default": "true"
|
||||
},
|
||||
"FLASHINFER_VERSION": {
|
||||
"default": "0.6.17"
|
||||
"default": "0.6.18"
|
||||
},
|
||||
"GDRCOPY_CUDA_VERSION": {
|
||||
"default": "12.8"
|
||||
|
||||
@@ -14,8 +14,8 @@ PyNvVideoCodec==2.0.4
|
||||
# flashinfer-cubin is not on PyPI since 0.6.14; setup.py excludes it from
|
||||
# install_requires so the published wheel does not carry an unresolvable pin
|
||||
--extra-index-url https://flashinfer.ai/whl/
|
||||
flashinfer-python==0.6.17
|
||||
flashinfer-cubin==0.6.17
|
||||
flashinfer-python==0.6.18
|
||||
flashinfer-cubin==0.6.18
|
||||
apache-tvm-ffi==0.1.11
|
||||
tilelang==0.1.12
|
||||
nvidia-cudnn-frontend>=1.19.1
|
||||
|
||||
@@ -3,4 +3,4 @@
|
||||
model_name: "openai/gpt-oss-20b"
|
||||
metric_threshold: 0.568
|
||||
reasoning_effort: "low"
|
||||
server_args: "--tensor-parallel-size 2 --moe-backend flashinfer_cutlass"
|
||||
server_args: "--tensor-parallel-size 2 --moe-backend flashinfer_cutlass --gpu-memory-utilization 0.85"
|
||||
|
||||
@@ -197,6 +197,8 @@ def fi_chunk_gated_delta_rule(
|
||||
fi_state = initial_state.to(torch.float32)
|
||||
fi_g = g.to(torch.float32)
|
||||
fi_beta = beta.to(torch.float32)
|
||||
if cu_seqlens is not None:
|
||||
cu_seqlens = cu_seqlens.to(torch.int64)
|
||||
result = chunk_gated_delta_rule_fi(
|
||||
q=q,
|
||||
k=k,
|
||||
|
||||
Reference in New Issue
Block a user