[Bugfix][Transformers backend] Fix CUDA graph capture for dynamic-rope models (#58064)

Signed-off-by: YongfengX <erkang040804@gmail.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
Frank
2026-10-14 05:56:54 -07:00
committed by GitHub
co-authored by Claude Opus 5 Harry Mellor Kevin H. Luu
parent 2631afd3fe
commit 3723a1ea96
4 changed files with 132 additions and 4 deletions
@@ -66,6 +66,7 @@ from vllm.model_executor.models.transformers.fusers.attention import (
from vllm.model_executor.models.transformers.utils import (
attrsetter,
can_enable_torch_compile,
disable_noop_dynamic_rope,
get_feature_request_tip,
init_on_device_without_buffers,
log_replacement,
@@ -182,6 +183,8 @@ class Base(
self._create_hf_to_vllm_mapper()
# Remove layers not on this pipeline parallel rank
self.pipeline_parallel()
# Skip the device sync of dynamic rope where it can never rescale
disable_noop_dynamic_rope(self.model.get_decoder(), vllm_config)
# Substitute remaining layers with vLLM's layers as needed
self.recursive_replace()
# Create attention instances for KV cache allocation