[Bugfix][Transformers backend] Fix CUDA graph capture for dynamic-rope models (#58064)
Signed-off-by: YongfengX <erkang040804@gmail.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 5
Harry Mellor
Kevin H. Luu
parent
2631afd3fe
commit
3723a1ea96
@@ -66,6 +66,7 @@ from vllm.model_executor.models.transformers.fusers.attention import (
|
||||
from vllm.model_executor.models.transformers.utils import (
|
||||
attrsetter,
|
||||
can_enable_torch_compile,
|
||||
disable_noop_dynamic_rope,
|
||||
get_feature_request_tip,
|
||||
init_on_device_without_buffers,
|
||||
log_replacement,
|
||||
@@ -182,6 +183,8 @@ class Base(
|
||||
self._create_hf_to_vllm_mapper()
|
||||
# Remove layers not on this pipeline parallel rank
|
||||
self.pipeline_parallel()
|
||||
# Skip the device sync of dynamic rope where it can never rescale
|
||||
disable_noop_dynamic_rope(self.model.get_decoder(), vllm_config)
|
||||
# Substitute remaining layers with vLLM's layers as needed
|
||||
self.recursive_replace()
|
||||
# Create attention instances for KV cache allocation
|
||||
|
||||
Reference in New Issue
Block a user