[Bugfix][Transformers backend] Fix CUDA graph capture for dynamic-rope models (#58064)
Signed-off-by: YongfengX <erkang040804@gmail.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 5
Harry Mellor
Kevin H. Luu
parent
2631afd3fe
commit
3723a1ea96
@@ -66,6 +66,7 @@ from vllm.model_executor.models.transformers.fusers.attention import (
|
||||
from vllm.model_executor.models.transformers.utils import (
|
||||
attrsetter,
|
||||
can_enable_torch_compile,
|
||||
disable_noop_dynamic_rope,
|
||||
get_feature_request_tip,
|
||||
init_on_device_without_buffers,
|
||||
log_replacement,
|
||||
@@ -182,6 +183,8 @@ class Base(
|
||||
self._create_hf_to_vllm_mapper()
|
||||
# Remove layers not on this pipeline parallel rank
|
||||
self.pipeline_parallel()
|
||||
# Skip the device sync of dynamic rope where it can never rescale
|
||||
disable_noop_dynamic_rope(self.model.get_decoder(), vllm_config)
|
||||
# Substitute remaining layers with vLLM's layers as needed
|
||||
self.recursive_replace()
|
||||
# Create attention instances for KV cache allocation
|
||||
|
||||
@@ -357,13 +357,51 @@ def get_feature_request_tip(
|
||||
return f"{prefix}{tip}"
|
||||
|
||||
|
||||
def dynamic_rope_update_is_noop(vllm_config: "VllmConfig") -> bool:
|
||||
"""Whether `max_model_len` keeps Transformers' dynamic rope from ever rescaling.
|
||||
|
||||
`@dynamic_rope_update` only recomputes `inv_freq` once positions grow past
|
||||
`max_position_embeddings`, which vLLM never feeds beyond `max_model_len`.
|
||||
"""
|
||||
text_config = vllm_config.model_config.hf_config.get_text_config()
|
||||
max_position_embeddings = getattr(text_config, "max_position_embeddings", None)
|
||||
if max_position_embeddings is None:
|
||||
return False
|
||||
return vllm_config.model_config.max_model_len <= max_position_embeddings
|
||||
|
||||
|
||||
def can_enable_torch_compile(vllm_config: "VllmConfig") -> bool:
|
||||
"""Callable to be passed to `@support_torch_compile`'s `enable_if` argument.
|
||||
|
||||
Defaults to `True` but is disabled in the following situations:
|
||||
|
||||
- The model uses dynamic rope scaling.
|
||||
- The model uses dynamic rope scaling that can still rescale within
|
||||
`max_model_len` (otherwise `disable_noop_dynamic_rope` removes it).
|
||||
"""
|
||||
text_config = vllm_config.model_config.hf_config.get_text_config()
|
||||
# Dynamic rope scaling is not compatible with torch.compile
|
||||
return all(rp["rope_type"] != "dynamic" for rp in iter_rope_parameters(text_config))
|
||||
rope_types = {rp["rope_type"] for rp in iter_rope_parameters(text_config)}
|
||||
if "dynamic" not in rope_types:
|
||||
return True
|
||||
# Layer types using longrope still sync, even once dynamic rope is disabled
|
||||
return "longrope" not in rope_types and dynamic_rope_update_is_noop(vllm_config)
|
||||
|
||||
|
||||
def disable_noop_dynamic_rope(model: nn.Module, vllm_config: "VllmConfig") -> None:
|
||||
"""Skip `@dynamic_rope_update` in rotary modules where it can never rescale.
|
||||
|
||||
The update syncs on `torch.max(position_ids)` every forward, which breaks
|
||||
torch.compile and CUDA graph capture. Relabelling the module's `rope_type` as
|
||||
"default" skips it while keeping the `inv_freq` computed at init.
|
||||
"""
|
||||
if not dynamic_rope_update_is_noop(vllm_config):
|
||||
return
|
||||
for module in model.modules():
|
||||
rope_type = getattr(module, "rope_type", None)
|
||||
if not hasattr(module, "max_seq_len_cached") or rope_type is None:
|
||||
continue
|
||||
if isinstance(rope_type, dict):
|
||||
module.rope_type = {
|
||||
k: "default" if v == "dynamic" else v for k, v in rope_type.items()
|
||||
}
|
||||
elif rope_type == "dynamic":
|
||||
module.rope_type = "default"
|
||||
|
||||
Reference in New Issue
Block a user