[Bugfix][Transformers backend] Fix CUDA graph capture for dynamic-rope models (#58064)

Signed-off-by: YongfengX <erkang040804@gmail.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
Frank
2026-10-14 05:56:54 -07:00
committed by GitHub
co-authored by Claude Opus 5 Harry Mellor Kevin H. Luu
parent 2631afd3fe
commit 3723a1ea96
4 changed files with 132 additions and 4 deletions
@@ -357,13 +357,51 @@ def get_feature_request_tip(
return f"{prefix}{tip}"
def dynamic_rope_update_is_noop(vllm_config: "VllmConfig") -> bool:
"""Whether `max_model_len` keeps Transformers' dynamic rope from ever rescaling.
`@dynamic_rope_update` only recomputes `inv_freq` once positions grow past
`max_position_embeddings`, which vLLM never feeds beyond `max_model_len`.
"""
text_config = vllm_config.model_config.hf_config.get_text_config()
max_position_embeddings = getattr(text_config, "max_position_embeddings", None)
if max_position_embeddings is None:
return False
return vllm_config.model_config.max_model_len <= max_position_embeddings
def can_enable_torch_compile(vllm_config: "VllmConfig") -> bool:
"""Callable to be passed to `@support_torch_compile`'s `enable_if` argument.
Defaults to `True` but is disabled in the following situations:
- The model uses dynamic rope scaling.
- The model uses dynamic rope scaling that can still rescale within
`max_model_len` (otherwise `disable_noop_dynamic_rope` removes it).
"""
text_config = vllm_config.model_config.hf_config.get_text_config()
# Dynamic rope scaling is not compatible with torch.compile
return all(rp["rope_type"] != "dynamic" for rp in iter_rope_parameters(text_config))
rope_types = {rp["rope_type"] for rp in iter_rope_parameters(text_config)}
if "dynamic" not in rope_types:
return True
# Layer types using longrope still sync, even once dynamic rope is disabled
return "longrope" not in rope_types and dynamic_rope_update_is_noop(vllm_config)
def disable_noop_dynamic_rope(model: nn.Module, vllm_config: "VllmConfig") -> None:
"""Skip `@dynamic_rope_update` in rotary modules where it can never rescale.
The update syncs on `torch.max(position_ids)` every forward, which breaks
torch.compile and CUDA graph capture. Relabelling the module's `rope_type` as
"default" skips it while keeping the `inv_freq` computed at init.
"""
if not dynamic_rope_update_is_noop(vllm_config):
return
for module in model.modules():
rope_type = getattr(module, "rope_type", None)
if not hasattr(module, "max_seq_len_cached") or rope_type is None:
continue
if isinstance(rope_type, dict):
module.rope_type = {
k: "default" if v == "dynamic" else v for k, v in rope_type.items()
}
elif rope_type == "dynamic":
module.rope_type = "default"