[Misc] Standardize RoPE handling for Qwen2-VL (#9250)

This commit is contained in:
Cyrus Leung
2024-10-16 13:56:17 +08:00
committed by GitHub
parent ed920135c8
commit 7e7eae338d
16 changed files with 102 additions and 200 deletions
+2 -4
View File
@@ -19,6 +19,7 @@ from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
MultiModalInputs)
from vllm.sequence import (IntermediateTensors, SequenceData,
SequenceGroupMetadata)
from vllm.transformers_utils.config import uses_mrope
from vllm.utils import make_tensor_with_pad
from vllm.worker.model_runner_base import (
ModelRunnerBase, ModelRunnerInputBase, ModelRunnerInputBuilderBase,
@@ -439,10 +440,7 @@ class CPUModelRunner(ModelRunnerBase[ModelInputForCPU]):
def model_is_mrope(self) -> bool:
"""Detect if the model has "mrope" rope_scaling type.
mrope requires keep "rope_deltas" between prompt and decoding phases."""
rope_scaling = getattr(self.model_config.hf_config, "rope_scaling", {})
if rope_scaling is None:
return False
return rope_scaling.get("type", None) == "mrope"
return uses_mrope(self.model_config.hf_config)
def load_model(self) -> None:
self.model = get_model(model_config=self.model_config,
+2 -4
View File
@@ -47,6 +47,7 @@ from vllm.prompt_adapter.worker_manager import (
LRUCacheWorkerPromptAdapterManager)
from vllm.sampling_params import SamplingParams
from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
from vllm.transformers_utils.config import uses_mrope
from vllm.utils import (DeviceMemoryProfiler, PyObjectCache, async_tensor_h2d,
flatten_2d_lists, is_hip, is_pin_memory_available,
supports_dynamo)
@@ -1379,10 +1380,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
def model_is_mrope(self) -> bool:
"""Detect if the model has "mrope" rope_scaling type.
mrope requires keep "rope_deltas" between prompt and decoding phases."""
rope_scaling = getattr(self.model_config.hf_config, "rope_scaling", {})
if rope_scaling is None:
return False
return rope_scaling.get("type", None) == "mrope"
return uses_mrope(self.model_config.hf_config)
@torch.inference_mode()
def capture_model(self, kv_caches: List[List[torch.Tensor]]) -> None: