[model][refactor] remove cuda hard code in models and layers (#13658)

This commit is contained in:
Mengqing Cao
2025-02-24 06:10:14 -08:00
committed by GitHub
parent 437b76ff59
commit 23eca9cf68
7 changed files with 29 additions and 14 deletions
+3 -2
View File
@@ -30,6 +30,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.sampling_metadata import SamplingMetadata
from vllm.model_executor.utils import set_weight_attrs
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.arctic import ArcticConfig
@@ -138,13 +139,13 @@ class ArcticMoE(nn.Module):
torch.empty(self.num_experts,
2 * self.intermediate_size,
self.hidden_size,
device="cuda",
device=current_platform.device_type,
dtype=self.params_dtype))
self.w2s = nn.Parameter(
torch.empty(self.num_experts,
self.hidden_size,
self.intermediate_size,
device="cuda",
device=current_platform.device_type,
dtype=self.params_dtype))
set_weight_attrs(self.ws, {
"weight_loader": self.weight_loader,
+3 -2
View File
@@ -51,6 +51,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.sampling_metadata import SamplingMetadata
from vllm.model_executor.utils import set_weight_attrs
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from .interfaces import SupportsLoRA, SupportsPP
@@ -98,13 +99,13 @@ class MiniCPMMoE(nn.Module):
torch.empty(self.num_total_experts,
2 * self.intermediate_size,
self.hidden_size,
device="cuda",
device=current_platform.device_type,
dtype=self.params_dtype))
self.w2s = nn.Parameter(
torch.empty(self.num_total_experts,
self.hidden_size,
self.intermediate_size,
device="cuda",
device=current_platform.device_type,
dtype=self.params_dtype))
set_weight_attrs(self.ws, {
+7 -3
View File
@@ -59,6 +59,7 @@ from vllm.multimodal.parse import (DictEmbeddingItems, ImageItem, ImageSize,
from vllm.multimodal.processing import (BaseMultiModalProcessor,
BaseProcessingInfo, PromptReplacement)
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from .idefics2_vision_model import Idefics2VisionTransformer
@@ -1184,7 +1185,8 @@ class MiniCPMV2_0(MiniCPMVBaseModel):
quant_config=quant_config,
prefix=prefix)
return resampler.to(device="cuda", dtype=torch.get_default_dtype())
return resampler.to(device=current_platform.device_type,
dtype=torch.get_default_dtype())
def get_vision_embedding(
self,
@@ -1266,7 +1268,8 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
quant_config=quant_config,
prefix=prefix)
return resampler.to(device="cuda", dtype=torch.get_default_dtype())
return resampler.to(device=current_platform.device_type,
dtype=torch.get_default_dtype())
def get_vision_embedding(
self,
@@ -1360,7 +1363,8 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
quant_config=quant_config,
prefix=prefix)
return resampler.to(device="cuda", dtype=torch.get_default_dtype())
return resampler.to(device=current_platform.device_type,
dtype=torch.get_default_dtype())
def get_vision_embedding(
self,