[Neuron] Support inference with transformers-neuronx (#2569)

This commit is contained in:
Liangfu Chen
2024-02-28 09:34:34 -08:00
committed by GitHub
parent e46fa5d52e
commit 3b7178cfa4
18 changed files with 516 additions and 42 deletions
+5 -5
View File
@@ -1,11 +1,11 @@
"""Utilities for selecting and loading models."""
import contextlib
from typing import Optional, Type
from typing import Type
import torch
import torch.nn as nn
from vllm.config import DeviceConfig, ModelConfig, LoRAConfig
from vllm.config import DeviceConfig, ModelConfig
from vllm.model_executor.models import ModelRegistry
from vllm.model_executor.weight_utils import (get_quant_config,
initialize_dummy_weights)
@@ -37,9 +37,9 @@ def _get_model_architecture(model_config: ModelConfig) -> Type[nn.Module]:
f"Supported architectures: {ModelRegistry.get_supported_archs()}")
def get_model(model_config: ModelConfig,
device_config: DeviceConfig,
lora_config: Optional[LoRAConfig] = None) -> nn.Module:
def get_model(model_config: ModelConfig, device_config: DeviceConfig,
**kwargs) -> nn.Module:
lora_config = kwargs.get("lora_config", None)
model_class = _get_model_architecture(model_config)
# Get the (maybe quantized) linear method.