[xpu]feat: support multi-lora on xpu (#20616)
Signed-off-by: yan <yan.ma@intel.com>
This commit is contained in:
@@ -27,6 +27,7 @@ from vllm.config import (ModelConfig, ParallelConfig, VllmConfig,
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils import FlexibleArgumentParser, PlaceholderModule
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -513,7 +514,9 @@ def deserialize_tensorizer_model(model: nn.Module,
|
||||
**tensorizer_args.stream_kwargs) as stream, TensorDeserializer(
|
||||
stream,
|
||||
dtype=tensorizer_config.dtype,
|
||||
device=torch.device("cuda", torch.cuda.current_device()),
|
||||
device=f'xpu:{torch.xpu.current_device()}'
|
||||
if current_platform.is_xpu() else
|
||||
f'cuda:{torch.cuda.current_device()}',
|
||||
**tensorizer_args.deserialization_kwargs) as deserializer:
|
||||
deserializer.load_into_module(model)
|
||||
end = time.perf_counter()
|
||||
|
||||
Reference in New Issue
Block a user