[Bugfix] Fix weight loading for Chameleon when TP>1 (#7410)
This commit is contained in:
+18
-8
@@ -4,7 +4,8 @@ import os
|
||||
import sys
|
||||
from collections import UserList
|
||||
from enum import Enum
|
||||
from typing import Any, Dict, List, Optional, Tuple, TypedDict, TypeVar, Union
|
||||
from typing import (Any, Callable, Dict, List, Optional, Tuple, TypedDict,
|
||||
TypeVar, Union)
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -27,7 +28,7 @@ from vllm.logger import init_logger
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.sequence import SampleLogprobs
|
||||
from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, cuda_device_count_stateless,
|
||||
is_cpu)
|
||||
identity, is_cpu)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -197,6 +198,8 @@ class HfRunner:
|
||||
is_embedding_model: bool = False,
|
||||
is_vision_model: bool = False,
|
||||
is_encoder_decoder_model: bool = False,
|
||||
postprocess_inputs: Callable[[BatchEncoding],
|
||||
BatchEncoding] = identity,
|
||||
) -> None:
|
||||
torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
|
||||
|
||||
@@ -242,12 +245,14 @@ class HfRunner:
|
||||
torch_dtype=torch_dtype,
|
||||
trust_remote_code=True,
|
||||
)
|
||||
except Exception:
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Unable to auto-load processor from HuggingFace for "
|
||||
"model %s. Using tokenizer instead.", model_name)
|
||||
"Unable to auto-load HuggingFace processor for model (%s). "
|
||||
"Using tokenizer instead. Reason: %s", model_name, exc)
|
||||
self.processor = self.tokenizer
|
||||
|
||||
self.postprocess_inputs = postprocess_inputs
|
||||
|
||||
def generate(
|
||||
self,
|
||||
prompts: List[str],
|
||||
@@ -267,6 +272,7 @@ class HfRunner:
|
||||
processor_kwargs["images"] = images[i]
|
||||
|
||||
inputs = self.processor(**processor_kwargs)
|
||||
inputs = self.postprocess_inputs(inputs)
|
||||
|
||||
output_ids = self.model.generate(
|
||||
**self.wrap_device(inputs),
|
||||
@@ -336,6 +342,7 @@ class HfRunner:
|
||||
processor_kwargs["images"] = images[i]
|
||||
|
||||
inputs = self.processor(**processor_kwargs)
|
||||
inputs = self.postprocess_inputs(inputs)
|
||||
|
||||
output = self.model.generate(
|
||||
**self.wrap_device(inputs),
|
||||
@@ -420,6 +427,7 @@ class HfRunner:
|
||||
processor_kwargs["images"] = images[i]
|
||||
|
||||
inputs = self.processor(**processor_kwargs)
|
||||
inputs = self.postprocess_inputs(inputs)
|
||||
|
||||
output = self.model.generate(
|
||||
**self.wrap_device(inputs),
|
||||
@@ -552,7 +560,8 @@ class VllmRunner:
|
||||
self,
|
||||
prompts: List[str],
|
||||
sampling_params: SamplingParams,
|
||||
images: Optional[List[Image.Image]] = None,
|
||||
images: Optional[Union[List[Image.Image],
|
||||
List[List[Image.Image]]]] = None,
|
||||
) -> List[Tuple[List[List[int]], List[str]]]:
|
||||
if images is not None:
|
||||
assert len(prompts) == len(images)
|
||||
@@ -587,7 +596,7 @@ class VllmRunner:
|
||||
for req_output in req_outputs:
|
||||
for sample in req_output.outputs:
|
||||
output_str = sample.text
|
||||
output_ids = sample.token_ids
|
||||
output_ids = list(sample.token_ids)
|
||||
output_logprobs = sample.logprobs
|
||||
outputs.append((output_ids, output_str, output_logprobs))
|
||||
return outputs
|
||||
@@ -596,7 +605,8 @@ class VllmRunner:
|
||||
self,
|
||||
prompts: List[str],
|
||||
sampling_params: SamplingParams,
|
||||
images: Optional[List[Image.Image]] = None,
|
||||
images: Optional[Union[List[Image.Image],
|
||||
List[List[Image.Image]]]] = None,
|
||||
) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
|
||||
assert sampling_params.logprobs is not None
|
||||
|
||||
|
||||
Reference in New Issue
Block a user