[Mypy Fix] Mypy fix for "vllm/model_executor/models/[eE][fF]" (#53381)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -112,8 +112,6 @@ SEPARATE_GROUPS = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
EXCLUDE = [
|
EXCLUDE = [
|
||||||
r"vllm/model_executor/models/[eE]",
|
|
||||||
r"vllm/model_executor/models/[fF]",
|
|
||||||
r"vllm/model_executor/models/[gG]",
|
r"vllm/model_executor/models/[gG]",
|
||||||
r"vllm/model_executor/models/[hH]",
|
r"vllm/model_executor/models/[hH]",
|
||||||
r"vllm/model_executor/models/[iI]",
|
r"vllm/model_executor/models/[iI]",
|
||||||
|
|||||||
@@ -106,8 +106,10 @@ def support_torch_compile(
|
|||||||
@overload
|
@overload
|
||||||
def support_torch_compile(
|
def support_torch_compile(
|
||||||
*,
|
*,
|
||||||
dynamic_arg_dims: dict[str, int | list[int] | dict[int, str]] | None,
|
dynamic_arg_dims: dict[str, int | list[int] | dict[int, str]] | None = None,
|
||||||
mark_unbacked_dims: dict[str, int | list[int]] | None,
|
mark_unbacked_dims: dict[str, int | list[int]] | None = None,
|
||||||
|
enable_if: Callable[[VllmConfig], bool] | None = None,
|
||||||
|
is_encoder: bool = False,
|
||||||
) -> Callable[[type[_T]], type[_T]]: ...
|
) -> Callable[[type[_T]], type[_T]]: ...
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -150,6 +150,7 @@ class Eagle2_5_VLForConditionalGeneration(
|
|||||||
config = vllm_config.model_config.hf_config
|
config = vllm_config.model_config.hf_config
|
||||||
quant_config = vllm_config.quant_config
|
quant_config = vllm_config.quant_config
|
||||||
multimodal_config = vllm_config.model_config.multimodal_config
|
multimodal_config = vllm_config.model_config.multimodal_config
|
||||||
|
assert multimodal_config is not None
|
||||||
|
|
||||||
self.config = config
|
self.config = config
|
||||||
self.multimodal_config = multimodal_config
|
self.multimodal_config = multimodal_config
|
||||||
@@ -187,7 +188,7 @@ class Eagle2_5_VLForConditionalGeneration(
|
|||||||
prefix=maybe_prefix(prefix, "language_model"),
|
prefix=maybe_prefix(prefix, "language_model"),
|
||||||
)
|
)
|
||||||
|
|
||||||
self.img_context_token_id = None
|
self.img_context_token_id: int | None = None
|
||||||
|
|
||||||
self.make_empty_intermediate_tensors = (
|
self.make_empty_intermediate_tensors = (
|
||||||
self.language_model.make_empty_intermediate_tensors
|
self.language_model.make_empty_intermediate_tensors
|
||||||
|
|||||||
@@ -127,7 +127,7 @@ class Ernie4_5_MoeMoE(nn.Module):
|
|||||||
self.layer_idx = layer_idx
|
self.layer_idx = layer_idx
|
||||||
self.tp_size = get_tensor_model_parallel_world_size()
|
self.tp_size = get_tensor_model_parallel_world_size()
|
||||||
|
|
||||||
self.moe_num_shared_experts = getattr(config, "moe_num_shared_experts", None)
|
self.moe_num_shared_experts = getattr(config, "moe_num_shared_experts", 0)
|
||||||
self.ep_group = get_ep_group().device_group
|
self.ep_group = get_ep_group().device_group
|
||||||
self.ep_size = self.ep_group.size()
|
self.ep_size = self.ep_group.size()
|
||||||
self.n_routed_experts: int = config.moe_num_experts
|
self.n_routed_experts: int = config.moe_num_experts
|
||||||
@@ -163,6 +163,7 @@ class Ernie4_5_MoeMoE(nn.Module):
|
|||||||
torch.empty(config.moe_num_experts, dtype=torch.float32)
|
torch.empty(config.moe_num_experts, dtype=torch.float32)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
self.shared_experts: Ernie4_5_MoeMLP | None
|
||||||
if self.has_shared_experts:
|
if self.has_shared_experts:
|
||||||
intermediate_size = (
|
intermediate_size = (
|
||||||
config.moe_intermediate_size * config.moe_num_shared_experts
|
config.moe_intermediate_size * config.moe_num_shared_experts
|
||||||
|
|||||||
@@ -1076,7 +1076,7 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
|
|||||||
def _pixel_values_norm(
|
def _pixel_values_norm(
|
||||||
self,
|
self,
|
||||||
pixel_values: torch.Tensor,
|
pixel_values: torch.Tensor,
|
||||||
mm_kwargs: object,
|
mm_kwargs: Mapping[str, object],
|
||||||
) -> torch.Tensor:
|
) -> torch.Tensor:
|
||||||
hf_config = self.info.get_hf_config()
|
hf_config = self.info.get_hf_config()
|
||||||
vision_config = hf_config.vision_config
|
vision_config = hf_config.vision_config
|
||||||
@@ -1125,6 +1125,7 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
|
|||||||
|
|
||||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||||
|
|
||||||
|
mm_data = dict(mm_data)
|
||||||
if "images" not in mm_data:
|
if "images" not in mm_data:
|
||||||
mm_data["images"] = []
|
mm_data["images"] = []
|
||||||
if "videos" not in mm_data:
|
if "videos" not in mm_data:
|
||||||
@@ -1136,15 +1137,15 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
|
|||||||
hf_processor, "supports_video_metadata", False
|
hf_processor, "supports_video_metadata", False
|
||||||
)
|
)
|
||||||
|
|
||||||
if mm_data["videos"] and not supports_video_metadata:
|
videos = mm_data["videos"]
|
||||||
|
assert isinstance(videos, Sequence)
|
||||||
|
if videos and not supports_video_metadata:
|
||||||
# Old HF processor, unwrap tuple to pure frames
|
# Old HF processor, unwrap tuple to pure frames
|
||||||
logger.warning_once(
|
logger.warning_once(
|
||||||
"HF processor doesn't support video metadata. "
|
"HF processor doesn't support video metadata. "
|
||||||
"Timestamps will NOT be rendered. Please upgrade the model."
|
"Timestamps will NOT be rendered. Please upgrade the model."
|
||||||
)
|
)
|
||||||
mm_data["videos"] = [
|
mm_data["videos"] = [v[0] if isinstance(v, tuple) else v for v in videos]
|
||||||
v[0] if isinstance(v, tuple) else v for v in mm_data["videos"]
|
|
||||||
]
|
|
||||||
|
|
||||||
processor_output = self.info.ctx.call_hf_processor(
|
processor_output = self.info.ctx.call_hf_processor(
|
||||||
hf_processor,
|
hf_processor,
|
||||||
@@ -1286,6 +1287,7 @@ class Ernie4_5_VLDummyInputsBuilder(BaseDummyInputsBuilder[Ernie4_5_VLProcessing
|
|||||||
|
|
||||||
image_overrides = mm_options.get("image")
|
image_overrides = mm_options.get("image")
|
||||||
video_overrides = mm_options.get("video")
|
video_overrides = mm_options.get("video")
|
||||||
|
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"image": self._get_dummy_images(
|
"image": self._get_dummy_images(
|
||||||
@@ -1541,10 +1543,14 @@ class Ernie4_5_VLMoeForConditionalGeneration(
|
|||||||
|
|
||||||
offset = mm_feature.mm_position.offset
|
offset = mm_feature.mm_position.offset
|
||||||
if mm_feature.modality == "image":
|
if mm_feature.modality == "image":
|
||||||
t, h, w = mm_feature.data["image_grid_thw"].data.tolist()
|
grid_thw = mm_feature.data["image_grid_thw"].data
|
||||||
|
assert isinstance(grid_thw, torch.Tensor)
|
||||||
|
t, h, w = grid_thw.tolist()
|
||||||
yield offset, t, h // spatial_conv_size, w // spatial_conv_size
|
yield offset, t, h // spatial_conv_size, w // spatial_conv_size
|
||||||
elif mm_feature.modality == "video":
|
elif mm_feature.modality == "video":
|
||||||
t, h, w = mm_feature.data["video_grid_thw"].data.tolist()
|
grid_thw = mm_feature.data["video_grid_thw"].data
|
||||||
|
assert isinstance(grid_thw, torch.Tensor)
|
||||||
|
t, h, w = grid_thw.tolist()
|
||||||
yield (
|
yield (
|
||||||
offset,
|
offset,
|
||||||
t // temporal_conv_size,
|
t // temporal_conv_size,
|
||||||
@@ -1563,12 +1569,11 @@ class Ernie4_5_VLMoeForConditionalGeneration(
|
|||||||
if pixel_values is None:
|
if pixel_values is None:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
if pixel_values is not None:
|
return Ernie4_5_VLImagePixelInputs(
|
||||||
return Ernie4_5_VLImagePixelInputs(
|
type="pixel_values",
|
||||||
type="pixel_values",
|
pixel_values=pixel_values,
|
||||||
pixel_values=pixel_values,
|
image_grid_thw=image_grid_thw,
|
||||||
image_grid_thw=image_grid_thw,
|
)
|
||||||
)
|
|
||||||
|
|
||||||
def _parse_and_validate_video_input(
|
def _parse_and_validate_video_input(
|
||||||
self, **kwargs: object
|
self, **kwargs: object
|
||||||
@@ -1579,12 +1584,11 @@ class Ernie4_5_VLMoeForConditionalGeneration(
|
|||||||
if pixel_values_videos is None:
|
if pixel_values_videos is None:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
if pixel_values_videos is not None:
|
return Ernie4_5_VLVideoPixelInputs(
|
||||||
return Ernie4_5_VLVideoPixelInputs(
|
type="pixel_values_videos",
|
||||||
type="pixel_values_videos",
|
pixel_values_videos=pixel_values_videos,
|
||||||
pixel_values_videos=pixel_values_videos,
|
video_grid_thw=video_grid_thw,
|
||||||
video_grid_thw=video_grid_thw,
|
)
|
||||||
)
|
|
||||||
|
|
||||||
# -- SupportsEncoderCudaGraph protocol methods --
|
# -- SupportsEncoderCudaGraph protocol methods --
|
||||||
#
|
#
|
||||||
@@ -1748,6 +1752,7 @@ class Ernie4_5_VLMoeForConditionalGeneration(
|
|||||||
# the actual batch grid_thw, then scatter the post-merge embeddings.
|
# the actual batch grid_thw, then scatter the post-merge embeddings.
|
||||||
# Ernie only uses the single "default" encoder path.
|
# Ernie only uses the single "default" encoder path.
|
||||||
output = outputs["default"]
|
output = outputs["default"]
|
||||||
|
assert batch_mm_kwargs is not None
|
||||||
grid_thw_cpu = batch_mm_kwargs["image_grid_thw"]
|
grid_thw_cpu = batch_mm_kwargs["image_grid_thw"]
|
||||||
grid_thw = grid_thw_cpu.to(output.device, non_blocking=True)
|
grid_thw = grid_thw_cpu.to(output.device, non_blocking=True)
|
||||||
# The valid token count slices the graph output for the eager
|
# The valid token count slices the graph output for the eager
|
||||||
@@ -1799,7 +1804,9 @@ class Ernie4_5_VLMoeForConditionalGeneration(
|
|||||||
return video_embeds.split(sizes.tolist())
|
return video_embeds.split(sizes.tolist())
|
||||||
|
|
||||||
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
|
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
|
||||||
modalities = {}
|
modalities: dict[
|
||||||
|
str, Ernie4_5_VLImageInputs | Ernie4_5_VLVideoInputs | None
|
||||||
|
] = {}
|
||||||
|
|
||||||
# Preserve the order of modalities if there are multiple of them
|
# Preserve the order of modalities if there are multiple of them
|
||||||
# from the order of kwargs.
|
# from the order of kwargs.
|
||||||
|
|||||||
@@ -232,6 +232,7 @@ class Ernie4_5_VLMoeMoE(nn.Module):
|
|||||||
|
|
||||||
assert text_moe_layer_start_index <= text_moe_layer_end_index
|
assert text_moe_layer_start_index <= text_moe_layer_end_index
|
||||||
|
|
||||||
|
self.shared_experts: Ernie4_5_VLMoeMLP | None
|
||||||
if self.has_shared_experts:
|
if self.has_shared_experts:
|
||||||
intermediate_size = (
|
intermediate_size = (
|
||||||
config.moe_intermediate_size[0] * config.moe_num_shared_experts
|
config.moe_intermediate_size[0] * config.moe_num_shared_experts
|
||||||
@@ -771,9 +772,10 @@ class Ernie4_5_VLMoeForCausalLM(nn.Module, SupportsPP):
|
|||||||
if is_pp_missing_parameter(name, self):
|
if is_pp_missing_parameter(name, self):
|
||||||
continue
|
continue
|
||||||
# Remapping the name of FP8 kv-scale.
|
# Remapping the name of FP8 kv-scale.
|
||||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
|
||||||
if name is None:
|
if remapped_name is None:
|
||||||
continue
|
continue
|
||||||
|
name = remapped_name
|
||||||
|
|
||||||
param = params_dict[name]
|
param = params_dict[name]
|
||||||
|
|
||||||
|
|||||||
@@ -332,6 +332,7 @@ class Exaone4_5_ForConditionalGeneration(Qwen2_5_VLForConditionalGeneration):
|
|||||||
config: Exaone4_5_Config = vllm_config.model_config.hf_config
|
config: Exaone4_5_Config = vllm_config.model_config.hf_config
|
||||||
self.vllm_config = vllm_config
|
self.vllm_config = vllm_config
|
||||||
multimodal_config = vllm_config.model_config.multimodal_config
|
multimodal_config = vllm_config.model_config.multimodal_config
|
||||||
|
assert multimodal_config is not None
|
||||||
|
|
||||||
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
|
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
|
||||||
self.config = config
|
self.config = config
|
||||||
|
|||||||
@@ -48,17 +48,11 @@ class Exaone4_5MultiTokenPredictor(ExaoneMoeMultiTokenPredictor):
|
|||||||
|
|
||||||
model_config = vllm_config.model_config
|
model_config = vllm_config.model_config
|
||||||
quant_config = vllm_config.quant_config
|
quant_config = vllm_config.quant_config
|
||||||
lora_config = vllm_config.lora_config
|
|
||||||
config = model_config.hf_config
|
config = model_config.hf_config
|
||||||
text_config = config.text_config
|
text_config = config.text_config
|
||||||
|
|
||||||
self.config = config
|
self.config = config
|
||||||
lora_vocab = (
|
self.vocab_size = config.vocab_size
|
||||||
(lora_config.lora_extra_vocab_size * (lora_config.max_loras or 1))
|
|
||||||
if lora_config
|
|
||||||
else 0
|
|
||||||
)
|
|
||||||
self.vocab_size = config.vocab_size + lora_vocab
|
|
||||||
self.org_vocab_size = config.vocab_size
|
self.org_vocab_size = config.vocab_size
|
||||||
|
|
||||||
self.mtp_start_layer_idx = text_config.num_hidden_layers
|
self.mtp_start_layer_idx = text_config.num_hidden_layers
|
||||||
@@ -67,7 +61,6 @@ class Exaone4_5MultiTokenPredictor(ExaoneMoeMultiTokenPredictor):
|
|||||||
self.embed_tokens = VocabParallelEmbedding(
|
self.embed_tokens = VocabParallelEmbedding(
|
||||||
self.vocab_size,
|
self.vocab_size,
|
||||||
text_config.hidden_size,
|
text_config.hidden_size,
|
||||||
org_num_embeddings=config.vocab_size,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
self.fc = ColumnParallelLinear(
|
self.fc = ColumnParallelLinear(
|
||||||
|
|||||||
@@ -41,7 +41,6 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
|||||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||||
DEFAULT_VOCAB_PADDING_SIZE,
|
|
||||||
ParallelLMHead,
|
ParallelLMHead,
|
||||||
VocabParallelEmbedding,
|
VocabParallelEmbedding,
|
||||||
)
|
)
|
||||||
@@ -112,6 +111,7 @@ class ExaoneMoe(nn.Module):
|
|||||||
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
||||||
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
||||||
|
|
||||||
|
self.shared_experts: ExaoneMoeGatedMLP | None
|
||||||
if getattr(config, "num_shared_experts", 0) > 0:
|
if getattr(config, "num_shared_experts", 0) > 0:
|
||||||
intermediate_size = config.moe_intermediate_size * config.num_shared_experts
|
intermediate_size = config.moe_intermediate_size * config.num_shared_experts
|
||||||
self.shared_experts = ExaoneMoeGatedMLP(
|
self.shared_experts = ExaoneMoeGatedMLP(
|
||||||
@@ -292,7 +292,7 @@ class ExaoneMoeDecoderLayer(nn.Module):
|
|||||||
config: PretrainedConfig,
|
config: PretrainedConfig,
|
||||||
cache_config: CacheConfig | None = None,
|
cache_config: CacheConfig | None = None,
|
||||||
quant_config: QuantizationConfig | None = None,
|
quant_config: QuantizationConfig | None = None,
|
||||||
is_mtp: bool = None,
|
is_mtp: bool = False,
|
||||||
prefix: str = "",
|
prefix: str = "",
|
||||||
) -> None:
|
) -> None:
|
||||||
super().__init__()
|
super().__init__()
|
||||||
@@ -381,26 +381,19 @@ class ExaoneMoeModel(nn.Module):
|
|||||||
config = vllm_config.model_config.hf_config
|
config = vllm_config.model_config.hf_config
|
||||||
cache_config = vllm_config.cache_config
|
cache_config = vllm_config.cache_config
|
||||||
quant_config = vllm_config.quant_config
|
quant_config = vllm_config.quant_config
|
||||||
lora_config = vllm_config.lora_config
|
|
||||||
self.num_redundant_experts = (
|
self.num_redundant_experts = (
|
||||||
vllm_config.parallel_config.eplb_config.num_redundant_experts
|
vllm_config.parallel_config.eplb_config.num_redundant_experts
|
||||||
)
|
)
|
||||||
|
|
||||||
self.config = config
|
self.config = config
|
||||||
self.quant_config = quant_config
|
self.quant_config = quant_config
|
||||||
lora_vocab = (
|
self.vocab_size = config.vocab_size
|
||||||
(lora_config.lora_extra_vocab_size * (lora_config.max_loras or 1))
|
|
||||||
if lora_config
|
|
||||||
else 0
|
|
||||||
)
|
|
||||||
self.vocab_size = config.vocab_size + lora_vocab
|
|
||||||
if get_pp_group().is_first_rank or (
|
if get_pp_group().is_first_rank or (
|
||||||
config.tie_word_embeddings and get_pp_group().is_last_rank
|
config.tie_word_embeddings and get_pp_group().is_last_rank
|
||||||
):
|
):
|
||||||
self.embed_tokens = VocabParallelEmbedding(
|
self.embed_tokens = VocabParallelEmbedding(
|
||||||
self.vocab_size,
|
self.vocab_size,
|
||||||
config.hidden_size,
|
config.hidden_size,
|
||||||
org_num_embeddings=config.vocab_size,
|
|
||||||
quant_config=quant_config,
|
quant_config=quant_config,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
@@ -497,10 +490,8 @@ class ExaoneMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
|||||||
super().__init__()
|
super().__init__()
|
||||||
config = vllm_config.model_config.hf_config.get_text_config()
|
config = vllm_config.model_config.hf_config.get_text_config()
|
||||||
quant_config = vllm_config.quant_config
|
quant_config = vllm_config.quant_config
|
||||||
lora_config = vllm_config.lora_config
|
|
||||||
|
|
||||||
self.config = config
|
self.config = config
|
||||||
self.lora_config = lora_config
|
|
||||||
self.quant_config = quant_config
|
self.quant_config = quant_config
|
||||||
|
|
||||||
self.model = ExaoneMoeModel(
|
self.model = ExaoneMoeModel(
|
||||||
@@ -509,17 +500,9 @@ class ExaoneMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
|||||||
)
|
)
|
||||||
if get_pp_group().is_last_rank:
|
if get_pp_group().is_last_rank:
|
||||||
self.unpadded_vocab_size = config.vocab_size
|
self.unpadded_vocab_size = config.vocab_size
|
||||||
if lora_config:
|
|
||||||
self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
|
|
||||||
self.lm_head = ParallelLMHead(
|
self.lm_head = ParallelLMHead(
|
||||||
self.unpadded_vocab_size,
|
self.unpadded_vocab_size,
|
||||||
config.hidden_size,
|
config.hidden_size,
|
||||||
org_num_embeddings=config.vocab_size,
|
|
||||||
padding_size=DEFAULT_VOCAB_PADDING_SIZE
|
|
||||||
# We need bigger padding if using lora for kernel
|
|
||||||
# compatibility
|
|
||||||
if not lora_config
|
|
||||||
else lora_config.lora_vocab_padding_size,
|
|
||||||
quant_config=quant_config,
|
quant_config=quant_config,
|
||||||
)
|
)
|
||||||
if config.tie_word_embeddings:
|
if config.tie_word_embeddings:
|
||||||
|
|||||||
@@ -46,16 +46,10 @@ class ExaoneMoeMultiTokenPredictor(nn.Module):
|
|||||||
|
|
||||||
model_config = vllm_config.model_config
|
model_config = vllm_config.model_config
|
||||||
quant_config = vllm_config.quant_config
|
quant_config = vllm_config.quant_config
|
||||||
lora_config = vllm_config.lora_config
|
|
||||||
config = model_config.hf_config
|
config = model_config.hf_config
|
||||||
|
|
||||||
self.config = config
|
self.config = config
|
||||||
lora_vocab = (
|
self.vocab_size = config.vocab_size
|
||||||
(lora_config.lora_extra_vocab_size * (lora_config.max_loras or 1))
|
|
||||||
if lora_config
|
|
||||||
else 0
|
|
||||||
)
|
|
||||||
self.vocab_size = config.vocab_size + lora_vocab
|
|
||||||
self.org_vocab_size = config.vocab_size
|
self.org_vocab_size = config.vocab_size
|
||||||
|
|
||||||
self.mtp_start_layer_idx = config.num_hidden_layers
|
self.mtp_start_layer_idx = config.num_hidden_layers
|
||||||
@@ -64,7 +58,6 @@ class ExaoneMoeMultiTokenPredictor(nn.Module):
|
|||||||
self.embed_tokens = VocabParallelEmbedding(
|
self.embed_tokens = VocabParallelEmbedding(
|
||||||
self.vocab_size,
|
self.vocab_size,
|
||||||
config.hidden_size,
|
config.hidden_size,
|
||||||
org_num_embeddings=config.vocab_size,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
self.fc = ColumnParallelLinear(
|
self.fc = ColumnParallelLinear(
|
||||||
|
|||||||
@@ -231,7 +231,7 @@ class CacheOnlyAttentionLayer(nn.Module, AttentionLayerBase):
|
|||||||
head_size: int,
|
head_size: int,
|
||||||
cache_config: CacheConfig | None = None,
|
cache_config: CacheConfig | None = None,
|
||||||
prefix: str = "",
|
prefix: str = "",
|
||||||
attn_type: str = AttentionType.DECODER,
|
attn_type: AttentionType = AttentionType.DECODER,
|
||||||
):
|
):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
|
|
||||||
@@ -329,7 +329,9 @@ class ExtractHiddenStatesModel(nn.Module):
|
|||||||
super().__init__()
|
super().__init__()
|
||||||
|
|
||||||
self.vllm_config = vllm_config
|
self.vllm_config = vllm_config
|
||||||
self.hf_config = vllm_config.speculative_config.draft_model_config.hf_config
|
speculative_config = vllm_config.speculative_config
|
||||||
|
assert speculative_config is not None
|
||||||
|
self.hf_config = speculative_config.draft_model_config.hf_config
|
||||||
self.hidden_size = vllm_config.model_config.get_hidden_size()
|
self.hidden_size = vllm_config.model_config.get_hidden_size()
|
||||||
self.target_num_hidden_layers = (
|
self.target_num_hidden_layers = (
|
||||||
vllm_config.model_config.get_total_num_hidden_layers()
|
vllm_config.model_config.get_total_num_hidden_layers()
|
||||||
|
|||||||
@@ -413,6 +413,7 @@ class FalconModel(nn.Module):
|
|||||||
else:
|
else:
|
||||||
hidden_states = self.embed_input_ids(input_ids)
|
hidden_states = self.embed_input_ids(input_ids)
|
||||||
else:
|
else:
|
||||||
|
assert intermediate_tensors is not None
|
||||||
hidden_states = intermediate_tensors["hidden_states"]
|
hidden_states = intermediate_tensors["hidden_states"]
|
||||||
for layer in islice(self.h, self.start_layer, self.end_layer):
|
for layer in islice(self.h, self.start_layer, self.end_layer):
|
||||||
hidden_states = layer(positions, hidden_states)
|
hidden_states = layer(positions, hidden_states)
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ from transformers import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
|
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
|
||||||
from vllm.config.speech_to_text import SpeechToTextParams
|
from vllm.config.speech_to_text import SpeechToTextParams
|
||||||
from vllm.inputs import MultiModalDataDict, PromptType
|
from vllm.inputs import MultiModalDataDict, PromptType
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
@@ -216,6 +216,7 @@ class FireRedASR2DummyInputsBuilder(BaseDummyInputsBuilder[FireRedASR2Processing
|
|||||||
num_audios = mm_counts.get("audio", 0)
|
num_audios = mm_counts.get("audio", 0)
|
||||||
|
|
||||||
audio_overrides = mm_options.get("audio")
|
audio_overrides = mm_options.get("audio")
|
||||||
|
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
|
||||||
|
|
||||||
ret = {
|
ret = {
|
||||||
"audio": self._get_dummy_audios(
|
"audio": self._get_dummy_audios(
|
||||||
|
|||||||
@@ -18,14 +18,13 @@ from __future__ import annotations
|
|||||||
from collections.abc import Iterable, Mapping, Sequence
|
from collections.abc import Iterable, Mapping, Sequence
|
||||||
from typing import Annotated, Literal
|
from typing import Annotated, Literal
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
import torch
|
import torch
|
||||||
from torch import nn
|
from torch import nn
|
||||||
from transformers import BatchFeature
|
from transformers import BatchFeature
|
||||||
|
|
||||||
from vllm.config import ModelConfig, VllmConfig
|
from vllm.config import ModelConfig, VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
|
||||||
from vllm.config.speech_to_text import SpeechToTextConfig
|
from vllm.config.speech_to_text import SpeechToTextConfig, SpeechToTextParams
|
||||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||||
from vllm.inputs import MultiModalDataDict, PromptType
|
from vllm.inputs import MultiModalDataDict, PromptType
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
@@ -431,6 +430,7 @@ class FireRedLIDDummyInputsBuilder(BaseDummyInputsBuilder[FireRedLIDProcessingIn
|
|||||||
audio_len = feature_extractor.chunk_length * sampling_rate
|
audio_len = feature_extractor.chunk_length * sampling_rate
|
||||||
num_audios = mm_counts.get("audio", 0)
|
num_audios = mm_counts.get("audio", 0)
|
||||||
audio_overrides = mm_options.get("audio")
|
audio_overrides = mm_options.get("audio")
|
||||||
|
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
|
||||||
return {
|
return {
|
||||||
"audio": self._get_dummy_audios(
|
"audio": self._get_dummy_audios(
|
||||||
length=audio_len,
|
length=audio_len,
|
||||||
@@ -747,13 +747,7 @@ class FireRedLIDForConditionalGeneration(
|
|||||||
@classmethod
|
@classmethod
|
||||||
def get_generation_prompt(
|
def get_generation_prompt(
|
||||||
cls,
|
cls,
|
||||||
audio: np.ndarray,
|
stt_params: SpeechToTextParams,
|
||||||
stt_config: SpeechToTextConfig,
|
|
||||||
model_config: ModelConfig,
|
|
||||||
language: str | None,
|
|
||||||
task_type: Literal["transcribe", "translate"],
|
|
||||||
request_prompt: str,
|
|
||||||
to_language: str | None,
|
|
||||||
) -> PromptType:
|
) -> PromptType:
|
||||||
"""Build the prompt for the FireRedLID encoder-decoder model.
|
"""Build the prompt for the FireRedLID encoder-decoder model.
|
||||||
|
|
||||||
@@ -764,7 +758,10 @@ class FireRedLIDForConditionalGeneration(
|
|||||||
"encoder_prompt": {
|
"encoder_prompt": {
|
||||||
"prompt": "",
|
"prompt": "",
|
||||||
"multi_modal_data": {
|
"multi_modal_data": {
|
||||||
"audio": (audio, int(stt_config.sample_rate)),
|
"audio": (
|
||||||
|
stt_params.audio,
|
||||||
|
int(stt_params.stt_config.sample_rate),
|
||||||
|
),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"decoder_prompt": {
|
"decoder_prompt": {
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ from transformers import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
|
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
|
||||||
from vllm.config.speech_to_text import SpeechToTextParams
|
from vllm.config.speech_to_text import SpeechToTextParams
|
||||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||||
from vllm.inputs import MultiModalDataDict, PromptType
|
from vllm.inputs import MultiModalDataDict, PromptType
|
||||||
@@ -261,8 +261,8 @@ class SinusoidalPositionEncoder(torch.nn.Module):
|
|||||||
|
|
||||||
def encode(
|
def encode(
|
||||||
self,
|
self,
|
||||||
positions: torch.Tensor = None,
|
positions: torch.Tensor,
|
||||||
depth: int = None,
|
depth: int,
|
||||||
dtype: torch.dtype = torch.float32,
|
dtype: torch.dtype = torch.float32,
|
||||||
):
|
):
|
||||||
batch_size = positions.size(0)
|
batch_size = positions.size(0)
|
||||||
@@ -573,8 +573,8 @@ class Transformer(nn.Module):
|
|||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
def forward(self, hidden_states: torch.Tensor, ilens: int = 0):
|
def forward(self, hidden_states: torch.Tensor, ilens: torch.Tensor):
|
||||||
max_len = max(ilens)
|
max_len = ilens.max()
|
||||||
hidden_states = hidden_states[:, :max_len, :]
|
hidden_states = hidden_states[:, :max_len, :]
|
||||||
batch_size, seq_len, dim = hidden_states.size()
|
batch_size, seq_len, dim = hidden_states.size()
|
||||||
chunk_num = (seq_len - 1) // self.k + 1
|
chunk_num = (seq_len - 1) // self.k + 1
|
||||||
@@ -682,6 +682,8 @@ class FunASRModel(nn.Module):
|
|||||||
self.feat_permute = False
|
self.feat_permute = False
|
||||||
|
|
||||||
if self.feat_permute:
|
if self.feat_permute:
|
||||||
|
if not isinstance(speech, torch.Tensor):
|
||||||
|
raise NotImplementedError("List audio inputs cannot be permuted")
|
||||||
encoder_out, encoder_out_lens = self.encoder.audio_encoder(
|
encoder_out, encoder_out_lens = self.encoder.audio_encoder(
|
||||||
speech.permute(0, 2, 1), speech_lengths
|
speech.permute(0, 2, 1), speech_lengths
|
||||||
)
|
)
|
||||||
@@ -739,6 +741,7 @@ class FunASRDummyInputsBuilder(BaseDummyInputsBuilder[FunASRProcessingInfo]):
|
|||||||
num_audios = mm_counts.get("audio", 0)
|
num_audios = mm_counts.get("audio", 0)
|
||||||
|
|
||||||
audio_overrides = mm_options.get("audio")
|
audio_overrides = mm_options.get("audio")
|
||||||
|
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"audio": self._get_dummy_audios(
|
"audio": self._get_dummy_audios(
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ from transformers.feature_extraction_utils import BatchFeature
|
|||||||
from transformers.modeling_outputs import BaseModelOutput
|
from transformers.modeling_outputs import BaseModelOutput
|
||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
|
||||||
from vllm.inputs import MultiModalDataDict
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention
|
from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention
|
||||||
from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLinear
|
from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLinear
|
||||||
@@ -600,6 +600,7 @@ class FunAudioChatDummyInputsBuilder(
|
|||||||
num_audios = int(mm_counts.get("audio", 0))
|
num_audios = int(mm_counts.get("audio", 0))
|
||||||
|
|
||||||
audio_overrides = mm_options.get("audio")
|
audio_overrides = mm_options.get("audio")
|
||||||
|
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
|
||||||
return {
|
return {
|
||||||
"audio": self._get_dummy_audios(
|
"audio": self._get_dummy_audios(
|
||||||
length=audio_len,
|
length=audio_len,
|
||||||
@@ -635,6 +636,7 @@ class FunAudioChatMultiModalProcessor(
|
|||||||
processed_data = BatchFeature({"input_ids": input_ids})
|
processed_data = BatchFeature({"input_ids": input_ids})
|
||||||
processed_data.update(passthrough_data)
|
processed_data.update(passthrough_data)
|
||||||
return processed_data
|
return processed_data
|
||||||
|
assert isinstance(audios, Sequence)
|
||||||
|
|
||||||
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
|
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
|
||||||
sr = int(feature_extractor.sampling_rate)
|
sr = int(feature_extractor.sampling_rate)
|
||||||
|
|||||||
Reference in New Issue
Block a user