[Model] Add BNB quantization support for Mllama (#9720)
This commit is contained in:
@@ -325,7 +325,10 @@ class MllamaPrecomputedPositionEmbedding(nn.Module):
|
||||
# TODO: support other attention backends for attention in vision model
|
||||
class MllamaVisionSdpaAttention(nn.Module):
|
||||
|
||||
def __init__(self, config: config_mllama.MllamaVisionConfig):
|
||||
def __init__(self,
|
||||
config: config_mllama.MllamaVisionConfig,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
model_parallel_size = get_tensor_model_parallel_world_size()
|
||||
@@ -341,12 +344,16 @@ class MllamaVisionSdpaAttention(nn.Module):
|
||||
self.head_dim,
|
||||
self.num_heads,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.qkv_proj",
|
||||
)
|
||||
self.o_proj = RowParallelLinear(
|
||||
self.num_heads * self.head_dim,
|
||||
self.embed_dim,
|
||||
bias=False,
|
||||
input_is_parallel=True,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -393,7 +400,8 @@ class MllamaVisionEncoderLayer(nn.Module):
|
||||
self.is_gated = is_gated
|
||||
self.intermediate_size = config.intermediate_size
|
||||
|
||||
self.self_attn = MllamaVisionSdpaAttention(config)
|
||||
self.self_attn = MllamaVisionSdpaAttention(
|
||||
config, quant_config=quant_config, prefix=f"{prefix}.self_attn")
|
||||
self.mlp = CLIPMLP(config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mlp")
|
||||
@@ -1002,6 +1010,7 @@ class MllamaForCausalLM(nn.Module):
|
||||
org_num_embeddings=config.vocab_size,
|
||||
padding_size=DEFAULT_VOCAB_PADDING_SIZE,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.lm_head",
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -1037,6 +1046,26 @@ class MllamaForCausalLM(nn.Module):
|
||||
@INPUT_REGISTRY.register_dummy_encoder_data(dummy_encoder_data_for_mllama)
|
||||
@INPUT_REGISTRY.register_input_processor(input_processor_for_mllama)
|
||||
class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
|
||||
# BitandBytes specific attributes
|
||||
default_bitsandbytes_target_modules = [
|
||||
".gate_proj.",
|
||||
".down_proj.",
|
||||
".up_proj.",
|
||||
".q_proj.",
|
||||
".k_proj.",
|
||||
".v_proj.",
|
||||
".o_proj.",
|
||||
]
|
||||
# in TP, these weights are partitioned along the column dimension (dim=-1)
|
||||
column_parallel_weights_modules = [".down_proj.", ".o_proj."]
|
||||
bitsandbytes_stacked_params_mapping = {
|
||||
# shard_name, weight_name, index
|
||||
"q_proj": ("qkv_proj", 0),
|
||||
"k_proj": ("qkv_proj", 1),
|
||||
"v_proj": ("qkv_proj", 2),
|
||||
"gate_proj": ("gate_up_proj", 0),
|
||||
"up_proj": ("gate_up_proj", 1),
|
||||
}
|
||||
|
||||
def __init__(self,
|
||||
config: config_mllama.MllamaConfig,
|
||||
@@ -1061,10 +1090,13 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
|
||||
quant_config=quant_config,
|
||||
prefix="language_model",
|
||||
)
|
||||
self.multi_modal_projector = nn.Linear(
|
||||
self.multi_modal_projector = ColumnParallelLinear(
|
||||
config.vision_config.vision_output_dim,
|
||||
config.text_config.hidden_size,
|
||||
bias=True,
|
||||
quant_config=quant_config,
|
||||
gather_output=True,
|
||||
prefix="multi_modal_projector",
|
||||
)
|
||||
self.logits_processor = LogitsProcessor(config.output_hidden_states,
|
||||
config.text_config.vocab_size)
|
||||
@@ -1128,7 +1160,7 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
|
||||
raise ValueError("No images provided.")
|
||||
max_num_tiles = max(
|
||||
max([len(x) for x in y[0]]) for y in pixel_values)
|
||||
device = self.multi_modal_projector.weight.device
|
||||
device = next(self.multi_modal_projector.parameters()).device
|
||||
bsz = len(pixel_values)
|
||||
out_num_tiles = []
|
||||
out_images = torch.zeros(
|
||||
@@ -1204,7 +1236,7 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
|
||||
cross_attention_states = self.vision_model(pixel_values,
|
||||
aspect_ratio_ids,
|
||||
aspect_ratio_mask)
|
||||
cross_attention_states = self.multi_modal_projector(
|
||||
cross_attention_states, _ = self.multi_modal_projector(
|
||||
cross_attention_states)
|
||||
|
||||
bsz, _, _, _, image_token_dim = tuple(cross_attention_states.shape)
|
||||
|
||||
Reference in New Issue
Block a user