27 lines
649 B
Python
27 lines
649 B
Python
# Adapted from llama.py
|
|
"""Inference-only Phi3 model code inherit from Llama.py"""
|
|
|
|
from vllm.model_executor.models.llama import LlamaForCausalLM
|
|
|
|
|
|
class Phi3ForCausalLM(LlamaForCausalLM):
|
|
|
|
packed_modules_mapping = {
|
|
"qkv_proj": [
|
|
"qkv_proj",
|
|
],
|
|
"gate_up_proj": [
|
|
"gate_up_proj",
|
|
],
|
|
}
|
|
|
|
# BitandBytes specific attributes
|
|
default_bitsandbytes_target_modules = [
|
|
".gate_up_proj.",
|
|
".down_proj.",
|
|
".qkv_proj.",
|
|
".o_proj.",
|
|
]
|
|
# Initialize an empty dict when there is no stacked parameter mapping.
|
|
bitsandbytes_stacked_params_mapping = {}
|