[CPU][Zen] Route Int8 MoE inference through zentorch on AMD (#44834)
Signed-off-by: R <Ganesh.R@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Cursor
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
c35551f892
commit
8f03625b3d
@@ -288,7 +288,14 @@ class MixtralModel(nn.Module):
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
}
|
||||
},
|
||||
orig_to_new_substr={
|
||||
# W8A8 compressed-tensors checkpoints name experts gate/up/down;
|
||||
# map to the w1/w3/w2 the FusedMoE loader expects (no-op if native).
|
||||
".gate_proj.": ".w1.",
|
||||
".up_proj.": ".w3.",
|
||||
".down_proj.": ".w2.",
|
||||
},
|
||||
)
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
|
||||
Reference in New Issue
Block a user