[Hy4] support Hy4-preview model (#54160)
Signed-off-by: chengvjiang <chengvjiang@tencent.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: chengvjiang <chengvjiang@tencent.com> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: russellfeng <russellfeng@tencent.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
co-authored by
chengvjiang
Jee Jee Li
Bugen Zhao
russellfeng
Jee Jee Li
parent
6c18a54648
commit
b2f685834a
@@ -15,6 +15,7 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.linear import UnquantizedLinearMethod
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
UnquantizedEmbeddingMethod,
|
||||
VocabParallelEmbedding,
|
||||
@@ -144,7 +145,12 @@ class LogitsProcessor(PluggableLayer):
|
||||
lm_head, hidden_states, bias=embedding_bias
|
||||
)
|
||||
|
||||
if not isinstance(lm_head.quant_method, UnquantizedEmbeddingMethod):
|
||||
# A quant config that excludes lm_head hands out UnquantizedLinearMethod
|
||||
# rather than UnquantizedEmbeddingMethod, so accept both: either way the
|
||||
# weight is plain and `lm_head.weight` can be cast directly.
|
||||
if not isinstance(
|
||||
lm_head.quant_method, (UnquantizedEmbeddingMethod, UnquantizedLinearMethod)
|
||||
):
|
||||
raise ValueError(
|
||||
"A head_dtype different from the model dtype is only "
|
||||
"supported for an unquantized lm_head."
|
||||
|
||||
Reference in New Issue
Block a user