[Model] Support Qwen3.8-Flash-Next (#53896)
Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com> Signed-off-by: andyluo7 <andy.luo@amd.com> Signed-off-by: lishunyang12 <lishunyang03@gmail.com> Signed-off-by: Harry Huang <huanghaoyan.hhy@alibaba-inc.com> Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: KungYork <firelikest@163.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: andyluo7 <43718156+andyluo7@users.noreply.github.com> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: andyluo7 <andy.luo@amd.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: lishunyang12 <lishunyang03@gmail.com> Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
zjy0516
KungYork
Thien Tran
Yifan Qiao
andyluo7
Isotr0py
andyluo7
OpenAI Codex
lishunyang12
Vadim Gimpelson
Isotr0py
Lucas Wilkinson
Claude Opus 5
OpenAI Codex
parent
687db59744
commit
e126687a9a
@@ -233,6 +233,7 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
quant_config: quant config for the layer
|
||||
prefix: full name of the layer in the state dict
|
||||
disable_tp: If true, tensor parallelism will be disabled for this layer.
|
||||
quant_method: Preselected quantization method for model-specific layers.
|
||||
""" # noqa: E501
|
||||
|
||||
# --8<-- [end:vocab_parallel_embedding]
|
||||
@@ -248,6 +249,7 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
prefix: str = "",
|
||||
*,
|
||||
disable_tp: bool = False,
|
||||
quant_method: QuantizeMethodBase | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
@@ -281,8 +283,9 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
)
|
||||
self.embedding_dim = embedding_dim
|
||||
|
||||
quant_method = None
|
||||
if quant_config is not None:
|
||||
# Avoid overriding a preselected model-specific method with generic
|
||||
# config-based dispatch.
|
||||
if quant_method is None and quant_config is not None:
|
||||
quant_method = quant_config.get_quant_method(self, prefix=prefix)
|
||||
if quant_method is None:
|
||||
quant_method = UnquantizedEmbeddingMethod()
|
||||
@@ -501,8 +504,16 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
output_parallel = self.quant_method.embedding(self, masked_input.long())
|
||||
# Mask the output embedding.
|
||||
if self.tp_size > 1:
|
||||
if output_parallel.dtype in (
|
||||
torch.float8_e4m3fn,
|
||||
torch.float8_e5m2,
|
||||
):
|
||||
# Each vocab token has one owner, so FP8 bytes can use int8 SUM.
|
||||
comm_output = output_parallel.view(torch.int8)
|
||||
comm_output.masked_fill_(input_mask.unsqueeze(-1), 0)
|
||||
output = tensor_model_parallel_all_reduce(comm_output)
|
||||
return output.view(output_parallel.dtype)
|
||||
output_parallel.masked_fill_(input_mask.unsqueeze(-1), 0)
|
||||
# Reduce across all the model parallel GPUs.
|
||||
return tensor_model_parallel_all_reduce(output_parallel)
|
||||
return output_parallel
|
||||
|
||||
|
||||
Reference in New Issue
Block a user