[Qwen4Exp] Support UVA PLE-offload and Engram tensor parallelism (#54371)
Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
62f3bf58a5
commit
3116c5d06b
@@ -23,6 +23,9 @@ else:
|
||||
class QuantizeMethodBase(ABC):
|
||||
"""Base class for different quantized methods."""
|
||||
|
||||
requires_device_loading: bool = True
|
||||
"""Whether post-load processing requires parameters on the target device."""
|
||||
|
||||
uses_meta_device: bool = False
|
||||
"""
|
||||
Whether this method creates weights on meta device for online quantization.
|
||||
|
||||
@@ -11,6 +11,7 @@ from torch.nn.parameter import Parameter
|
||||
import vllm.envs as envs
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.distributed import (
|
||||
GroupCoordinator,
|
||||
divide,
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
@@ -240,6 +241,7 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
prefix: full name of the layer in the state dict
|
||||
disable_tp: If true, tensor parallelism will be disabled for this layer.
|
||||
quant_method: Preselected quantization method for model-specific layers.
|
||||
parallel_group: Process group used to shard and reduce the embedding.
|
||||
""" # noqa: E501
|
||||
|
||||
# --8<-- [end:vocab_parallel_embedding]
|
||||
@@ -256,13 +258,18 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
*,
|
||||
disable_tp: bool = False,
|
||||
quant_method: QuantizeMethodBase | None = None,
|
||||
parallel_group: GroupCoordinator | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
# Keep the input dimensions.
|
||||
self.disable_tp = disable_tp
|
||||
self.parallel_group = parallel_group
|
||||
if disable_tp:
|
||||
tp_rank, self.tp_size = 0, 1
|
||||
elif parallel_group is not None:
|
||||
tp_rank = parallel_group.rank_in_group
|
||||
self.tp_size = parallel_group.world_size
|
||||
else:
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
self.tp_size = get_tensor_model_parallel_world_size()
|
||||
@@ -535,10 +542,16 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
# Each vocab token has one owner, so FP8 bytes can use int8 SUM.
|
||||
comm_output = output_parallel.view(torch.int8)
|
||||
comm_output.masked_fill_(input_mask.unsqueeze(-1), 0)
|
||||
output = tensor_model_parallel_all_reduce(comm_output)
|
||||
output = (
|
||||
self.parallel_group.all_reduce(comm_output)
|
||||
if self.parallel_group is not None
|
||||
else tensor_model_parallel_all_reduce(comm_output)
|
||||
)
|
||||
return output.view(output_parallel.dtype)
|
||||
output_parallel.masked_fill_(input_mask.unsqueeze(-1), 0)
|
||||
# Reduce across all the model parallel GPUs.
|
||||
if self.parallel_group is not None:
|
||||
return self.parallel_group.all_reduce(output_parallel)
|
||||
return tensor_model_parallel_all_reduce(output_parallel)
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
|
||||
Reference in New Issue
Block a user