[Qwen4Exp] Support UVA PLE-offload and Engram tensor parallelism (#54371)

Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Harry Huang
2026-09-09 07:32:34 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 62f3bf58a5
commit 3116c5d06b
16 changed files with 1712 additions and 495 deletions
@@ -23,6 +23,9 @@ else:
class QuantizeMethodBase(ABC):
"""Base class for different quantized methods."""
requires_device_loading: bool = True
"""Whether post-load processing requires parameters on the target device."""
uses_meta_device: bool = False
"""
Whether this method creates weights on meta device for online quantization.
@@ -11,6 +11,7 @@ from torch.nn.parameter import Parameter
import vllm.envs as envs
from vllm import _custom_ops as ops
from vllm.distributed import (
GroupCoordinator,
divide,
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
@@ -240,6 +241,7 @@ class VocabParallelEmbedding(PluggableLayer):
prefix: full name of the layer in the state dict
disable_tp: If true, tensor parallelism will be disabled for this layer.
quant_method: Preselected quantization method for model-specific layers.
parallel_group: Process group used to shard and reduce the embedding.
""" # noqa: E501
# --8<-- [end:vocab_parallel_embedding]
@@ -256,13 +258,18 @@ class VocabParallelEmbedding(PluggableLayer):
*,
disable_tp: bool = False,
quant_method: QuantizeMethodBase | None = None,
parallel_group: GroupCoordinator | None = None,
):
super().__init__()
# Keep the input dimensions.
self.disable_tp = disable_tp
self.parallel_group = parallel_group
if disable_tp:
tp_rank, self.tp_size = 0, 1
elif parallel_group is not None:
tp_rank = parallel_group.rank_in_group
self.tp_size = parallel_group.world_size
else:
tp_rank = get_tensor_model_parallel_rank()
self.tp_size = get_tensor_model_parallel_world_size()
@@ -535,10 +542,16 @@ class VocabParallelEmbedding(PluggableLayer):
# Each vocab token has one owner, so FP8 bytes can use int8 SUM.
comm_output = output_parallel.view(torch.int8)
comm_output.masked_fill_(input_mask.unsqueeze(-1), 0)
output = tensor_model_parallel_all_reduce(comm_output)
output = (
self.parallel_group.all_reduce(comm_output)
if self.parallel_group is not None
else tensor_model_parallel_all_reduce(comm_output)
)
return output.view(output_parallel.dtype)
output_parallel.masked_fill_(input_mask.unsqueeze(-1), 0)
# Reduce across all the model parallel GPUs.
if self.parallel_group is not None:
return self.parallel_group.all_reduce(output_parallel)
return tensor_model_parallel_all_reduce(output_parallel)
def extra_repr(self) -> str:
+10 -3
View File
@@ -4,7 +4,7 @@
import inspect
import warnings
from contextlib import contextmanager
from contextlib import contextmanager, nullcontext
from typing import Any
import torch
@@ -120,11 +120,18 @@ def process_weights_after_loading(
"does not support pre-processed weights"
)
# When quant methods need to process weights after loading
# (for repacking, quantizing, etc), they expect parameters
# (for repacking, quantizing, etc), they typically expect parameters
# to be on the global target device. This scope is for the
# case where cpu offloading is used, where we will move the
# parameters onto device for processing and back off after.
with device_loading_context(module, target_device):
# Methods that can process weights in place (e.g. PLE scale
# validation) set requires_device_loading=False to skip this move.
loading_context = (
device_loading_context(module, target_device)
if quant_method.requires_device_loading
else nullcontext()
)
with loading_context:
quant_method.process_weights_after_loading(module)
# process_weights_after_loading may swap in freshly-created
# Parameters (e.g. FP8 requantization), which are stamped with the