[Qwen4Exp] Support UVA PLE-offload and Engram tensor parallelism (#54371)
Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
62f3bf58a5
commit
3116c5d06b
@@ -4,7 +4,7 @@
|
||||
|
||||
import inspect
|
||||
import warnings
|
||||
from contextlib import contextmanager
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
@@ -120,11 +120,18 @@ def process_weights_after_loading(
|
||||
"does not support pre-processed weights"
|
||||
)
|
||||
# When quant methods need to process weights after loading
|
||||
# (for repacking, quantizing, etc), they expect parameters
|
||||
# (for repacking, quantizing, etc), they typically expect parameters
|
||||
# to be on the global target device. This scope is for the
|
||||
# case where cpu offloading is used, where we will move the
|
||||
# parameters onto device for processing and back off after.
|
||||
with device_loading_context(module, target_device):
|
||||
# Methods that can process weights in place (e.g. PLE scale
|
||||
# validation) set requires_device_loading=False to skip this move.
|
||||
loading_context = (
|
||||
device_loading_context(module, target_device)
|
||||
if quant_method.requires_device_loading
|
||||
else nullcontext()
|
||||
)
|
||||
with loading_context:
|
||||
quant_method.process_weights_after_loading(module)
|
||||
# process_weights_after_loading may swap in freshly-created
|
||||
# Parameters (e.g. FP8 requantization), which are stamped with the
|
||||
|
||||
Reference in New Issue
Block a user