[Attention Backend] add HPC-Ops Attention backend (#46020)
Signed-off-by: chengvjiang <chengvjiang@tencent.com> Co-authored-by: chengvjiang <chengvjiang@tencent.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
chengvjiang
Andreas Karatzas
parent
8e9d70fdd5
commit
8cf7c4d8ad
@@ -20,6 +20,7 @@ from vllm.model_executor.layers.attention import (
|
||||
MLAAttention,
|
||||
MMEncoderAttention,
|
||||
)
|
||||
from vllm.model_executor.layers.hpc import HpcModule
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
QuantizeMethodBase,
|
||||
@@ -125,6 +126,15 @@ def process_weights_after_loading(
|
||||
with device_loading_context(module, target_device):
|
||||
module.process_weights_after_loading(model_config.dtype)
|
||||
|
||||
# Process HPC modules (HpcRopeNorm, etc.) that rely on
|
||||
# process_weights_after_loading being called from the model's
|
||||
# load_weights(). When using DummyModelLoader (e.g. profiling or
|
||||
# sleep/wake_up reload), the model's load_weights() is not called, so we
|
||||
# must handle HPC modules here generically.
|
||||
for _, module in model.named_modules():
|
||||
if isinstance(module, HpcModule):
|
||||
module.process_weights_after_loading(model)
|
||||
|
||||
# Needed for torchao model reloading via model.reload_weights
|
||||
# @kylesayrs @jerryzh168 this can be removed if callers move to `reload_weights`
|
||||
if model_config.quantization == "torchao":
|
||||
|
||||
Reference in New Issue
Block a user