[Bugfix] Fix stale HPC QK-norm weights after weight refit (#56706)

Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
This commit is contained in:
aoshen02
2026-09-14 21:34:41 -07:00
committed by GitHub
parent 7dbc3d62c3
commit eb61707851
6 changed files with 143 additions and 98 deletions
+13 -38
View File
@@ -20,9 +20,8 @@ Constraints:
- x / residual must be bfloat16; hc_fn weights stay float32 (the checkpoint
keeps them out of fp8 quantization via modules_to_not_convert)
The fused modules own no parameters: they hold references to the weights of the
eager layer they replace, bound in ``process_weights_after_loading`` (invoked
generically for every ``HpcModule`` by the model loader).
The fused modules own no parameters: their forwards read the weights of the
eager layer they replace directly via the captured owner reference.
NOTE: The reference implementation also offers a cross-layer post+pre fusion
(``HpcIHCPostPre``) that folds one segment's post into the next segment's pre.
@@ -147,41 +146,26 @@ class HpcIHCPre(HpcModule):
object.__setattr__(self, "_fallback_op", fallback_op)
# Same anti-cycle idiom for the fused RMSNorm owner (a sibling module).
object.__setattr__(self, "_norm_owner", norm_owner)
self.weight = None
self.hc_scale = None
self.hc_base = None
self.rms_weight = None
self.rms_eps = 0.0
@classmethod
def support(cls, hc_mult: int, hidden_size: int) -> bool:
return _ihc_supported(hc_mult, hidden_size)
def process_weights_after_loading(self, model=None):
# hc_fn is a ReplicatedLinear whose weight is [2 * hc_mult, hc_mult * d].
# It stays float32 -- the HPC kernel consumes fp32 weights directly, so
# there is no conversion (and no precision loss) here.
owner = self._fallback_op
self.weight = owner.hc_fn.weight
self.hc_scale = owner.hc_scale
self.hc_base = owner.hc_base
if self._norm_owner is not None:
self.rms_weight = self._norm_owner.weight
self.rms_eps = self._norm_owner.variance_epsilon
def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
import hpc
owner = self._fallback_op
norm_owner = self._norm_owner
return hpc.fuse_ihc_pre(
x,
self.weight,
self.hc_scale,
self.hc_base,
owner.hc_fn.weight,
owner.hc_scale,
owner.hc_base,
self.norm_eps,
self.hc_eps,
self.magnitude,
self.rms_weight,
self.rms_eps,
norm_owner.weight if norm_owner is not None else None,
norm_owner.variance_epsilon if norm_owner is not None else 0.0,
)
@@ -233,29 +217,20 @@ class HpcIHCHead(HpcModule):
self.norm_eps = norm_eps
# See HpcIHCPre: keep the owner out of the module tree.
object.__setattr__(self, "_fallback_op", fallback_op)
self.weight = None
self.hc_scale = None
self.hc_base = None
@classmethod
def support(cls, hc_mult: int, hidden_size: int) -> bool:
return _ihc_supported(hc_mult, hidden_size)
def process_weights_after_loading(self, model=None):
# hc_head_fn weight is [hc_mult, hc_mult * d], float32.
owner = self._fallback_op
self.weight = owner.hc_head_fn.weight
self.hc_scale = owner.hc_head_scale
self.hc_base = owner.hc_head_base
def forward(self, x: torch.Tensor) -> torch.Tensor:
import hpc
owner = self._fallback_op
return hpc.fuse_ihc_head(
x,
self.weight,
self.hc_scale,
self.hc_base,
owner.hc_head_fn.weight,
owner.hc_head_scale,
owner.hc_head_base,
self.norm_eps,
self.hc_eps,
)
@@ -11,8 +11,5 @@ class HpcModule(nn.Module):
def support(cls, *args, **kwargs):
return True
def process_weights_after_loading(self, model):
pass
def forward(self, *args, **kwargs):
pass
+18 -45
View File
@@ -90,13 +90,13 @@ class HpcRopeNorm(CustomOp, HpcModule):
"""HPC fused RoPE + QK-Norm + KV-Cache-Write (+ optional FP8 Q quant).
Registered as a sub-module in model layers (e.g. HunYuanAttention).
Norm weights are extracted from fallback norm modules via
process_weights_after_loading() after all weights are loaded.
The QK-Norm weights are read directly from the fallback norm modules
(built in float32 when HPC is active), eliminating the derived copy.
forward() is dispatched by CustomOp framework:
- In compiled mode: forward_cuda() calls torch.ops.vllm.hpc_rope_norm_forward
as a splitting point — internal Python control flow is opaque
to torch.compile and not captured by CUDA Graph.
to torch.compile (its kernel launches can still be CUDA-graph captured).
- In eager/native mode: forward_native() falls back to forward_cuda().
"""
@@ -124,8 +124,6 @@ class HpcRopeNorm(CustomOp, HpcModule):
self.num_kv_heads = num_kv_heads
self.head_dim = head_dim
self.use_qk_norm = use_qk_norm
self.q_size = num_heads * head_dim
self.kv_size = num_kv_heads * head_dim
@@ -139,31 +137,18 @@ class HpcRopeNorm(CustomOp, HpcModule):
self.head_per_group = num_heads // num_kv_heads
# Pre-allocate norm weight tensors as Parameters so they are tracked by
# CuMemAllocator (for sleep/wake_up) and have stable addresses for CUDA
# Graph replay. process_weights_after_loading() updates them inplace via
# copy_() so refit does not invalidate captured graph tensor pointers.
# Shape is [head_dim] to match the HPC kernel's q/k_norm_weight layout.
if use_qk_norm and fallback_qnorm is not None:
self.qnorm_weight: torch.nn.Parameter | None = torch.nn.Parameter(
torch.empty(head_dim, dtype=torch.float32),
requires_grad=False,
)
else:
self.qnorm_weight = None
if use_qk_norm and fallback_knorm is not None:
self.knorm_weight: torch.nn.Parameter | None = torch.nn.Parameter(
torch.empty(head_dim, dtype=torch.float32),
requires_grad=False,
)
else:
self.knorm_weight = None
self.use_fp8 = "fp8" in kv_cache_dtype
# The RMSNorm/RoPE ordering is model dependent (e.g. HunYuan V3 applies
# QK-Norm before RoPE -> NORM_THEN_ROPE), so it is supplied by the
# caller. When QK-Norm is disabled the policy is forced to NONE.
self.qk_norm_policy = qk_norm_policy if use_qk_norm else QkNormPolicy.NONE
if self.qk_norm_policy != QkNormPolicy.NONE and (
fallback_qnorm is None or fallback_knorm is None
):
raise ValueError(
"QK-Norm is enabled but fallback_qnorm/fallback_knorm were not "
"supplied; the fused kernel reads their weights directly."
)
# Register layer_name + add self to the global instance registry so the
# module-level custom op (hpc_rope_norm_forward) can route back here.
@@ -215,20 +200,6 @@ class HpcRopeNorm(CustomOp, HpcModule):
logger.info_once("enable hpc rope_norm")
return True
def process_weights_after_loading(self, model: torch.nn.Module = None) -> None:
"""Copy norm weights (float32) from fallback norm modules inplace.
Uses copy_() to preserve tensor addresses for CUDA Graph / refit
compatibility. Called by the model's load_weights() after all weights
are loaded (and generically from the model loader for DummyModelLoader
/ sleep-wake_up reload paths).
"""
if self.use_qk_norm:
if self.fallback_qnorm is not None and self.qnorm_weight is not None:
self.qnorm_weight.data.copy_(self.fallback_qnorm.weight.data.float())
if self.fallback_knorm is not None and self.knorm_weight is not None:
self.knorm_weight.data.copy_(self.fallback_knorm.weight.data.float())
def register_layer_name(self, layer_name: str) -> None:
"""Register layer_name and add self to the global registry.
@@ -430,12 +401,14 @@ class HpcRopeNorm(CustomOp, HpcModule):
k_scale = attn_layer._k_scale.reshape(1)
v_scale = attn_layer._v_scale.reshape(1)
q_norm_weight = (
self.qnorm_weight if self.qk_norm_policy != QkNormPolicy.NONE else None
)
k_norm_weight = (
self.knorm_weight if self.qk_norm_policy != QkNormPolicy.NONE else None
)
if self.qk_norm_policy != QkNormPolicy.NONE:
assert self.fallback_qnorm is not None
assert self.fallback_knorm is not None
q_norm_weight = self.fallback_qnorm.weight
k_norm_weight = self.fallback_knorm.weight
else:
q_norm_weight = None
k_norm_weight = None
# --- Prefill ---
if num_prefill_reqs > 0: