[Bugfix] Fix stale HPC QK-norm weights after weight refit (#56706)
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
This commit is contained in:
@@ -1,7 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import gc
|
||||
import importlib.machinery
|
||||
import inspect
|
||||
import sys
|
||||
import types
|
||||
from unittest.mock import Mock
|
||||
from weakref import WeakKeyDictionary, ref
|
||||
|
||||
@@ -817,6 +820,110 @@ def test_layerwise_reload_updates_loaded_non_persistent_buffers(monkeypatch):
|
||||
assert "0.scale" not in model.state_dict()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def hpc_rope_norm(monkeypatch, default_vllm_config):
|
||||
"""Import HpcRopeNorm with the external ``hpc`` package stubbed out."""
|
||||
if "hpc" not in sys.modules:
|
||||
stub = types.ModuleType("hpc")
|
||||
stub.__spec__ = importlib.machinery.ModuleSpec("hpc", loader=None)
|
||||
stub.QuantType = types.SimpleNamespace( # type: ignore[attr-defined]
|
||||
QPERTOKEN_PERHEAD_KPERTENSOR_VPERTENSOR=types.SimpleNamespace(value=0)
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "hpc", stub)
|
||||
from vllm.model_executor.layers.hpc import rope_norm
|
||||
|
||||
monkeypatch.setattr(rope_norm, "_hpc_rope_norm_instances", {})
|
||||
return rope_norm
|
||||
|
||||
|
||||
def test_hpc_rope_norm_kernel_sees_refit_norm_weights(monkeypatch, hpc_rope_norm):
|
||||
"""The fused HPC kernel is handed the live QK-norm weights after a refit.
|
||||
|
||||
Drives the production ``_forward_impl`` with a recording ``hpc`` stub. The
|
||||
Q/K norm weights it receives must be the model's own float32 parameters,
|
||||
so a layerwise reload that rewrites them in place (same storage) is what
|
||||
the kernel sees. Previously the kernel read separate mirrors that no
|
||||
reload path refreshed.
|
||||
"""
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
|
||||
head_dim, num_heads, num_kv_heads, block_size = 128, 8, 1, 4
|
||||
layer = torch.nn.Module()
|
||||
layer.q_norm = RMSNorm(head_dim, 1e-6, dtype=torch.float32)
|
||||
layer.k_norm = RMSNorm(head_dim, 1e-6, dtype=torch.float32)
|
||||
layer.hpc_rope_norm = hpc_rope_norm.HpcRopeNorm(
|
||||
num_heads=num_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_dim=head_dim,
|
||||
cos_sin_cache=torch.ones(16, head_dim),
|
||||
use_qk_norm=True,
|
||||
fallback_qnorm=layer.q_norm,
|
||||
fallback_knorm=layer.k_norm,
|
||||
kv_cache_dtype="auto",
|
||||
layer_name="hpc_test_layer",
|
||||
)
|
||||
model = torch.nn.Sequential(layer)
|
||||
rnorm = layer.hpc_rope_norm
|
||||
|
||||
calls: list[dict] = []
|
||||
|
||||
def record(*args, **kwargs):
|
||||
calls.append(kwargs)
|
||||
|
||||
monkeypatch.setattr(sys.modules["hpc"], "rope_norm_store_kv", record, raising=False)
|
||||
|
||||
def kernel_norm_weights():
|
||||
q_size, kv_size = num_heads * head_dim, num_kv_heads * head_dim
|
||||
qkv = torch.zeros(1, q_size + 2 * kv_size, dtype=torch.bfloat16)
|
||||
kv_cache = torch.zeros(
|
||||
2, num_kv_heads, block_size, 2 * head_dim, dtype=torch.bfloat16
|
||||
)
|
||||
attn_layer = types.SimpleNamespace(
|
||||
_k_scale=torch.ones(1), _v_scale=torch.ones(1)
|
||||
)
|
||||
attn_metadata = types.SimpleNamespace(
|
||||
num_actual_tokens=1,
|
||||
num_decodes=1,
|
||||
num_decode_tokens=1,
|
||||
num_prefills=0,
|
||||
num_prefill_tokens=0,
|
||||
max_query_len=1,
|
||||
decode_query_len=1,
|
||||
qo_indptr=None,
|
||||
qo_indptr_decode=None,
|
||||
slot_mapping=torch.tensor([4]),
|
||||
seq_lens=torch.tensor([1]),
|
||||
block_table_tensor=torch.tensor([[1]]),
|
||||
hpc_kv_written=False,
|
||||
)
|
||||
output = torch.zeros(1, q_size, dtype=torch.bfloat16)
|
||||
rnorm._forward_impl(qkv, kv_cache, attn_metadata, attn_layer, output)
|
||||
return calls[-1]["q_norm_weight"], calls[-1]["k_norm_weight"]
|
||||
|
||||
def loaded(value):
|
||||
return torch.full((head_dim,), value, dtype=torch.bfloat16)
|
||||
|
||||
default_weight_loader(layer.q_norm.weight, loaded(0.5))
|
||||
default_weight_loader(layer.k_norm.weight, loaded(0.25))
|
||||
q, k = kernel_norm_weights()
|
||||
assert torch.equal(q, loaded(0.5).float())
|
||||
assert torch.equal(k, loaded(0.25).float())
|
||||
q_ptr, k_ptr = q.data_ptr(), k.data_ptr()
|
||||
|
||||
record_metadata_for_reloading(model)
|
||||
initialize_layerwise_reload(model)
|
||||
layer.q_norm.weight.weight_loader(layer.q_norm.weight, loaded(2.0))
|
||||
layer.k_norm.weight.weight_loader(layer.k_norm.weight, loaded(3.0))
|
||||
finalize_layerwise_reload(model, model_config=None)
|
||||
|
||||
q, k = kernel_norm_weights()
|
||||
assert q.dtype == k.dtype == torch.float32
|
||||
assert torch.equal(q, loaded(2.0).float())
|
||||
assert torch.equal(k, loaded(3.0).float())
|
||||
assert (q.data_ptr(), k.data_ptr()) == (q_ptr, k_ptr)
|
||||
assert not hasattr(rnorm, "qnorm_weight")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tp_size", [pytest.param(1), pytest.param(2, marks=[pytest.mark.slow_test])]
|
||||
)
|
||||
|
||||
@@ -20,9 +20,8 @@ Constraints:
|
||||
- x / residual must be bfloat16; hc_fn weights stay float32 (the checkpoint
|
||||
keeps them out of fp8 quantization via modules_to_not_convert)
|
||||
|
||||
The fused modules own no parameters: they hold references to the weights of the
|
||||
eager layer they replace, bound in ``process_weights_after_loading`` (invoked
|
||||
generically for every ``HpcModule`` by the model loader).
|
||||
The fused modules own no parameters: their forwards read the weights of the
|
||||
eager layer they replace directly via the captured owner reference.
|
||||
|
||||
NOTE: The reference implementation also offers a cross-layer post+pre fusion
|
||||
(``HpcIHCPostPre``) that folds one segment's post into the next segment's pre.
|
||||
@@ -147,41 +146,26 @@ class HpcIHCPre(HpcModule):
|
||||
object.__setattr__(self, "_fallback_op", fallback_op)
|
||||
# Same anti-cycle idiom for the fused RMSNorm owner (a sibling module).
|
||||
object.__setattr__(self, "_norm_owner", norm_owner)
|
||||
self.weight = None
|
||||
self.hc_scale = None
|
||||
self.hc_base = None
|
||||
self.rms_weight = None
|
||||
self.rms_eps = 0.0
|
||||
|
||||
@classmethod
|
||||
def support(cls, hc_mult: int, hidden_size: int) -> bool:
|
||||
return _ihc_supported(hc_mult, hidden_size)
|
||||
|
||||
def process_weights_after_loading(self, model=None):
|
||||
# hc_fn is a ReplicatedLinear whose weight is [2 * hc_mult, hc_mult * d].
|
||||
# It stays float32 -- the HPC kernel consumes fp32 weights directly, so
|
||||
# there is no conversion (and no precision loss) here.
|
||||
owner = self._fallback_op
|
||||
self.weight = owner.hc_fn.weight
|
||||
self.hc_scale = owner.hc_scale
|
||||
self.hc_base = owner.hc_base
|
||||
if self._norm_owner is not None:
|
||||
self.rms_weight = self._norm_owner.weight
|
||||
self.rms_eps = self._norm_owner.variance_epsilon
|
||||
|
||||
def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
import hpc
|
||||
|
||||
owner = self._fallback_op
|
||||
norm_owner = self._norm_owner
|
||||
return hpc.fuse_ihc_pre(
|
||||
x,
|
||||
self.weight,
|
||||
self.hc_scale,
|
||||
self.hc_base,
|
||||
owner.hc_fn.weight,
|
||||
owner.hc_scale,
|
||||
owner.hc_base,
|
||||
self.norm_eps,
|
||||
self.hc_eps,
|
||||
self.magnitude,
|
||||
self.rms_weight,
|
||||
self.rms_eps,
|
||||
norm_owner.weight if norm_owner is not None else None,
|
||||
norm_owner.variance_epsilon if norm_owner is not None else 0.0,
|
||||
)
|
||||
|
||||
|
||||
@@ -233,29 +217,20 @@ class HpcIHCHead(HpcModule):
|
||||
self.norm_eps = norm_eps
|
||||
# See HpcIHCPre: keep the owner out of the module tree.
|
||||
object.__setattr__(self, "_fallback_op", fallback_op)
|
||||
self.weight = None
|
||||
self.hc_scale = None
|
||||
self.hc_base = None
|
||||
|
||||
@classmethod
|
||||
def support(cls, hc_mult: int, hidden_size: int) -> bool:
|
||||
return _ihc_supported(hc_mult, hidden_size)
|
||||
|
||||
def process_weights_after_loading(self, model=None):
|
||||
# hc_head_fn weight is [hc_mult, hc_mult * d], float32.
|
||||
owner = self._fallback_op
|
||||
self.weight = owner.hc_head_fn.weight
|
||||
self.hc_scale = owner.hc_head_scale
|
||||
self.hc_base = owner.hc_head_base
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
import hpc
|
||||
|
||||
owner = self._fallback_op
|
||||
return hpc.fuse_ihc_head(
|
||||
x,
|
||||
self.weight,
|
||||
self.hc_scale,
|
||||
self.hc_base,
|
||||
owner.hc_head_fn.weight,
|
||||
owner.hc_head_scale,
|
||||
owner.hc_head_base,
|
||||
self.norm_eps,
|
||||
self.hc_eps,
|
||||
)
|
||||
|
||||
@@ -11,8 +11,5 @@ class HpcModule(nn.Module):
|
||||
def support(cls, *args, **kwargs):
|
||||
return True
|
||||
|
||||
def process_weights_after_loading(self, model):
|
||||
pass
|
||||
|
||||
def forward(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
@@ -90,13 +90,13 @@ class HpcRopeNorm(CustomOp, HpcModule):
|
||||
"""HPC fused RoPE + QK-Norm + KV-Cache-Write (+ optional FP8 Q quant).
|
||||
|
||||
Registered as a sub-module in model layers (e.g. HunYuanAttention).
|
||||
Norm weights are extracted from fallback norm modules via
|
||||
process_weights_after_loading() after all weights are loaded.
|
||||
The QK-Norm weights are read directly from the fallback norm modules
|
||||
(built in float32 when HPC is active), eliminating the derived copy.
|
||||
|
||||
forward() is dispatched by CustomOp framework:
|
||||
- In compiled mode: forward_cuda() calls torch.ops.vllm.hpc_rope_norm_forward
|
||||
as a splitting point — internal Python control flow is opaque
|
||||
to torch.compile and not captured by CUDA Graph.
|
||||
to torch.compile (its kernel launches can still be CUDA-graph captured).
|
||||
- In eager/native mode: forward_native() falls back to forward_cuda().
|
||||
"""
|
||||
|
||||
@@ -124,8 +124,6 @@ class HpcRopeNorm(CustomOp, HpcModule):
|
||||
self.num_kv_heads = num_kv_heads
|
||||
self.head_dim = head_dim
|
||||
|
||||
self.use_qk_norm = use_qk_norm
|
||||
|
||||
self.q_size = num_heads * head_dim
|
||||
self.kv_size = num_kv_heads * head_dim
|
||||
|
||||
@@ -139,31 +137,18 @@ class HpcRopeNorm(CustomOp, HpcModule):
|
||||
|
||||
self.head_per_group = num_heads // num_kv_heads
|
||||
|
||||
# Pre-allocate norm weight tensors as Parameters so they are tracked by
|
||||
# CuMemAllocator (for sleep/wake_up) and have stable addresses for CUDA
|
||||
# Graph replay. process_weights_after_loading() updates them inplace via
|
||||
# copy_() so refit does not invalidate captured graph tensor pointers.
|
||||
# Shape is [head_dim] to match the HPC kernel's q/k_norm_weight layout.
|
||||
if use_qk_norm and fallback_qnorm is not None:
|
||||
self.qnorm_weight: torch.nn.Parameter | None = torch.nn.Parameter(
|
||||
torch.empty(head_dim, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
else:
|
||||
self.qnorm_weight = None
|
||||
if use_qk_norm and fallback_knorm is not None:
|
||||
self.knorm_weight: torch.nn.Parameter | None = torch.nn.Parameter(
|
||||
torch.empty(head_dim, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
else:
|
||||
self.knorm_weight = None
|
||||
|
||||
self.use_fp8 = "fp8" in kv_cache_dtype
|
||||
# The RMSNorm/RoPE ordering is model dependent (e.g. HunYuan V3 applies
|
||||
# QK-Norm before RoPE -> NORM_THEN_ROPE), so it is supplied by the
|
||||
# caller. When QK-Norm is disabled the policy is forced to NONE.
|
||||
self.qk_norm_policy = qk_norm_policy if use_qk_norm else QkNormPolicy.NONE
|
||||
if self.qk_norm_policy != QkNormPolicy.NONE and (
|
||||
fallback_qnorm is None or fallback_knorm is None
|
||||
):
|
||||
raise ValueError(
|
||||
"QK-Norm is enabled but fallback_qnorm/fallback_knorm were not "
|
||||
"supplied; the fused kernel reads their weights directly."
|
||||
)
|
||||
|
||||
# Register layer_name + add self to the global instance registry so the
|
||||
# module-level custom op (hpc_rope_norm_forward) can route back here.
|
||||
@@ -215,20 +200,6 @@ class HpcRopeNorm(CustomOp, HpcModule):
|
||||
logger.info_once("enable hpc rope_norm")
|
||||
return True
|
||||
|
||||
def process_weights_after_loading(self, model: torch.nn.Module = None) -> None:
|
||||
"""Copy norm weights (float32) from fallback norm modules inplace.
|
||||
|
||||
Uses copy_() to preserve tensor addresses for CUDA Graph / refit
|
||||
compatibility. Called by the model's load_weights() after all weights
|
||||
are loaded (and generically from the model loader for DummyModelLoader
|
||||
/ sleep-wake_up reload paths).
|
||||
"""
|
||||
if self.use_qk_norm:
|
||||
if self.fallback_qnorm is not None and self.qnorm_weight is not None:
|
||||
self.qnorm_weight.data.copy_(self.fallback_qnorm.weight.data.float())
|
||||
if self.fallback_knorm is not None and self.knorm_weight is not None:
|
||||
self.knorm_weight.data.copy_(self.fallback_knorm.weight.data.float())
|
||||
|
||||
def register_layer_name(self, layer_name: str) -> None:
|
||||
"""Register layer_name and add self to the global registry.
|
||||
|
||||
@@ -430,12 +401,14 @@ class HpcRopeNorm(CustomOp, HpcModule):
|
||||
k_scale = attn_layer._k_scale.reshape(1)
|
||||
v_scale = attn_layer._v_scale.reshape(1)
|
||||
|
||||
q_norm_weight = (
|
||||
self.qnorm_weight if self.qk_norm_policy != QkNormPolicy.NONE else None
|
||||
)
|
||||
k_norm_weight = (
|
||||
self.knorm_weight if self.qk_norm_policy != QkNormPolicy.NONE else None
|
||||
)
|
||||
if self.qk_norm_policy != QkNormPolicy.NONE:
|
||||
assert self.fallback_qnorm is not None
|
||||
assert self.fallback_knorm is not None
|
||||
q_norm_weight = self.fallback_qnorm.weight
|
||||
k_norm_weight = self.fallback_knorm.weight
|
||||
else:
|
||||
q_norm_weight = None
|
||||
k_norm_weight = None
|
||||
|
||||
# --- Prefill ---
|
||||
if num_prefill_reqs > 0:
|
||||
|
||||
@@ -15,7 +15,6 @@ import vllm.envs as envs
|
||||
from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import is_deferred_attention_layer
|
||||
from vllm.model_executor.layers.hpc import HpcModule
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
QuantizeMethodBase,
|
||||
@@ -154,15 +153,6 @@ def process_weights_after_loading(
|
||||
with device_loading_context(module, target_device):
|
||||
module.process_weights_after_loading(model_config.dtype)
|
||||
|
||||
# Process HPC modules (HpcRopeNorm, etc.) that rely on
|
||||
# process_weights_after_loading being called from the model's
|
||||
# load_weights(). When using DummyModelLoader (e.g. profiling or
|
||||
# sleep/wake_up reload), the model's load_weights() is not called, so we
|
||||
# must handle HPC modules here generically.
|
||||
for _, module in model.named_modules():
|
||||
if isinstance(module, HpcModule):
|
||||
module.process_weights_after_loading(model)
|
||||
|
||||
# Model-level post-load hook, after the per-layer quant finalize.
|
||||
if hasattr(model, "process_weights_after_loading"):
|
||||
model.process_weights_after_loading()
|
||||
|
||||
@@ -300,8 +300,11 @@ class HYV3Attention(nn.Module):
|
||||
prefix=f"{prefix}.attn",
|
||||
)
|
||||
if self.use_qk_norm:
|
||||
self.q_norm = RMSNorm(self.head_dim, rms_norm_eps)
|
||||
self.k_norm = RMSNorm(self.head_dim, rms_norm_eps)
|
||||
# The HPC fused kernel reads norm weights directly in float32;
|
||||
# the fallback path keeps the model default dtype.
|
||||
norm_dtype = torch.float32 if rope_support else None
|
||||
self.q_norm = RMSNorm(self.head_dim, rms_norm_eps, dtype=norm_dtype)
|
||||
self.k_norm = RMSNorm(self.head_dim, rms_norm_eps, dtype=norm_dtype)
|
||||
|
||||
# HPC fused RoPE + QK-Norm + KV-Cache-Write (+ optional FP8 Q quant).
|
||||
# HunYuan V3 applies QK-Norm *before* RoPE, so NORM_THEN_ROPE.
|
||||
|
||||
Reference in New Issue
Block a user