diff --git a/tests/model_executor/model_loader/test_reload.py b/tests/model_executor/model_loader/test_reload.py index f19ec35560..b4a544d2fc 100644 --- a/tests/model_executor/model_loader/test_reload.py +++ b/tests/model_executor/model_loader/test_reload.py @@ -1,7 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import gc +import importlib.machinery import inspect +import sys +import types from unittest.mock import Mock from weakref import WeakKeyDictionary, ref @@ -817,6 +820,110 @@ def test_layerwise_reload_updates_loaded_non_persistent_buffers(monkeypatch): assert "0.scale" not in model.state_dict() +@pytest.fixture +def hpc_rope_norm(monkeypatch, default_vllm_config): + """Import HpcRopeNorm with the external ``hpc`` package stubbed out.""" + if "hpc" not in sys.modules: + stub = types.ModuleType("hpc") + stub.__spec__ = importlib.machinery.ModuleSpec("hpc", loader=None) + stub.QuantType = types.SimpleNamespace( # type: ignore[attr-defined] + QPERTOKEN_PERHEAD_KPERTENSOR_VPERTENSOR=types.SimpleNamespace(value=0) + ) + monkeypatch.setitem(sys.modules, "hpc", stub) + from vllm.model_executor.layers.hpc import rope_norm + + monkeypatch.setattr(rope_norm, "_hpc_rope_norm_instances", {}) + return rope_norm + + +def test_hpc_rope_norm_kernel_sees_refit_norm_weights(monkeypatch, hpc_rope_norm): + """The fused HPC kernel is handed the live QK-norm weights after a refit. + + Drives the production ``_forward_impl`` with a recording ``hpc`` stub. The + Q/K norm weights it receives must be the model's own float32 parameters, + so a layerwise reload that rewrites them in place (same storage) is what + the kernel sees. Previously the kernel read separate mirrors that no + reload path refreshed. + """ + from vllm.model_executor.layers.layernorm import RMSNorm + + head_dim, num_heads, num_kv_heads, block_size = 128, 8, 1, 4 + layer = torch.nn.Module() + layer.q_norm = RMSNorm(head_dim, 1e-6, dtype=torch.float32) + layer.k_norm = RMSNorm(head_dim, 1e-6, dtype=torch.float32) + layer.hpc_rope_norm = hpc_rope_norm.HpcRopeNorm( + num_heads=num_heads, + num_kv_heads=num_kv_heads, + head_dim=head_dim, + cos_sin_cache=torch.ones(16, head_dim), + use_qk_norm=True, + fallback_qnorm=layer.q_norm, + fallback_knorm=layer.k_norm, + kv_cache_dtype="auto", + layer_name="hpc_test_layer", + ) + model = torch.nn.Sequential(layer) + rnorm = layer.hpc_rope_norm + + calls: list[dict] = [] + + def record(*args, **kwargs): + calls.append(kwargs) + + monkeypatch.setattr(sys.modules["hpc"], "rope_norm_store_kv", record, raising=False) + + def kernel_norm_weights(): + q_size, kv_size = num_heads * head_dim, num_kv_heads * head_dim + qkv = torch.zeros(1, q_size + 2 * kv_size, dtype=torch.bfloat16) + kv_cache = torch.zeros( + 2, num_kv_heads, block_size, 2 * head_dim, dtype=torch.bfloat16 + ) + attn_layer = types.SimpleNamespace( + _k_scale=torch.ones(1), _v_scale=torch.ones(1) + ) + attn_metadata = types.SimpleNamespace( + num_actual_tokens=1, + num_decodes=1, + num_decode_tokens=1, + num_prefills=0, + num_prefill_tokens=0, + max_query_len=1, + decode_query_len=1, + qo_indptr=None, + qo_indptr_decode=None, + slot_mapping=torch.tensor([4]), + seq_lens=torch.tensor([1]), + block_table_tensor=torch.tensor([[1]]), + hpc_kv_written=False, + ) + output = torch.zeros(1, q_size, dtype=torch.bfloat16) + rnorm._forward_impl(qkv, kv_cache, attn_metadata, attn_layer, output) + return calls[-1]["q_norm_weight"], calls[-1]["k_norm_weight"] + + def loaded(value): + return torch.full((head_dim,), value, dtype=torch.bfloat16) + + default_weight_loader(layer.q_norm.weight, loaded(0.5)) + default_weight_loader(layer.k_norm.weight, loaded(0.25)) + q, k = kernel_norm_weights() + assert torch.equal(q, loaded(0.5).float()) + assert torch.equal(k, loaded(0.25).float()) + q_ptr, k_ptr = q.data_ptr(), k.data_ptr() + + record_metadata_for_reloading(model) + initialize_layerwise_reload(model) + layer.q_norm.weight.weight_loader(layer.q_norm.weight, loaded(2.0)) + layer.k_norm.weight.weight_loader(layer.k_norm.weight, loaded(3.0)) + finalize_layerwise_reload(model, model_config=None) + + q, k = kernel_norm_weights() + assert q.dtype == k.dtype == torch.float32 + assert torch.equal(q, loaded(2.0).float()) + assert torch.equal(k, loaded(3.0).float()) + assert (q.data_ptr(), k.data_ptr()) == (q_ptr, k_ptr) + assert not hasattr(rnorm, "qnorm_weight") + + @pytest.mark.parametrize( "tp_size", [pytest.param(1), pytest.param(2, marks=[pytest.mark.slow_test])] ) diff --git a/vllm/model_executor/layers/hpc/hpc_ihc.py b/vllm/model_executor/layers/hpc/hpc_ihc.py index 1a8681aa8e..9a8065a052 100644 --- a/vllm/model_executor/layers/hpc/hpc_ihc.py +++ b/vllm/model_executor/layers/hpc/hpc_ihc.py @@ -20,9 +20,8 @@ Constraints: - x / residual must be bfloat16; hc_fn weights stay float32 (the checkpoint keeps them out of fp8 quantization via modules_to_not_convert) -The fused modules own no parameters: they hold references to the weights of the -eager layer they replace, bound in ``process_weights_after_loading`` (invoked -generically for every ``HpcModule`` by the model loader). +The fused modules own no parameters: their forwards read the weights of the +eager layer they replace directly via the captured owner reference. NOTE: The reference implementation also offers a cross-layer post+pre fusion (``HpcIHCPostPre``) that folds one segment's post into the next segment's pre. @@ -147,41 +146,26 @@ class HpcIHCPre(HpcModule): object.__setattr__(self, "_fallback_op", fallback_op) # Same anti-cycle idiom for the fused RMSNorm owner (a sibling module). object.__setattr__(self, "_norm_owner", norm_owner) - self.weight = None - self.hc_scale = None - self.hc_base = None - self.rms_weight = None - self.rms_eps = 0.0 @classmethod def support(cls, hc_mult: int, hidden_size: int) -> bool: return _ihc_supported(hc_mult, hidden_size) - def process_weights_after_loading(self, model=None): - # hc_fn is a ReplicatedLinear whose weight is [2 * hc_mult, hc_mult * d]. - # It stays float32 -- the HPC kernel consumes fp32 weights directly, so - # there is no conversion (and no precision loss) here. - owner = self._fallback_op - self.weight = owner.hc_fn.weight - self.hc_scale = owner.hc_scale - self.hc_base = owner.hc_base - if self._norm_owner is not None: - self.rms_weight = self._norm_owner.weight - self.rms_eps = self._norm_owner.variance_epsilon - def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: import hpc + owner = self._fallback_op + norm_owner = self._norm_owner return hpc.fuse_ihc_pre( x, - self.weight, - self.hc_scale, - self.hc_base, + owner.hc_fn.weight, + owner.hc_scale, + owner.hc_base, self.norm_eps, self.hc_eps, self.magnitude, - self.rms_weight, - self.rms_eps, + norm_owner.weight if norm_owner is not None else None, + norm_owner.variance_epsilon if norm_owner is not None else 0.0, ) @@ -233,29 +217,20 @@ class HpcIHCHead(HpcModule): self.norm_eps = norm_eps # See HpcIHCPre: keep the owner out of the module tree. object.__setattr__(self, "_fallback_op", fallback_op) - self.weight = None - self.hc_scale = None - self.hc_base = None @classmethod def support(cls, hc_mult: int, hidden_size: int) -> bool: return _ihc_supported(hc_mult, hidden_size) - def process_weights_after_loading(self, model=None): - # hc_head_fn weight is [hc_mult, hc_mult * d], float32. - owner = self._fallback_op - self.weight = owner.hc_head_fn.weight - self.hc_scale = owner.hc_head_scale - self.hc_base = owner.hc_head_base - def forward(self, x: torch.Tensor) -> torch.Tensor: import hpc + owner = self._fallback_op return hpc.fuse_ihc_head( x, - self.weight, - self.hc_scale, - self.hc_base, + owner.hc_head_fn.weight, + owner.hc_head_scale, + owner.hc_head_base, self.norm_eps, self.hc_eps, ) diff --git a/vllm/model_executor/layers/hpc/hpc_module.py b/vllm/model_executor/layers/hpc/hpc_module.py index e1efad832e..c3a0dab676 100644 --- a/vllm/model_executor/layers/hpc/hpc_module.py +++ b/vllm/model_executor/layers/hpc/hpc_module.py @@ -11,8 +11,5 @@ class HpcModule(nn.Module): def support(cls, *args, **kwargs): return True - def process_weights_after_loading(self, model): - pass - def forward(self, *args, **kwargs): pass diff --git a/vllm/model_executor/layers/hpc/rope_norm.py b/vllm/model_executor/layers/hpc/rope_norm.py index c1503c754c..02f06ed89c 100644 --- a/vllm/model_executor/layers/hpc/rope_norm.py +++ b/vllm/model_executor/layers/hpc/rope_norm.py @@ -90,13 +90,13 @@ class HpcRopeNorm(CustomOp, HpcModule): """HPC fused RoPE + QK-Norm + KV-Cache-Write (+ optional FP8 Q quant). Registered as a sub-module in model layers (e.g. HunYuanAttention). - Norm weights are extracted from fallback norm modules via - process_weights_after_loading() after all weights are loaded. + The QK-Norm weights are read directly from the fallback norm modules + (built in float32 when HPC is active), eliminating the derived copy. forward() is dispatched by CustomOp framework: - In compiled mode: forward_cuda() calls torch.ops.vllm.hpc_rope_norm_forward as a splitting point — internal Python control flow is opaque - to torch.compile and not captured by CUDA Graph. + to torch.compile (its kernel launches can still be CUDA-graph captured). - In eager/native mode: forward_native() falls back to forward_cuda(). """ @@ -124,8 +124,6 @@ class HpcRopeNorm(CustomOp, HpcModule): self.num_kv_heads = num_kv_heads self.head_dim = head_dim - self.use_qk_norm = use_qk_norm - self.q_size = num_heads * head_dim self.kv_size = num_kv_heads * head_dim @@ -139,31 +137,18 @@ class HpcRopeNorm(CustomOp, HpcModule): self.head_per_group = num_heads // num_kv_heads - # Pre-allocate norm weight tensors as Parameters so they are tracked by - # CuMemAllocator (for sleep/wake_up) and have stable addresses for CUDA - # Graph replay. process_weights_after_loading() updates them inplace via - # copy_() so refit does not invalidate captured graph tensor pointers. - # Shape is [head_dim] to match the HPC kernel's q/k_norm_weight layout. - if use_qk_norm and fallback_qnorm is not None: - self.qnorm_weight: torch.nn.Parameter | None = torch.nn.Parameter( - torch.empty(head_dim, dtype=torch.float32), - requires_grad=False, - ) - else: - self.qnorm_weight = None - if use_qk_norm and fallback_knorm is not None: - self.knorm_weight: torch.nn.Parameter | None = torch.nn.Parameter( - torch.empty(head_dim, dtype=torch.float32), - requires_grad=False, - ) - else: - self.knorm_weight = None - self.use_fp8 = "fp8" in kv_cache_dtype # The RMSNorm/RoPE ordering is model dependent (e.g. HunYuan V3 applies # QK-Norm before RoPE -> NORM_THEN_ROPE), so it is supplied by the # caller. When QK-Norm is disabled the policy is forced to NONE. self.qk_norm_policy = qk_norm_policy if use_qk_norm else QkNormPolicy.NONE + if self.qk_norm_policy != QkNormPolicy.NONE and ( + fallback_qnorm is None or fallback_knorm is None + ): + raise ValueError( + "QK-Norm is enabled but fallback_qnorm/fallback_knorm were not " + "supplied; the fused kernel reads their weights directly." + ) # Register layer_name + add self to the global instance registry so the # module-level custom op (hpc_rope_norm_forward) can route back here. @@ -215,20 +200,6 @@ class HpcRopeNorm(CustomOp, HpcModule): logger.info_once("enable hpc rope_norm") return True - def process_weights_after_loading(self, model: torch.nn.Module = None) -> None: - """Copy norm weights (float32) from fallback norm modules inplace. - - Uses copy_() to preserve tensor addresses for CUDA Graph / refit - compatibility. Called by the model's load_weights() after all weights - are loaded (and generically from the model loader for DummyModelLoader - / sleep-wake_up reload paths). - """ - if self.use_qk_norm: - if self.fallback_qnorm is not None and self.qnorm_weight is not None: - self.qnorm_weight.data.copy_(self.fallback_qnorm.weight.data.float()) - if self.fallback_knorm is not None and self.knorm_weight is not None: - self.knorm_weight.data.copy_(self.fallback_knorm.weight.data.float()) - def register_layer_name(self, layer_name: str) -> None: """Register layer_name and add self to the global registry. @@ -430,12 +401,14 @@ class HpcRopeNorm(CustomOp, HpcModule): k_scale = attn_layer._k_scale.reshape(1) v_scale = attn_layer._v_scale.reshape(1) - q_norm_weight = ( - self.qnorm_weight if self.qk_norm_policy != QkNormPolicy.NONE else None - ) - k_norm_weight = ( - self.knorm_weight if self.qk_norm_policy != QkNormPolicy.NONE else None - ) + if self.qk_norm_policy != QkNormPolicy.NONE: + assert self.fallback_qnorm is not None + assert self.fallback_knorm is not None + q_norm_weight = self.fallback_qnorm.weight + k_norm_weight = self.fallback_knorm.weight + else: + q_norm_weight = None + k_norm_weight = None # --- Prefill --- if num_prefill_reqs > 0: diff --git a/vllm/model_executor/model_loader/utils.py b/vllm/model_executor/model_loader/utils.py index e12910865e..11e6be5cf8 100644 --- a/vllm/model_executor/model_loader/utils.py +++ b/vllm/model_executor/model_loader/utils.py @@ -15,7 +15,6 @@ import vllm.envs as envs from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config from vllm.logger import init_logger from vllm.model_executor.layers.attention import is_deferred_attention_layer -from vllm.model_executor.layers.hpc import HpcModule from vllm.model_executor.layers.quantization.base_config import ( QuantizationConfig, QuantizeMethodBase, @@ -154,15 +153,6 @@ def process_weights_after_loading( with device_loading_context(module, target_device): module.process_weights_after_loading(model_config.dtype) - # Process HPC modules (HpcRopeNorm, etc.) that rely on - # process_weights_after_loading being called from the model's - # load_weights(). When using DummyModelLoader (e.g. profiling or - # sleep/wake_up reload), the model's load_weights() is not called, so we - # must handle HPC modules here generically. - for _, module in model.named_modules(): - if isinstance(module, HpcModule): - module.process_weights_after_loading(model) - # Model-level post-load hook, after the per-layer quant finalize. if hasattr(model, "process_weights_after_loading"): model.process_weights_after_loading() diff --git a/vllm/model_executor/models/hy_v3.py b/vllm/model_executor/models/hy_v3.py index 0b54c26f47..248ae04747 100644 --- a/vllm/model_executor/models/hy_v3.py +++ b/vllm/model_executor/models/hy_v3.py @@ -300,8 +300,11 @@ class HYV3Attention(nn.Module): prefix=f"{prefix}.attn", ) if self.use_qk_norm: - self.q_norm = RMSNorm(self.head_dim, rms_norm_eps) - self.k_norm = RMSNorm(self.head_dim, rms_norm_eps) + # The HPC fused kernel reads norm weights directly in float32; + # the fallback path keeps the model default dtype. + norm_dtype = torch.float32 if rope_support else None + self.q_norm = RMSNorm(self.head_dim, rms_norm_eps, dtype=norm_dtype) + self.k_norm = RMSNorm(self.head_dim, rms_norm_eps, dtype=norm_dtype) # HPC fused RoPE + QK-Norm + KV-Cache-Write (+ optional FP8 Q quant). # HunYuan V3 applies QK-Norm *before* RoPE, so NORM_THEN_ROPE.