[Bugfix][Attention] Preserve post-load tensors across weight reloads (#48251)
Signed-off-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
@@ -240,6 +240,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8StaticTensorSym,
|
||||
kNvfp4Dynamic,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
from vllm.utils.math_utils import cdiv, round_down
|
||||
@@ -989,9 +990,9 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
)
|
||||
else:
|
||||
# Convert from (L, N, V) to (N, L, V)
|
||||
self.W_UV = W_UV.transpose(0, 1)
|
||||
replace_parameter(self, "W_UV", W_UV.transpose(0, 1), prefer_copy=True)
|
||||
# Convert from (L, N, P) to (N, P, L)
|
||||
self.W_UK_T = W_UK.permute(1, 2, 0)
|
||||
replace_parameter(self, "W_UK_T", W_UK.permute(1, 2, 0), prefer_copy=True)
|
||||
|
||||
# If we should not load quant weights, we initialize the scales to 1.0
|
||||
# as the default value. See [Note: Register q/k/v/prob scales in state dict]
|
||||
|
||||
@@ -1573,6 +1573,8 @@ class FlashInferImpl(AttentionImpl):
|
||||
)
|
||||
|
||||
self.sinks: torch.Tensor | None = None
|
||||
# Keep the source so RL weight updates can refresh the runtime tensor.
|
||||
self._sinks_source = sinks
|
||||
if sinks is not None:
|
||||
if sinks.shape[0] != num_heads:
|
||||
raise ValueError(
|
||||
@@ -1633,8 +1635,15 @@ class FlashInferImpl(AttentionImpl):
|
||||
|
||||
# FlashInfer requires attention sinks to be float32
|
||||
def process_weights_after_loading(self, act_dtype: torch.dtype):
|
||||
if self.sinks is not None and self.sinks.dtype != torch.float32:
|
||||
self.sinks = self.sinks.to(torch.float32)
|
||||
source_sinks = self._sinks_source
|
||||
if source_sinks is None:
|
||||
return
|
||||
if source_sinks.dtype == torch.float32:
|
||||
self.sinks = source_sinks
|
||||
elif self.sinks is None or self.sinks.dtype != torch.float32:
|
||||
self.sinks = source_sinks.to(torch.float32)
|
||||
else:
|
||||
self.sinks.copy_(source_sinks)
|
||||
|
||||
def get_xqa_bmm1_scale(self, layer: torch.nn.Module, q_data_type: torch.dtype):
|
||||
bmm1_scale = self.scale
|
||||
|
||||
Reference in New Issue
Block a user