[Bugfix][Attention] Preserve post-load tensors across weight reloads (#48251)
Signed-off-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
@@ -240,6 +240,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8StaticTensorSym,
|
||||
kNvfp4Dynamic,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
from vllm.utils.math_utils import cdiv, round_down
|
||||
@@ -989,9 +990,9 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
)
|
||||
else:
|
||||
# Convert from (L, N, V) to (N, L, V)
|
||||
self.W_UV = W_UV.transpose(0, 1)
|
||||
replace_parameter(self, "W_UV", W_UV.transpose(0, 1), prefer_copy=True)
|
||||
# Convert from (L, N, P) to (N, P, L)
|
||||
self.W_UK_T = W_UK.permute(1, 2, 0)
|
||||
replace_parameter(self, "W_UK_T", W_UK.permute(1, 2, 0), prefer_copy=True)
|
||||
|
||||
# If we should not load quant weights, we initialize the scales to 1.0
|
||||
# as the default value. See [Note: Register q/k/v/prob scales in state dict]
|
||||
|
||||
Reference in New Issue
Block a user