Merge branch 'main' into patches
This commit is contained in:
@@ -79,6 +79,24 @@ def _make_deepseek_v4_vl_weights_mapper(
|
||||
)
|
||||
|
||||
|
||||
def stream_language_model_first(
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
"""Yield ``language_model.*`` weights as they arrive, then the rest.
|
||||
|
||||
Keeps the language model's weights in one contiguous group without
|
||||
materializing the stream. Deferred weights are cloned so they do not keep
|
||||
the loader's per-shard storage alive until the end of the load.
|
||||
"""
|
||||
rest = []
|
||||
for name, tensor in weights:
|
||||
if name.startswith("language_model."):
|
||||
yield name, tensor
|
||||
else:
|
||||
rest.append((name, tensor.clone()))
|
||||
yield from rest
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
DeepseekV4VLMultiModalProcessor,
|
||||
info=DeepseekV4VLProcessingInfo,
|
||||
@@ -334,7 +352,7 @@ class DeepseekV4ForConditionalGeneration(
|
||||
if child_finalizes:
|
||||
# A child which finalizes inside load_weights must see all of its
|
||||
# weights in one contiguous delegation from AutoWeightsLoader.
|
||||
mapped = iter(sorted(mapped, key=lambda x: x[0]))
|
||||
mapped = stream_language_model_first(mapped)
|
||||
loader = AutoWeightsLoader(self)
|
||||
loaded_params = loader.load_weights(mapped)
|
||||
self._weights_finalized = child_finalizes
|
||||
|
||||
@@ -41,6 +41,7 @@ from vllm.models.deepseek_v4.common.vision import (
|
||||
DeepseekV4ViT,
|
||||
run_dp_sharded_vision_tower,
|
||||
)
|
||||
from vllm.models.deepseek_v4.common.vl_model import stream_language_model_first
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
@@ -317,12 +318,9 @@ class DeepseekV41ForCausalLM(
|
||||
return self.language_model.get_mtp_target_hidden_states()
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
# Map HF names into this wrapper's namespace up front and sort, so
|
||||
# the "language_model." group reaches the child loader as one
|
||||
# contiguous block (AutoWeightsLoader delegates per contiguous group,
|
||||
# and the child's load_weights finalizes fused expert weights, which
|
||||
# must not run on a partially loaded model).
|
||||
mapped = sorted(self.hf_to_vllm_mapper.apply(weights), key=lambda x: x[0])
|
||||
# The child finalizes in load_weights, so its weights must reach it as
|
||||
# one contiguous group.
|
||||
mapped = stream_language_model_first(self.hf_to_vllm_mapper.apply(weights))
|
||||
loader = AutoWeightsLoader(self)
|
||||
loaded_params = loader.load_weights(mapped)
|
||||
# The child's load_weights already ran its post-load finalization.
|
||||
|
||||
@@ -75,7 +75,7 @@ from vllm.model_executor.layers.quantization import (
|
||||
resolve_quant_method,
|
||||
)
|
||||
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding, get_rope
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.model_executor.utils import is_weights_pre_processed, replace_parameter
|
||||
from vllm.models.common.ops import fused_q_kv_rmsnorm
|
||||
from vllm.models.kimi_k3.nvidia.low_latency_gemm import try_low_latency_gemm
|
||||
from vllm.models.kimi_k3.nvidia.ops.fused_mla_key_concat_kv_cache import (
|
||||
@@ -440,18 +440,21 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
|
||||
projected into latent space by ``W_UK_T`` and the attention output is
|
||||
projected back to ``v`` by ``W_UV`` -- avoiding materializing full K/V.
|
||||
"""
|
||||
W_UK, W_UV = split_kv_b_proj(
|
||||
self.kv_b_proj,
|
||||
act_dtype,
|
||||
self.kv_lora_rank,
|
||||
self.num_local_heads,
|
||||
self.qk_nope_head_dim,
|
||||
self.v_head_dim,
|
||||
)
|
||||
# (L, N, V) -> (N, L, V)
|
||||
replace_parameter(self, "W_UV", W_UV.transpose(0, 1), prefer_copy=True)
|
||||
# (L, N, P) -> (N, P, L)
|
||||
replace_parameter(self, "W_UK_T", W_UK.permute(1, 2, 0), prefer_copy=True)
|
||||
# The weight cache daemon exports the absorbed weights. Repeating the
|
||||
# absorption would mutate its shared CUDA allocation.
|
||||
if not (is_weights_pre_processed() and getattr(self, "W_UV", None) is not None):
|
||||
W_UK, W_UV = split_kv_b_proj(
|
||||
self.kv_b_proj,
|
||||
act_dtype,
|
||||
self.kv_lora_rank,
|
||||
self.num_local_heads,
|
||||
self.qk_nope_head_dim,
|
||||
self.v_head_dim,
|
||||
)
|
||||
# (L, N, V) -> (N, L, V)
|
||||
replace_parameter(self, "W_UV", W_UV.transpose(0, 1), prefer_copy=True)
|
||||
# (L, N, P) -> (N, P, L)
|
||||
replace_parameter(self, "W_UK_T", W_UK.permute(1, 2, 0), prefer_copy=True)
|
||||
|
||||
quant_method = (
|
||||
resolve_quant_method(self.quant_config, self, prefix=self.layer_name)
|
||||
|
||||
Reference in New Issue
Block a user