Merge branch 'main' into patches

This commit is contained in:
2026-10-18 16:04:36 +07:00
299 changed files with 51483 additions and 2195 deletions
+19 -1
View File
@@ -79,6 +79,24 @@ def _make_deepseek_v4_vl_weights_mapper(
)
def stream_language_model_first(
weights: Iterable[tuple[str, torch.Tensor]],
) -> Iterable[tuple[str, torch.Tensor]]:
"""Yield ``language_model.*`` weights as they arrive, then the rest.
Keeps the language model's weights in one contiguous group without
materializing the stream. Deferred weights are cloned so they do not keep
the loader's per-shard storage alive until the end of the load.
"""
rest = []
for name, tensor in weights:
if name.startswith("language_model."):
yield name, tensor
else:
rest.append((name, tensor.clone()))
yield from rest
@MULTIMODAL_REGISTRY.register_processor(
DeepseekV4VLMultiModalProcessor,
info=DeepseekV4VLProcessingInfo,
@@ -334,7 +352,7 @@ class DeepseekV4ForConditionalGeneration(
if child_finalizes:
# A child which finalizes inside load_weights must see all of its
# weights in one contiguous delegation from AutoWeightsLoader.
mapped = iter(sorted(mapped, key=lambda x: x[0]))
mapped = stream_language_model_first(mapped)
loader = AutoWeightsLoader(self)
loaded_params = loader.load_weights(mapped)
self._weights_finalized = child_finalizes
+4 -6
View File
@@ -41,6 +41,7 @@ from vllm.models.deepseek_v4.common.vision import (
DeepseekV4ViT,
run_dp_sharded_vision_tower,
)
from vllm.models.deepseek_v4.common.vl_model import stream_language_model_first
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -317,12 +318,9 @@ class DeepseekV41ForCausalLM(
return self.language_model.get_mtp_target_hidden_states()
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
# Map HF names into this wrapper's namespace up front and sort, so
# the "language_model." group reaches the child loader as one
# contiguous block (AutoWeightsLoader delegates per contiguous group,
# and the child's load_weights finalizes fused expert weights, which
# must not run on a partially loaded model).
mapped = sorted(self.hf_to_vllm_mapper.apply(weights), key=lambda x: x[0])
# The child finalizes in load_weights, so its weights must reach it as
# one contiguous group.
mapped = stream_language_model_first(self.hf_to_vllm_mapper.apply(weights))
loader = AutoWeightsLoader(self)
loaded_params = loader.load_weights(mapped)
# The child's load_weights already ran its post-load finalization.
+16 -13
View File
@@ -75,7 +75,7 @@ from vllm.model_executor.layers.quantization import (
resolve_quant_method,
)
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding, get_rope
from vllm.model_executor.utils import replace_parameter
from vllm.model_executor.utils import is_weights_pre_processed, replace_parameter
from vllm.models.common.ops import fused_q_kv_rmsnorm
from vllm.models.kimi_k3.nvidia.low_latency_gemm import try_low_latency_gemm
from vllm.models.kimi_k3.nvidia.ops.fused_mla_key_concat_kv_cache import (
@@ -440,18 +440,21 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
projected into latent space by ``W_UK_T`` and the attention output is
projected back to ``v`` by ``W_UV`` -- avoiding materializing full K/V.
"""
W_UK, W_UV = split_kv_b_proj(
self.kv_b_proj,
act_dtype,
self.kv_lora_rank,
self.num_local_heads,
self.qk_nope_head_dim,
self.v_head_dim,
)
# (L, N, V) -> (N, L, V)
replace_parameter(self, "W_UV", W_UV.transpose(0, 1), prefer_copy=True)
# (L, N, P) -> (N, P, L)
replace_parameter(self, "W_UK_T", W_UK.permute(1, 2, 0), prefer_copy=True)
# The weight cache daemon exports the absorbed weights. Repeating the
# absorption would mutate its shared CUDA allocation.
if not (is_weights_pre_processed() and getattr(self, "W_UV", None) is not None):
W_UK, W_UV = split_kv_b_proj(
self.kv_b_proj,
act_dtype,
self.kv_lora_rank,
self.num_local_heads,
self.qk_nope_head_dim,
self.v_head_dim,
)
# (L, N, V) -> (N, L, V)
replace_parameter(self, "W_UV", W_UV.transpose(0, 1), prefer_copy=True)
# (L, N, P) -> (N, P, L)
replace_parameter(self, "W_UK_T", W_UK.permute(1, 2, 0), prefer_copy=True)
quant_method = (
resolve_quant_method(self.quant_config, self, prefix=self.layer_name)