[Attention][MLA][DCP] Query replication for MLA decode (DeepSeek-V2/R1 + Kimi-K2.5) (#45964)

Signed-off-by: Sungsoo Ha <sungsooh@nvidia.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
sungsoo ha
2026-07-20 23:51:27 +00:00
committed by GitHub
co-authored by Claude Opus 4.8 Lucas Wilkinson Matthew Bonanni
parent 97a668152b
commit 2396a61108
6 changed files with 157 additions and 15 deletions
+75 -4
View File
@@ -11,6 +11,7 @@ from torch.nn.parameter import Parameter
from typing_extensions import TypeIs
import vllm.envs as envs
from vllm.config import get_current_vllm_config
from vllm.distributed import (
divide,
get_tensor_model_parallel_rank,
@@ -254,6 +255,8 @@ class LinearBase(PluggableLayer):
*,
return_bias: bool = True,
disable_tp: bool = False,
tp_rank: int | None = None,
tp_size: int | None = None,
):
super().__init__()
@@ -277,8 +280,17 @@ class LinearBase(PluggableLayer):
raise ValueError("All linear layers should support quant method.")
self.return_bias = return_bias
self.disable_tp = disable_tp
self.tp_rank = get_tensor_model_parallel_rank() if not disable_tp else 0
self.tp_size = get_tensor_model_parallel_world_size() if not disable_tp else 1
if disable_tp:
self.tp_rank, self.tp_size = 0, 1
else:
self.tp_rank = (
tp_rank if tp_rank is not None else get_tensor_model_parallel_rank()
)
self.tp_size = (
tp_size
if tp_size is not None
else get_tensor_model_parallel_world_size()
)
def update_param_tp_status(self):
# Single source of truth for a parameter's TP state. BasevLLMParameter
@@ -425,6 +437,11 @@ class ColumnParallelLinear(LinearBase):
(e.g. model.layers.0.qkv_proj)
return_bias: If true, return bias together with outputs in forward pass.
disable_tp: If true, weights matrix won't be sharded through tp rank.
tp_rank: Override the tensor-parallel rank used for sharding. Defaults to
the global TP rank. Used to shard at a coarser granularity than one
shard per rank (see ``DCPGroupColumnParallelLinear``).
tp_size: Override the tensor-parallel world size used for sharding.
Defaults to the global TP world size.
"""
# --8<-- [end:column_parallel_linear]
@@ -442,10 +459,21 @@ class ColumnParallelLinear(LinearBase):
*,
return_bias: bool = True,
disable_tp: bool = False,
tp_rank: int | None = None,
tp_size: int | None = None,
):
# Divide the weight matrix along the last dimension.
self.tp_rank = get_tensor_model_parallel_rank() if not disable_tp else 0
self.tp_size = get_tensor_model_parallel_world_size() if not disable_tp else 1
if disable_tp:
self.tp_rank, self.tp_size = 0, 1
else:
self.tp_rank = (
tp_rank if tp_rank is not None else get_tensor_model_parallel_rank()
)
self.tp_size = (
tp_size
if tp_size is not None
else get_tensor_model_parallel_world_size()
)
self.input_size_per_partition = input_size
self.output_size_per_partition = divide(output_size, self.tp_size)
self.output_partition_sizes = [self.output_size_per_partition]
@@ -465,6 +493,8 @@ class ColumnParallelLinear(LinearBase):
prefix,
return_bias=return_bias,
disable_tp=disable_tp,
tp_rank=self.tp_rank,
tp_size=self.tp_size,
)
self._maybe_allow_fp8_block_shape_mismatch()
@@ -586,6 +616,47 @@ class ColumnParallelLinear(LinearBase):
return s
class DCPGroupColumnParallelLinear(ColumnParallelLinear):
"""Column-parallel linear whose weight is sharded across DCP groups.
With Decode Context Parallelism (DCP) the KV cache is sharded across a DCP
group, so MLA decode must attend the group's full head set. This layer shards
its output across DCP *groups* (effective tp size ``tp_size //
dcp_world_size``) rather than across every rank, so each rank in a group
holds the whole group's heads, letting decode skip the query all-gather.
:meth:`forward` returns the group's full head set. :meth:`_local_view`
extracts this rank's TP shard for prefill.
"""
def __init__(self, *args, **kwargs):
dcp_world_size = (
get_current_vllm_config().parallel_config.decode_context_parallel_size
)
rank = get_tensor_model_parallel_rank()
world_size = get_tensor_model_parallel_world_size()
self.group_size = max(dcp_world_size, 1)
self.qrep_active = self.group_size > 1
self.rank_in_group = rank % self.group_size
super().__init__(
*args,
**kwargs,
tp_rank=rank // self.group_size,
tp_size=world_size // self.group_size,
)
def _local_view(self, out: torch.Tensor) -> torch.Tensor:
"""Slice this rank's tp head shard from a group-heads output.
``out`` is head-shaped, i.e. ``(..., group_heads, head_dim)``.
"""
if self.group_size == 1:
return out
n = out.shape[-2] // self.group_size
start = self.rank_in_group * n
return out[..., start : start + n, :].contiguous()
class MergedColumnParallelLinear(ColumnParallelLinear):
"""Packed linear layers with column parallelism.