[Attention][MLA][DCP] Query replication for MLA decode (DeepSeek-V2/R1 + Kimi-K2.5) (#45964)
Signed-off-by: Sungsoo Ha <sungsooh@nvidia.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
Lucas Wilkinson
Matthew Bonanni
parent
97a668152b
commit
2396a61108
@@ -11,6 +11,7 @@ from torch.nn.parameter import Parameter
|
||||
from typing_extensions import TypeIs
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
divide,
|
||||
get_tensor_model_parallel_rank,
|
||||
@@ -254,6 +255,8 @@ class LinearBase(PluggableLayer):
|
||||
*,
|
||||
return_bias: bool = True,
|
||||
disable_tp: bool = False,
|
||||
tp_rank: int | None = None,
|
||||
tp_size: int | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
@@ -277,8 +280,17 @@ class LinearBase(PluggableLayer):
|
||||
raise ValueError("All linear layers should support quant method.")
|
||||
self.return_bias = return_bias
|
||||
self.disable_tp = disable_tp
|
||||
self.tp_rank = get_tensor_model_parallel_rank() if not disable_tp else 0
|
||||
self.tp_size = get_tensor_model_parallel_world_size() if not disable_tp else 1
|
||||
if disable_tp:
|
||||
self.tp_rank, self.tp_size = 0, 1
|
||||
else:
|
||||
self.tp_rank = (
|
||||
tp_rank if tp_rank is not None else get_tensor_model_parallel_rank()
|
||||
)
|
||||
self.tp_size = (
|
||||
tp_size
|
||||
if tp_size is not None
|
||||
else get_tensor_model_parallel_world_size()
|
||||
)
|
||||
|
||||
def update_param_tp_status(self):
|
||||
# Single source of truth for a parameter's TP state. BasevLLMParameter
|
||||
@@ -425,6 +437,11 @@ class ColumnParallelLinear(LinearBase):
|
||||
(e.g. model.layers.0.qkv_proj)
|
||||
return_bias: If true, return bias together with outputs in forward pass.
|
||||
disable_tp: If true, weights matrix won't be sharded through tp rank.
|
||||
tp_rank: Override the tensor-parallel rank used for sharding. Defaults to
|
||||
the global TP rank. Used to shard at a coarser granularity than one
|
||||
shard per rank (see ``DCPGroupColumnParallelLinear``).
|
||||
tp_size: Override the tensor-parallel world size used for sharding.
|
||||
Defaults to the global TP world size.
|
||||
"""
|
||||
|
||||
# --8<-- [end:column_parallel_linear]
|
||||
@@ -442,10 +459,21 @@ class ColumnParallelLinear(LinearBase):
|
||||
*,
|
||||
return_bias: bool = True,
|
||||
disable_tp: bool = False,
|
||||
tp_rank: int | None = None,
|
||||
tp_size: int | None = None,
|
||||
):
|
||||
# Divide the weight matrix along the last dimension.
|
||||
self.tp_rank = get_tensor_model_parallel_rank() if not disable_tp else 0
|
||||
self.tp_size = get_tensor_model_parallel_world_size() if not disable_tp else 1
|
||||
if disable_tp:
|
||||
self.tp_rank, self.tp_size = 0, 1
|
||||
else:
|
||||
self.tp_rank = (
|
||||
tp_rank if tp_rank is not None else get_tensor_model_parallel_rank()
|
||||
)
|
||||
self.tp_size = (
|
||||
tp_size
|
||||
if tp_size is not None
|
||||
else get_tensor_model_parallel_world_size()
|
||||
)
|
||||
self.input_size_per_partition = input_size
|
||||
self.output_size_per_partition = divide(output_size, self.tp_size)
|
||||
self.output_partition_sizes = [self.output_size_per_partition]
|
||||
@@ -465,6 +493,8 @@ class ColumnParallelLinear(LinearBase):
|
||||
prefix,
|
||||
return_bias=return_bias,
|
||||
disable_tp=disable_tp,
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.tp_size,
|
||||
)
|
||||
|
||||
self._maybe_allow_fp8_block_shape_mismatch()
|
||||
@@ -586,6 +616,47 @@ class ColumnParallelLinear(LinearBase):
|
||||
return s
|
||||
|
||||
|
||||
class DCPGroupColumnParallelLinear(ColumnParallelLinear):
|
||||
"""Column-parallel linear whose weight is sharded across DCP groups.
|
||||
|
||||
With Decode Context Parallelism (DCP) the KV cache is sharded across a DCP
|
||||
group, so MLA decode must attend the group's full head set. This layer shards
|
||||
its output across DCP *groups* (effective tp size ``tp_size //
|
||||
dcp_world_size``) rather than across every rank, so each rank in a group
|
||||
holds the whole group's heads, letting decode skip the query all-gather.
|
||||
|
||||
:meth:`forward` returns the group's full head set. :meth:`_local_view`
|
||||
extracts this rank's TP shard for prefill.
|
||||
"""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
dcp_world_size = (
|
||||
get_current_vllm_config().parallel_config.decode_context_parallel_size
|
||||
)
|
||||
rank = get_tensor_model_parallel_rank()
|
||||
world_size = get_tensor_model_parallel_world_size()
|
||||
self.group_size = max(dcp_world_size, 1)
|
||||
self.qrep_active = self.group_size > 1
|
||||
self.rank_in_group = rank % self.group_size
|
||||
super().__init__(
|
||||
*args,
|
||||
**kwargs,
|
||||
tp_rank=rank // self.group_size,
|
||||
tp_size=world_size // self.group_size,
|
||||
)
|
||||
|
||||
def _local_view(self, out: torch.Tensor) -> torch.Tensor:
|
||||
"""Slice this rank's tp head shard from a group-heads output.
|
||||
|
||||
``out`` is head-shaped, i.e. ``(..., group_heads, head_dim)``.
|
||||
"""
|
||||
if self.group_size == 1:
|
||||
return out
|
||||
n = out.shape[-2] // self.group_size
|
||||
start = self.rank_in_group * n
|
||||
return out[..., start : start + n, :].contiguous()
|
||||
|
||||
|
||||
class MergedColumnParallelLinear(ColumnParallelLinear):
|
||||
"""Packed linear layers with column parallelism.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user