[Nixl][PD] DCP support for MLA models (#50611)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
@@ -3157,9 +3157,6 @@ class MLACommonImpl(MLACommonBaseImpl[M], Generic[M]):
|
||||
parallel_config = get_current_vllm_config().parallel_config
|
||||
# Avoid requiring an initialized DCP group in tests.
|
||||
self.dcp_world_size: int = parallel_config.decode_context_parallel_size
|
||||
self.cp_kv_cache_interleave_size: int = (
|
||||
parallel_config.cp_kv_cache_interleave_size
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
def forward_mqa(
|
||||
|
||||
@@ -771,14 +771,30 @@ class SparseAttnIndexer(CustomOp):
|
||||
parallel_config = get_current_vllm_config().parallel_config
|
||||
self.dcp_world_size = parallel_config.decode_context_parallel_size
|
||||
self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_world_size > 1 else 0
|
||||
self.cp_kv_cache_interleave_size = parallel_config.cp_kv_cache_interleave_size
|
||||
self.use_pcp = parallel_config.prefill_context_parallel_size > 1
|
||||
self._cp_kv_cache_interleave_size: int | None = None
|
||||
if current_platform.is_cuda() and not has_deep_gemm():
|
||||
raise RuntimeError(
|
||||
"Sparse Attention Indexer CUDA op requires DeepGEMM support in "
|
||||
"the current vLLM environment."
|
||||
)
|
||||
|
||||
@property
|
||||
def cp_kv_cache_interleave_size(self) -> int:
|
||||
"""With PD+DCP, the real value isn't known until block_size is finalized,
|
||||
which happens after this layer is built. Safe to cache after the first access,
|
||||
as long as the adjustment always runs before any forward pass
|
||||
(it's set up in Worker.initialize_from_config, ahead of warmup/serving).
|
||||
"""
|
||||
if self._cp_kv_cache_interleave_size is None:
|
||||
value = (
|
||||
get_current_vllm_config().parallel_config.cp_kv_cache_interleave_size
|
||||
)
|
||||
if isinstance(get_forward_context().attn_metadata, dict):
|
||||
self._cp_kv_cache_interleave_size = value
|
||||
return value
|
||||
return self._cp_kv_cache_interleave_size
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user