[PCP][DCP] Support DCP target model with non-DCP Dspark (#56723)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Claude Opus 5
Codex
parent
77871126f9
commit
927c87b347
@@ -21,6 +21,7 @@ import gc
|
||||
import time
|
||||
from contextlib import AbstractContextManager
|
||||
from copy import deepcopy
|
||||
from dataclasses import replace
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
import numpy as np
|
||||
@@ -561,7 +562,16 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
return encoder_runner.get_encoder_timing_stats()
|
||||
|
||||
def get_kv_cache_spec(self):
|
||||
return get_kv_cache_spec(self.vllm_config)
|
||||
kv_cache_spec = get_kv_cache_spec(self.vllm_config)
|
||||
# A draft running at DCP=1 under a DCP target keeps a replicated cache.
|
||||
if (
|
||||
isinstance(self.speculator, DraftModelSpeculator)
|
||||
and self.speculator.dcp_size == 1
|
||||
and self.dcp_size > 1
|
||||
):
|
||||
for name in self.speculator.draft_attn_layer_names & kv_cache_spec.keys():
|
||||
kv_cache_spec[name] = replace(kv_cache_spec[name], dcp_sharded=False)
|
||||
return kv_cache_spec
|
||||
|
||||
def initialize_kv_cache(
|
||||
self,
|
||||
@@ -587,6 +597,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
block_sizes = []
|
||||
max_num_blocks_per_group = []
|
||||
slot_mapping_enabled = []
|
||||
dcp_sharded = []
|
||||
for kv_cache_group in kv_cache_config.kv_cache_groups:
|
||||
spec = kv_cache_group.kv_cache_spec
|
||||
block_sizes.append(spec.block_size)
|
||||
@@ -594,6 +605,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
spec.first_spec if isinstance(spec, UniformTypeKVCacheSpecs) else spec
|
||||
)
|
||||
slot_mapping_enabled.append(layer_spec.uses_slot_mapping)
|
||||
dcp_sharded.append(spec.dcp_sharded)
|
||||
# Let each cache type account for CP. Attention KV is DCP-sharded,
|
||||
# while Mamba/GDN recurrent state is replicated across DCP ranks.
|
||||
max_num_blocks = spec.max_num_blocks_per_req(
|
||||
@@ -657,6 +669,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
device=self.device,
|
||||
kernel_block_sizes=self.kernel_block_sizes,
|
||||
slot_mapping_enabled=slot_mapping_enabled,
|
||||
dcp_sharded=dcp_sharded,
|
||||
cp_size=self.dcp_size,
|
||||
cp_rank=self.dcp_rank,
|
||||
cp_interleave=self.cp_interleave,
|
||||
|
||||
Reference in New Issue
Block a user