[PCP][DCP] Support DCP target model with non-DCP Dspark (#56723)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Claude Opus 5
Codex
parent
77871126f9
commit
927c87b347
@@ -158,6 +158,8 @@ class KVCacheSpec:
|
||||
|
||||
# number of tokens in a block
|
||||
block_size: int
|
||||
dcp_sharded: bool = field(default=False, kw_only=True)
|
||||
"""Whether DCP shards this cache's token positions across ranks."""
|
||||
|
||||
block_stride_alignment: int | None = field(default=None, kw_only=True)
|
||||
"""Required byte alignment between physical blocks, including packed layers."""
|
||||
@@ -481,6 +483,7 @@ class HiSparseResidentSpec(KVCacheSpec):
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class AttentionSpec(KVCacheSpec):
|
||||
dcp_sharded: bool = True
|
||||
num_kv_heads: int
|
||||
head_size: int
|
||||
dtype: torch.dtype
|
||||
@@ -536,7 +539,9 @@ class AttentionSpec(KVCacheSpec):
|
||||
|
||||
def max_num_blocks_per_req(self, vllm_config: VllmConfig, max_len: int) -> int:
|
||||
parallel_config = vllm_config.parallel_config
|
||||
kv_shard_count = parallel_config.decode_context_parallel_size
|
||||
kv_shard_count = (
|
||||
parallel_config.decode_context_parallel_size if self.dcp_sharded else 1
|
||||
)
|
||||
return cdiv(max_len, self.block_size * kv_shard_count)
|
||||
|
||||
|
||||
@@ -568,7 +573,7 @@ class FullAttentionSpec(AttentionSpec):
|
||||
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
|
||||
max_model_len = vllm_config.model_config.max_model_len
|
||||
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
|
||||
if dcp_world_size > 1:
|
||||
if self.dcp_sharded and dcp_world_size > 1:
|
||||
max_model_len = cdiv(max_model_len, dcp_world_size)
|
||||
return cdiv(max_model_len, self.block_size) * self.page_size_bytes
|
||||
|
||||
@@ -612,6 +617,7 @@ class FullAttentionSpec(AttentionSpec):
|
||||
head_size_v=specs[0].head_size_v,
|
||||
dtype=specs[0].dtype,
|
||||
kv_quant_mode=specs[0].kv_quant_mode,
|
||||
dcp_sharded=specs[0].dcp_sharded,
|
||||
page_size_padded=specs[0].page_size_padded,
|
||||
num_head_slots=specs[0].num_head_slots,
|
||||
state_content_bytes=specs[0].state_content_bytes,
|
||||
@@ -698,6 +704,7 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
head_size=specs[0].head_size,
|
||||
dtype=specs[0].dtype,
|
||||
kv_quant_mode=specs[0].kv_quant_mode,
|
||||
dcp_sharded=specs[0].dcp_sharded,
|
||||
page_size_padded=specs[0].page_size_padded,
|
||||
num_head_slots=specs[0].num_head_slots,
|
||||
state_content_bytes=specs[0].state_content_bytes,
|
||||
@@ -760,6 +767,7 @@ class RSWASpec(FullAttentionSpec):
|
||||
head_size_v=base.head_size_v,
|
||||
dtype=base.dtype,
|
||||
kv_quant_mode=base.kv_quant_mode,
|
||||
dcp_sharded=base.dcp_sharded,
|
||||
page_size_padded=base.page_size_padded,
|
||||
num_head_slots=base.num_head_slots,
|
||||
state_content_bytes=base.state_content_bytes,
|
||||
@@ -850,9 +858,10 @@ class SlidingWindowSpec(AttentionSpec):
|
||||
return cdiv(num_tokens, self.block_size) + 1
|
||||
|
||||
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
|
||||
assert vllm_config.parallel_config.decode_context_parallel_size == 1, (
|
||||
"DCP not support sliding window."
|
||||
)
|
||||
assert (
|
||||
not self.dcp_sharded
|
||||
or vllm_config.parallel_config.decode_context_parallel_size == 1
|
||||
), "DCP not support sliding window."
|
||||
max_blocks = self.max_admission_blocks_per_request(
|
||||
max_in_flight_tokens=vllm_config.max_in_flight_tokens,
|
||||
max_model_len=vllm_config.model_config.max_model_len,
|
||||
@@ -950,6 +959,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
extra_retained_set = set(spec.extra_retained_tokens for spec in specs)
|
||||
bounded_replay_set = set(spec.bounded_replay for spec in specs)
|
||||
block_stride_alignment_set = {spec.block_stride_alignment for spec in specs}
|
||||
assert len({spec.dcp_sharded for spec in specs}) == 1
|
||||
assert (
|
||||
len(cache_dtype_str_set) == 1
|
||||
and len(tokens_per_state_set) == 1
|
||||
@@ -969,6 +979,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
num_kv_heads=specs[0].num_kv_heads,
|
||||
head_size=specs[0].head_size,
|
||||
dtype=specs[0].dtype,
|
||||
dcp_sharded=specs[0].dcp_sharded,
|
||||
page_size_padded=specs[0].page_size_padded,
|
||||
num_head_slots=specs[0].num_head_slots,
|
||||
state_content_bytes=specs[0].state_content_bytes,
|
||||
@@ -1185,6 +1196,7 @@ class SinkFullAttentionSpec(FullAttentionSpec):
|
||||
block_stride_alignment=specs[0].block_stride_alignment,
|
||||
dtype=specs[0].dtype,
|
||||
kv_quant_mode=specs[0].kv_quant_mode,
|
||||
dcp_sharded=specs[0].dcp_sharded,
|
||||
page_size_padded=specs[0].page_size_padded,
|
||||
num_head_slots=specs[0].num_head_slots,
|
||||
state_content_bytes=specs[0].state_content_bytes,
|
||||
@@ -1217,6 +1229,11 @@ class UniformTypeKVCacheSpecs(KVCacheSpec):
|
||||
|
||||
kv_cache_specs: dict[str, KVCacheSpec]
|
||||
|
||||
def __post_init__(self):
|
||||
super().__post_init__()
|
||||
if self.kv_cache_specs:
|
||||
object.__setattr__(self, "dcp_sharded", self.first_spec.dcp_sharded)
|
||||
|
||||
@property
|
||||
def prefix_cacheable(self) -> bool:
|
||||
return all(spec.prefix_cacheable for spec in self.kv_cache_specs.values())
|
||||
@@ -1270,6 +1287,8 @@ class UniformTypeKVCacheSpecs(KVCacheSpec):
|
||||
if len(block_sizes) > 1:
|
||||
# Different block sizes, not uniform.
|
||||
return False
|
||||
if len({spec.dcp_sharded for spec in kv_cache_specs.values()}) > 1:
|
||||
return False
|
||||
first_spec = next(iter(kv_cache_specs.values()))
|
||||
return first_spec.is_uniform_with_collection(kv_cache_specs)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user