[Bugfix][KV Offload] Compact canonical MLA rows (#56799)
Signed-off-by: Itay Etelis <92247226+Etelis@users.noreply.github.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
@@ -233,6 +233,20 @@ def build_offloading_config(
|
||||
and parallel_config.world_size == tp_size
|
||||
)
|
||||
|
||||
if canonical_layout and is_parallelism_agnostic:
|
||||
replicated_layout = (
|
||||
all(
|
||||
type(spec) is MLAAttentionSpec
|
||||
for _, group in selected_groups
|
||||
for spec in iter_layer_specs(group.kv_cache_spec)
|
||||
)
|
||||
and parallel_config.nnodes_within_dp == 1
|
||||
and (
|
||||
parallel_config.world_size == 1
|
||||
or parallel_config.distributed_executor_backend == "mp"
|
||||
)
|
||||
)
|
||||
|
||||
kv_events_config = vllm_config.kv_events_config
|
||||
cache_dtype = (
|
||||
vllm_config.model_config.dtype
|
||||
|
||||
@@ -53,7 +53,9 @@ class OffloadingConnectorWorker:
|
||||
self.worker: OffloadingWorker | None = None
|
||||
# Non-writers still ack: pending_count waits for world_size per job.
|
||||
self._is_store_writer = (
|
||||
not self.spec.replicated_layout or self.spec.config.parallel.rank == 0
|
||||
not self.spec.replicated_layout
|
||||
or self.spec.config.canonical_layout
|
||||
or self.spec.config.parallel.rank == 0
|
||||
)
|
||||
|
||||
# job_id -> req_id for in-flight loads.
|
||||
|
||||
Reference in New Issue
Block a user