[KV Offload] Deduplicate replicated MLA KV in the shared CPU region (#48906)

Signed-off-by: Change72 <changg@nvidia.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Chang Guo
2026-07-26 08:22:30 +03:00
committed by GitHub
co-authored by OpenAI Codex
parent 7a29a3c54c
commit 7eca0e1a64
14 changed files with 1143 additions and 506 deletions
@@ -102,19 +102,39 @@ def build_offloading_config(
)
worker_kv_bytes_per_block = total_gpu_kv_bytes // kv_cache_config.num_blocks
# Only a single non-MLA full-attention group is parallelism-invariant:
# MLA latent KV is replicated per rank (never head-sharded), and the V2
# model runner's KV layout is not known to be parallelism-invariant.
single_group = (
single_group_spec = (
kv_cache_config.kv_cache_groups[0].kv_cache_spec
if len(kv_cache_config.kv_cache_groups) == 1
else None
)
replicated_layout = (
vllm_config.model_config.use_mla
# Exact type: fail closed on wrappers and sliding-window variants.
and type(single_group_spec) is MLAAttentionSpec
# Page accounting: one MLA page per layer, no packed/mixed rows.
and worker_kv_bytes_per_block > 0
and worker_kv_bytes_per_block
== single_group_spec.page_size_bytes
* len(kv_cache_config.kv_cache_groups[0].layer_names)
# Safe MVP boundary: TP-only, no other parallel axes.
and parallel_config.tensor_parallel_size > 1
and parallel_config.pipeline_parallel_size == 1
and parallel_config.prefill_context_parallel_size == 1
and parallel_config.decode_context_parallel_size == 1
and parallel_config.world_size == parallel_config.tensor_parallel_size
# Shared /dev/shm mmap layout is single-node mp only.
and parallel_config.distributed_executor_backend == "mp"
and parallel_config.nnodes_within_dp == 1
)
# Only a single non-MLA full-attention group is parallelism-invariant:
# MLA latent KV is replicated per rank (never head-sharded), and the V2
# model runner's KV layout is not known to be parallelism-invariant.
is_parallelism_agnostic = (
not vllm_config.use_v2_model_runner
and single_group is not None
and isinstance(single_group, FullAttentionSpec)
and not isinstance(single_group, MLAAttentionSpec)
and single_group_spec is not None
and isinstance(single_group_spec, FullAttentionSpec)
and not isinstance(single_group_spec, MLAAttentionSpec)
)
kv_events_config = vllm_config.kv_events_config
@@ -144,4 +164,5 @@ def build_offloading_config(
data_parallel_index=parallel_config.data_parallel_index,
is_parallelism_agnostic=is_parallelism_agnostic,
),
replicated_layout=replicated_layout,
)
@@ -45,6 +45,10 @@ class OffloadingConnectorWorker:
self.spec = spec
self.kv_cache_config = kv_cache_config
self.worker: OffloadingWorker | None = None
# Non-writers still ack: pending_count waits for world_size per job.
self._is_store_writer = (
not self.spec.replicated_layout or self.spec.config.parallel.rank == 0
)
# job_id -> req_id for in-flight loads.
self._load_jobs: dict[int, ReqId] = {}
@@ -274,6 +278,9 @@ class OffloadingConnectorWorker:
for job_id in kv_connector_metadata.jobs_to_flush:
entry = kv_connector_metadata.store_jobs.pop(job_id, None)
if entry is not None:
if not self._is_store_writer:
self._connector_worker_meta.mark_completed(job_id)
continue
assert isinstance(entry.src_spec, GPULoadStoreSpec)
self._unsubmitted_store_jobs.append(
(job_id, entry.src_spec, entry.dst_spec)
@@ -304,6 +311,10 @@ class OffloadingConnectorWorker:
def prepare_store_kv(self, metadata: OffloadingConnectorMetadata):
for job_id, entry in metadata.store_jobs.items():
if not self._is_store_writer:
# Gate before queueing: no _unsubmitted_store_jobs entry.
self._connector_worker_meta.mark_completed(job_id)
continue
# NOTE(orozery): defer the store to the beginning of the next
# engine step, so that offloading starts AFTER transfers related
# to token sampling, thereby avoiding delays to token generation.