[KV Offload] Deduplicate replicated MLA KV in the shared CPU region (#48906)
Signed-off-by: Change72 <changg@nvidia.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
7a29a3c54c
commit
7eca0e1a64
@@ -102,19 +102,39 @@ def build_offloading_config(
|
||||
)
|
||||
worker_kv_bytes_per_block = total_gpu_kv_bytes // kv_cache_config.num_blocks
|
||||
|
||||
# Only a single non-MLA full-attention group is parallelism-invariant:
|
||||
# MLA latent KV is replicated per rank (never head-sharded), and the V2
|
||||
# model runner's KV layout is not known to be parallelism-invariant.
|
||||
single_group = (
|
||||
single_group_spec = (
|
||||
kv_cache_config.kv_cache_groups[0].kv_cache_spec
|
||||
if len(kv_cache_config.kv_cache_groups) == 1
|
||||
else None
|
||||
)
|
||||
replicated_layout = (
|
||||
vllm_config.model_config.use_mla
|
||||
# Exact type: fail closed on wrappers and sliding-window variants.
|
||||
and type(single_group_spec) is MLAAttentionSpec
|
||||
# Page accounting: one MLA page per layer, no packed/mixed rows.
|
||||
and worker_kv_bytes_per_block > 0
|
||||
and worker_kv_bytes_per_block
|
||||
== single_group_spec.page_size_bytes
|
||||
* len(kv_cache_config.kv_cache_groups[0].layer_names)
|
||||
# Safe MVP boundary: TP-only, no other parallel axes.
|
||||
and parallel_config.tensor_parallel_size > 1
|
||||
and parallel_config.pipeline_parallel_size == 1
|
||||
and parallel_config.prefill_context_parallel_size == 1
|
||||
and parallel_config.decode_context_parallel_size == 1
|
||||
and parallel_config.world_size == parallel_config.tensor_parallel_size
|
||||
# Shared /dev/shm mmap layout is single-node mp only.
|
||||
and parallel_config.distributed_executor_backend == "mp"
|
||||
and parallel_config.nnodes_within_dp == 1
|
||||
)
|
||||
|
||||
# Only a single non-MLA full-attention group is parallelism-invariant:
|
||||
# MLA latent KV is replicated per rank (never head-sharded), and the V2
|
||||
# model runner's KV layout is not known to be parallelism-invariant.
|
||||
is_parallelism_agnostic = (
|
||||
not vllm_config.use_v2_model_runner
|
||||
and single_group is not None
|
||||
and isinstance(single_group, FullAttentionSpec)
|
||||
and not isinstance(single_group, MLAAttentionSpec)
|
||||
and single_group_spec is not None
|
||||
and isinstance(single_group_spec, FullAttentionSpec)
|
||||
and not isinstance(single_group_spec, MLAAttentionSpec)
|
||||
)
|
||||
|
||||
kv_events_config = vllm_config.kv_events_config
|
||||
@@ -144,4 +164,5 @@ def build_offloading_config(
|
||||
data_parallel_index=parallel_config.data_parallel_index,
|
||||
is_parallelism_agnostic=is_parallelism_agnostic,
|
||||
),
|
||||
replicated_layout=replicated_layout,
|
||||
)
|
||||
|
||||
@@ -45,6 +45,10 @@ class OffloadingConnectorWorker:
|
||||
self.spec = spec
|
||||
self.kv_cache_config = kv_cache_config
|
||||
self.worker: OffloadingWorker | None = None
|
||||
# Non-writers still ack: pending_count waits for world_size per job.
|
||||
self._is_store_writer = (
|
||||
not self.spec.replicated_layout or self.spec.config.parallel.rank == 0
|
||||
)
|
||||
|
||||
# job_id -> req_id for in-flight loads.
|
||||
self._load_jobs: dict[int, ReqId] = {}
|
||||
@@ -274,6 +278,9 @@ class OffloadingConnectorWorker:
|
||||
for job_id in kv_connector_metadata.jobs_to_flush:
|
||||
entry = kv_connector_metadata.store_jobs.pop(job_id, None)
|
||||
if entry is not None:
|
||||
if not self._is_store_writer:
|
||||
self._connector_worker_meta.mark_completed(job_id)
|
||||
continue
|
||||
assert isinstance(entry.src_spec, GPULoadStoreSpec)
|
||||
self._unsubmitted_store_jobs.append(
|
||||
(job_id, entry.src_spec, entry.dst_spec)
|
||||
@@ -304,6 +311,10 @@ class OffloadingConnectorWorker:
|
||||
|
||||
def prepare_store_kv(self, metadata: OffloadingConnectorMetadata):
|
||||
for job_id, entry in metadata.store_jobs.items():
|
||||
if not self._is_store_writer:
|
||||
# Gate before queueing: no _unsubmitted_store_jobs entry.
|
||||
self._connector_worker_meta.mark_completed(job_id)
|
||||
continue
|
||||
# NOTE(orozery): defer the store to the beginning of the next
|
||||
# engine step, so that offloading starts AFTER transfers related
|
||||
# to token sampling, thereby avoiding delays to token generation.
|
||||
|
||||
Reference in New Issue
Block a user