[KV Offload] Deduplicate replicated MLA KV in the shared CPU region (#48906)

Signed-off-by: Change72 <changg@nvidia.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Chang Guo
2026-07-26 08:22:30 +03:00
committed by GitHub
co-authored by OpenAI Codex
parent 7a29a3c54c
commit 7eca0e1a64
14 changed files with 1143 additions and 506 deletions
+25 -5
View File
@@ -16,11 +16,13 @@ KV drops accuracy below threshold. The reload itself is not asserted, so
a silently skipped reload (e.g. offloading disabled) would not be flagged.
Covers both KV offloading connectors (OffloadingConnector and
SimpleCPUOffloadConnector) across four architecture families:
SimpleCPUOffloadConnector) across five architecture families:
- Hybrid Mamba (NemotronH: attention + Mamba)
- Heterogeneous head dim (Gemma 4)
- Hybrid GDN (Qwen 3.5: attention + GatedDeltaNet)
- Compressed attention (DeepSeek-V4-Flash: CSA)
- Pure MLA (DeepSeek-V2-Lite: TieringOffloadingSpec, TP=2, replicated
single-slot host layout)
Usage:
pytest -s -v evals/gsm8k/test_gsm8k_offloading.py
@@ -47,14 +49,16 @@ NUM_FEWSHOT = 5
_OFFLOAD_SYNC_TIMEOUT = 60
def _kv_transfer_config(connector: str, cpu_gib: int = 4) -> str:
def _kv_transfer_config(
connector: str, cpu_gib: int = 4, spec_name: str = "CPUOffloadingSpec"
) -> str:
if connector == "OffloadingConnector":
return json.dumps(
{
"kv_connector": "OffloadingConnector",
"kv_role": "kv_both",
"kv_connector_extra_config": {
"spec_name": "CPUOffloadingSpec",
"spec_name": spec_name,
"cpu_bytes_to_use": cpu_gib << 30,
"eviction_policy": "lru",
},
@@ -115,8 +119,10 @@ class OffloadingModelConfig:
accuracy_threshold: float
tolerance: float = 0.05
extra_server_args: list[str] = field(default_factory=list)
env_dict: dict[str, str] = field(default_factory=dict)
cpu_offload_gib: int = 4
startup_timeout: int = 600
spec_name: str = "CPUOffloadingSpec"
MODELS = [
@@ -165,6 +171,20 @@ MODELS = [
cpu_offload_gib=16,
startup_timeout=1200,
),
OffloadingModelConfig(
id="offloading-deepseek-v2-lite-tiering-tp2",
model="deepseek-ai/DeepSeek-V2-Lite",
connector="OffloadingConnector",
# Baseline 0.360/0.345 over two runs (measured on 2xA100).
accuracy_threshold=0.35,
extra_server_args=[
"--tensor-parallel-size",
"2",
],
cpu_offload_gib=8,
startup_timeout=1200,
spec_name="TieringOffloadingSpec",
),
# ── SimpleCPUOffloadConnector ────────────────────────────────────
OffloadingModelConfig(
id="simple-nemotron-h-8b",
@@ -229,7 +249,7 @@ def test_gsm8k_offloading_correctness(cfg: OffloadingModelConfig):
"--enable-prefix-caching",
"--no-disable-hybrid-kv-cache-manager",
"--kv-transfer-config",
_kv_transfer_config(cfg.connector, cfg.cpu_offload_gib),
_kv_transfer_config(cfg.connector, cfg.cpu_offload_gib, cfg.spec_name),
"--trust-remote-code",
"--disable-uvicorn-access-log",
*cfg.extra_server_args,
@@ -239,7 +259,7 @@ def test_gsm8k_offloading_correctness(cfg: OffloadingModelConfig):
cfg.model,
server_args,
# /reset_prefix_cache requires dev mode.
env_dict={"VLLM_SERVER_DEV_MODE": "1"},
env_dict={"VLLM_SERVER_DEV_MODE": "1", **cfg.env_dict},
max_wait_seconds=cfg.startup_timeout,
) as server:
base_url = f"http://{server.host}:{server.port}"