[KV Offload] Deduplicate replicated MLA KV in the shared CPU region (#48906)
Signed-off-by: Change72 <changg@nvidia.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
7a29a3c54c
commit
7eca0e1a64
@@ -16,11 +16,13 @@ KV drops accuracy below threshold. The reload itself is not asserted, so
|
||||
a silently skipped reload (e.g. offloading disabled) would not be flagged.
|
||||
|
||||
Covers both KV offloading connectors (OffloadingConnector and
|
||||
SimpleCPUOffloadConnector) across four architecture families:
|
||||
SimpleCPUOffloadConnector) across five architecture families:
|
||||
- Hybrid Mamba (NemotronH: attention + Mamba)
|
||||
- Heterogeneous head dim (Gemma 4)
|
||||
- Hybrid GDN (Qwen 3.5: attention + GatedDeltaNet)
|
||||
- Compressed attention (DeepSeek-V4-Flash: CSA)
|
||||
- Pure MLA (DeepSeek-V2-Lite: TieringOffloadingSpec, TP=2, replicated
|
||||
single-slot host layout)
|
||||
|
||||
Usage:
|
||||
pytest -s -v evals/gsm8k/test_gsm8k_offloading.py
|
||||
@@ -47,14 +49,16 @@ NUM_FEWSHOT = 5
|
||||
_OFFLOAD_SYNC_TIMEOUT = 60
|
||||
|
||||
|
||||
def _kv_transfer_config(connector: str, cpu_gib: int = 4) -> str:
|
||||
def _kv_transfer_config(
|
||||
connector: str, cpu_gib: int = 4, spec_name: str = "CPUOffloadingSpec"
|
||||
) -> str:
|
||||
if connector == "OffloadingConnector":
|
||||
return json.dumps(
|
||||
{
|
||||
"kv_connector": "OffloadingConnector",
|
||||
"kv_role": "kv_both",
|
||||
"kv_connector_extra_config": {
|
||||
"spec_name": "CPUOffloadingSpec",
|
||||
"spec_name": spec_name,
|
||||
"cpu_bytes_to_use": cpu_gib << 30,
|
||||
"eviction_policy": "lru",
|
||||
},
|
||||
@@ -115,8 +119,10 @@ class OffloadingModelConfig:
|
||||
accuracy_threshold: float
|
||||
tolerance: float = 0.05
|
||||
extra_server_args: list[str] = field(default_factory=list)
|
||||
env_dict: dict[str, str] = field(default_factory=dict)
|
||||
cpu_offload_gib: int = 4
|
||||
startup_timeout: int = 600
|
||||
spec_name: str = "CPUOffloadingSpec"
|
||||
|
||||
|
||||
MODELS = [
|
||||
@@ -165,6 +171,20 @@ MODELS = [
|
||||
cpu_offload_gib=16,
|
||||
startup_timeout=1200,
|
||||
),
|
||||
OffloadingModelConfig(
|
||||
id="offloading-deepseek-v2-lite-tiering-tp2",
|
||||
model="deepseek-ai/DeepSeek-V2-Lite",
|
||||
connector="OffloadingConnector",
|
||||
# Baseline 0.360/0.345 over two runs (measured on 2xA100).
|
||||
accuracy_threshold=0.35,
|
||||
extra_server_args=[
|
||||
"--tensor-parallel-size",
|
||||
"2",
|
||||
],
|
||||
cpu_offload_gib=8,
|
||||
startup_timeout=1200,
|
||||
spec_name="TieringOffloadingSpec",
|
||||
),
|
||||
# ── SimpleCPUOffloadConnector ────────────────────────────────────
|
||||
OffloadingModelConfig(
|
||||
id="simple-nemotron-h-8b",
|
||||
@@ -229,7 +249,7 @@ def test_gsm8k_offloading_correctness(cfg: OffloadingModelConfig):
|
||||
"--enable-prefix-caching",
|
||||
"--no-disable-hybrid-kv-cache-manager",
|
||||
"--kv-transfer-config",
|
||||
_kv_transfer_config(cfg.connector, cfg.cpu_offload_gib),
|
||||
_kv_transfer_config(cfg.connector, cfg.cpu_offload_gib, cfg.spec_name),
|
||||
"--trust-remote-code",
|
||||
"--disable-uvicorn-access-log",
|
||||
*cfg.extra_server_args,
|
||||
@@ -239,7 +259,7 @@ def test_gsm8k_offloading_correctness(cfg: OffloadingModelConfig):
|
||||
cfg.model,
|
||||
server_args,
|
||||
# /reset_prefix_cache requires dev mode.
|
||||
env_dict={"VLLM_SERVER_DEV_MODE": "1"},
|
||||
env_dict={"VLLM_SERVER_DEV_MODE": "1", **cfg.env_dict},
|
||||
max_wait_seconds=cfg.startup_timeout,
|
||||
) as server:
|
||||
base_url = f"http://{server.host}:{server.port}"
|
||||
|
||||
Reference in New Issue
Block a user