[PD][Core] Fix Mamba prefix cache hit rate in PD disaggregation (#44243)

Co-authored-by: lHrHenry233 <2381623149@qq.com>
Co-authored-by: underfituu <hzhucong@163.com>
Signed-off-by: Zhanqiu Hu <zhu@redhat.com>
This commit is contained in:
zhanqiuhu
2026-06-11 14:10:25 +00:00
committed by GitHub
co-authored by lHrHenry233 underfituu
parent cc640ee8bc
commit 55911db580
5 changed files with 534 additions and 3 deletions
+14
View File
@@ -61,6 +61,20 @@ steps:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- HYBRID_SSM=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
- label: Hybrid SSM NixlConnector PD prefix cache test (2 GPUs)
key: hybrid-ssm-nixlconnector-pd-prefix-cache-2-gpus
timeout_in_minutes: 25
working_dir: "/vllm-workspace/tests"
num_devices: 2
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- vllm/v1/core/sched/
- vllm/v1/core/kv_cache_coordinator.py
- tests/v1/kv_connector/nixl_integration/
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- bash v1/kv_connector/nixl_integration/run_mamba_prefix_cache_test.sh
- label: MultiConnector (Nixl+Offloading) PD accuracy (2 GPUs)
key: multiconnector-nixl-offloading-pd-accuracy-2-gpus
timeout_in_minutes: 30
@@ -0,0 +1,96 @@
#!/bin/bash
set -xe
# E2E test: Mamba hybrid prefix cache hits in PD disaggregation.
# Spins up a 1P1D setup with a Mamba hybrid model and verifies
# repeated prompts yield non-zero D-side prefix cache hits.
PREFILL_GPU_ID=${PREFILL_GPU_ID:-0}
DECODE_GPU_ID=${DECODE_GPU_ID:-1}
MODEL=${MODEL:-"ibm-granite/granite-4.0-h-tiny"}
GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.8}
echo "Running Mamba prefix cache test (GPUs: P=$PREFILL_GPU_ID, D=$DECODE_GPU_ID, model=$MODEL)"
KV_CONFIG='{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
# Resolve repository root
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
GIT_ROOT="${GIT_ROOT:-$(cd -- "${SCRIPT_DIR}/../../../.." && pwd -P)}"
trap 'kill $(jobs -pr) 2>/dev/null' SIGINT SIGTERM EXIT
wait_for_server() {
local port=$1
timeout 600 bash -c "
until curl -s localhost:${port}/v1/completions > /dev/null; do
sleep 1
done" && return 0 || return 1
}
cleanup_instances() {
echo "Cleaning up any running vLLM instances..."
pkill -f "vllm serve" || true
sleep 2
}
cleanup_instances
# Start prefill instance
PREFILL_PORT=8001
CUDA_VISIBLE_DEVICES=$PREFILL_GPU_ID \
VLLM_SSM_CONV_STATE_LAYOUT=DS \
VLLM_KV_CACHE_LAYOUT=HND \
VLLM_NIXL_SIDE_CHANNEL_PORT=5559 \
vllm serve $MODEL \
--port $PREFILL_PORT \
--enforce-eager \
--gpu-memory-utilization $GPU_MEMORY_UTILIZATION \
--max-model-len 16384 \
--block-size 128 \
--trust-remote-code \
--enable-prefix-caching \
--mamba-cache-mode all \
--kv-transfer-config "$KV_CONFIG" &
# Start decode instance
DECODE_PORT=8002
CUDA_VISIBLE_DEVICES=$DECODE_GPU_ID \
VLLM_SSM_CONV_STATE_LAYOUT=DS \
VLLM_KV_CACHE_LAYOUT=HND \
VLLM_NIXL_SIDE_CHANNEL_PORT=6000 \
vllm serve $MODEL \
--port $DECODE_PORT \
--enforce-eager \
--gpu-memory-utilization $GPU_MEMORY_UTILIZATION \
--max-model-len 16384 \
--block-size 128 \
--trust-remote-code \
--enable-prefix-caching \
--mamba-cache-mode all \
--kv-transfer-config "$KV_CONFIG" &
echo "Waiting for prefill instance on port $PREFILL_PORT..."
wait_for_server "$PREFILL_PORT"
echo "Waiting for decode instance on port $DECODE_PORT..."
wait_for_server "$DECODE_PORT"
# Start proxy
PROXY_PORT=8192
python3 "${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/toy_proxy_server.py" \
--port $PROXY_PORT \
--prefiller-ports $PREFILL_PORT \
--decoder-ports $DECODE_PORT &
sleep 5
echo "Running Mamba prefix cache test..."
PREFILL_PORT=$PREFILL_PORT \
DECODE_PORT=$DECODE_PORT \
PROXY_PORT=$PROXY_PORT \
python3 -m pytest -s -v \
"${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/test_mamba_prefix_cache.py"
echo "Mamba prefix cache test passed!"
cleanup_instances
@@ -0,0 +1,346 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Verify D-side prefix cache hits reduce transfer for Mamba hybrid PD.
Sends the same long prompt twice through P/D and asserts that the second
request transfers fewer bytes (because cached blocks are skipped).
"""
import os
import time
import openai
import regex as re
import requests
PREFILL_HOST = os.getenv("PREFILL_HOST", "localhost")
PREFILL_PORT = os.environ["PREFILL_PORT"]
DECODE_HOST = os.getenv("DECODE_HOST", "localhost")
DECODE_PORT = os.environ["DECODE_PORT"]
PROXY_HOST = os.getenv("PROXY_HOST", "localhost")
PROXY_PORT = os.environ["PROXY_PORT"]
# Long prompt (~9000 tokens) to span many blocks so prefix caching kicks in.
_BASE_PROMPT = """\
The following is a comprehensive overview of distributed systems, covering \
their history, design principles, and modern applications.
Distributed systems emerged from the need to connect multiple computers to \
work together on shared tasks. In the 1960s, ARPANET demonstrated that \
geographically dispersed machines could communicate through packet switching. \
This laid the groundwork for decades of research into fault tolerance, \
consistency, and performance.
Leslie Lamport's 1978 paper on logical clocks introduced the concept of \
causal ordering in distributed systems. His later work on the Paxos algorithm \
provided a practical solution to the consensus problem, enabling multiple \
nodes to agree on a single value despite failures. The Byzantine Generals \
Problem, also formulated by Lamport, addressed the challenge of reaching \
agreement when some participants may be malicious.
The CAP theorem, proposed by Eric Brewer in 2000 and formally proved by Seth \
Gilbert and Nancy Lynch in 2002, states that a distributed system cannot \
simultaneously provide Consistency, Availability, and Partition tolerance. \
This fundamental trade-off has guided the design of distributed databases and \
storage systems ever since. Systems like Google's Bigtable chose consistency \
and partition tolerance, while Amazon's Dynamo prioritized availability and \
partition tolerance.
Google's MapReduce framework, published in 2004, popularized the concept of \
processing large datasets across clusters of commodity hardware. The \
programming model was simple: users specified a map function to process \
key-value pairs and a reduce function to merge intermediate values. The \
framework handled distribution, fault tolerance, and load balancing \
automatically. This inspired the open-source Hadoop ecosystem, which became \
the foundation for big data processing throughout the 2010s.
The Google File System (GFS) and its open-source counterpart HDFS provided \
the distributed storage layer beneath MapReduce. These systems replicated \
data across multiple nodes, using a single master for metadata management \
and chunk servers for actual data storage. The master maintained a mapping \
from files to chunks and tracked which chunk servers held each replica.
Apache Kafka, developed at LinkedIn and open-sourced in 2011, introduced a \
distributed commit log that could handle millions of messages per second. \
Its design separated producers from consumers through topic-based \
publish-subscribe semantics. Partitioning allowed horizontal scaling, while \
replication ensured durability. Kafka's exactly-once semantics, achieved \
through idempotent producers and transactional writes, made it suitable for \
financial and mission-critical applications.
Raft, published by Diego Ongaro and John Ousterhout in 2014, provided an \
understandable alternative to Paxos for consensus. Its key insight was \
decomposing consensus into leader election, log replication, and safety. A \
leader would be elected through randomized timeouts, then would replicate \
its log entries to followers. Committed entries were guaranteed to be present \
on a majority of servers. Raft's clarity led to its adoption in systems like \
etcd, CockroachDB, and TiKV.
Container orchestration systems like Kubernetes, released by Google in 2014, \
brought distributed systems concepts to application deployment. Kubernetes \
managed clusters of machines, scheduling containers across nodes while \
maintaining desired state. Its control plane used etcd for consistent state \
storage, an API server for client communication, a scheduler for placement \
decisions, and controllers for reconciliation loops.
Service meshes emerged to handle the networking complexity of microservices \
architectures. Istio, Linkerd, and Envoy provided transparent proxying, load \
balancing, circuit breaking, and observability without requiring application \
code changes. They implemented the sidecar pattern, deploying a proxy \
alongside each service instance to intercept all network traffic.
Modern distributed databases like CockroachDB, TiDB, and YugabyteDB combine \
the SQL interface that developers expect with the horizontal scalability of \
NoSQL systems. They use Raft for consensus, multi-version concurrency control \
for transactions, and range-based sharding for data distribution. These \
systems can span multiple data centers while providing serializable isolation.
Stream processing frameworks evolved from batch-oriented MapReduce to \
real-time systems. Apache Flink provided exactly-once processing with \
event-time semantics, handling out-of-order data through watermarks. Its \
checkpoint mechanism, based on Chandy-Lamport distributed snapshots, allowed \
recovery without data loss. Google's Dataflow model unified batch and \
streaming under a single programming model.
The rise of machine learning at scale introduced new distributed systems \
challenges. Training large neural networks required distributing computation \
across hundreds or thousands of GPUs. Data parallelism split batches across \
workers, while model parallelism partitioned the network itself. Pipeline \
parallelism overlapped computation stages to maximize utilization. \
Ring-allreduce and parameter server architectures provided different \
trade-offs for gradient synchronization.
Inference serving systems like vLLM, TensorRT-LLM, and SGLang optimized the \
deployment of large language models. They introduced techniques like \
continuous batching to maximize GPU utilization, PagedAttention for efficient \
KV cache memory management, and speculative decoding to reduce latency. \
Prefill-decode disaggregation separated the compute-intensive prefill phase \
from the memory-bound decode phase across different GPU pools.
KV cache transfer in disaggregated serving requires careful coordination \
between prefill and decode nodes. The prefill node computes the full KV cache \
for a request's prompt and transfers it to the decode node via high-bandwidth \
interconnects like NVLink, InfiniBand, or RDMA. The decode node then uses \
this transferred cache to generate tokens autoregressively without \
recomputing the prefix.
Prefix caching optimizes this further by recognizing that multiple requests \
often share common prefixes, such as system prompts or few-shot examples. \
When a decode node receives a new request whose prefix matches a previously \
transferred KV cache, it can skip the transfer for those shared blocks and \
only fetch the new, unique portion. This dramatically reduces both network \
bandwidth consumption and time-to-first-token latency.
For hybrid architectures combining attention mechanisms with state-space \
models like Mamba, prefix caching becomes more complex. Attention layers \
maintain a KV cache that can be trivially split into independent blocks, \
making prefix matching straightforward. However, Mamba layers maintain a \
recurrent hidden state that represents the entire sequence history in a \
single fixed-size tensor. This state cannot be meaningfully split into \
prefix-aligned blocks the way attention KV caches can.
The challenge in disaggregated serving of hybrid models is that the cache \
coordination logic must handle these heterogeneous cache types simultaneously. \
A naive approach that requires all cache groups to agree on a single prefix \
hit length will always report zero hits for the Mamba group on a cold decode \
node, dragging the entire prefix cache hit rate to zero even when the \
attention layers have perfect cache hits.
The solution is to evaluate each cache group independently, allowing the \
attention groups to report their actual cache hits while the Mamba group \
reports zero. The transfer logic then only fetches the blocks that each group \
actually needs: for attention, only the new uncached blocks; for Mamba, \
always the full state. This per-group evaluation preserves the prefix caching \
benefits for attention layers while correctly handling the all-or-nothing \
nature of Mamba state.
Consistency models in distributed systems range from strong linearizability \
to weak eventual consistency. Linearizability requires that operations appear \
to occur atomically at some point between their invocation and response. \
Sequential consistency relaxes this by only requiring that operations from \
each process appear in program order. Causal consistency preserves causal \
relationships between operations. Eventual consistency only guarantees that \
all replicas will eventually converge to the same state.
Vector clocks extend Lamport timestamps to capture causality precisely. Each \
process maintains a vector of logical clocks, one per process in the system. \
When a process performs a local event, it increments its own entry. When \
sending a message, it attaches its current vector. Upon receiving a message, \
a process takes the element-wise maximum of its vector and the received \
vector, then increments its own entry. Two events are concurrent if and only \
if neither vector dominates the other.
Conflict-free replicated data types (CRDTs) provide eventual consistency \
without coordination. They achieve this through mathematical properties: \
either operations are commutative and idempotent (operation-based CRDTs), or \
states form a join-semilattice where merging always produces a valid result \
(state-based CRDTs). Examples include grow-only counters, positive-negative \
counters, grow-only sets, observed-remove sets, and last-writer-wins registers.
Distributed hash tables (DHTs) like Chord, Kademlia, and Pastry provide \
decentralized key-value lookup. Chord arranges nodes on a circular identifier \
space, using finger tables for O(log n) routing. Kademlia uses XOR distance \
for routing, enabling parallel lookups and natural load balancing. These \
systems underpin peer-to-peer networks, content distribution, and \
decentralized storage.
Leader election algorithms ensure that exactly one node acts as coordinator \
at any time. The Bully algorithm selects the node with the highest \
identifier. Ring-based algorithms pass election messages around a logical \
ring. In practice, systems often use lease-based leadership where a leader \
must periodically renew its lease, allowing automatic failover when a leader \
becomes unresponsive.
Distributed transactions spanning multiple partitions require coordination \
protocols. Two-phase commit (2PC) provides atomicity but blocks if the \
coordinator fails. Three-phase commit (3PC) adds a prepare-to-commit phase \
to avoid blocking but does not handle network partitions. Saga patterns \
decompose long-running transactions into compensable sub-transactions, \
providing eventual consistency without global locks.
Load balancing in distributed systems takes many forms. Round-robin \
distributes requests evenly but ignores server capacity. Weighted round-robin \
accounts for heterogeneous servers. Least-connections routes to the server \
with fewest active requests. Consistent hashing minimizes redistribution when \
servers join or leave. Power-of-two-choices selects the less loaded of two \
randomly chosen servers, providing near-optimal balance with minimal \
coordination.
Observability in distributed systems requires correlated telemetry across \
service boundaries. Distributed tracing, pioneered by Google's Dapper and \
standardized through OpenTelemetry, propagates trace context through request \
chains. Each service adds spans representing its processing, creating a tree \
structure that reveals latency bottlenecks and error sources. Combined with \
metrics and structured logs, traces provide the visibility needed to operate \
complex distributed systems reliably."""
# Pad to ~23000 chars (~9000 tokens) to fill many blocks.
PROMPT = _BASE_PROMPT
while len(PROMPT) < 23000:
n = len(PROMPT)
PROMPT += f" The value at position {n} is {n * 7 % 9973}."
METRICS_OF_INTEREST = [
"vllm:nixl_bytes_transferred_sum",
"vllm:nixl_bytes_transferred_count",
"vllm:nixl_num_descriptors_sum",
"vllm:nixl_num_descriptors_count",
"vllm:prefix_cache_hits",
"vllm:prefix_cache_queries",
]
def get_metric(host: str, port: str, metric_name: str) -> float:
"""Scrape a single Prometheus metric from /metrics."""
url = f"http://{host}:{port}/metrics"
resp = requests.get(url, timeout=10)
resp.raise_for_status()
total = 0.0
for line in resp.text.splitlines():
if line.startswith("#"):
continue
if line.startswith(metric_name):
match = re.search(r"[\d.eE+\-]+$", line)
if match:
total += float(match.group())
return total
def get_all_metrics(host: str, port: str) -> dict[str, float]:
"""Scrape all metrics of interest."""
return {name: get_metric(host, port, name) for name in METRICS_OF_INTEREST}
def print_metrics(label: str, metrics: dict[str, float]) -> None:
print(f"\n [{label}]")
for name, val in metrics.items():
print(f" {name} = {val}")
def test_mamba_prefix_cache_hit():
"""Repeated prompts through PD should transfer fewer bytes on D-side."""
proxy_client = openai.OpenAI(
api_key="MY_KEY",
base_url=f"http://{PROXY_HOST}:{PROXY_PORT}/v1",
)
decode_client = openai.OpenAI(
api_key="MY_KEY",
base_url=f"http://{DECODE_HOST}:{DECODE_PORT}/v1",
)
models = decode_client.models.list()
MODEL = models.data[0].id
print(f"\nModel: {MODEL}")
print(f"Prompt length: {len(PROMPT)} chars")
# Baseline
m_baseline = get_all_metrics(DECODE_HOST, DECODE_PORT)
print_metrics("D-side baseline", m_baseline)
# Request 1: cold, primes the D-side cache
print("\n--- Request 1 (cold) ---")
resp1 = proxy_client.completions.create(
model=MODEL, prompt=PROMPT, max_tokens=10, temperature=0, seed=42
)
output1 = resp1.choices[0].text
print(f" Output: {output1!r}")
time.sleep(2)
m_after_req1 = get_all_metrics(DECODE_HOST, DECODE_PORT)
print_metrics("D-side after req1", m_after_req1)
transfer_req1 = (
m_after_req1["vllm:nixl_bytes_transferred_sum"]
- m_baseline["vllm:nixl_bytes_transferred_sum"]
)
descs_req1 = (
m_after_req1["vllm:nixl_num_descriptors_sum"]
- m_baseline["vllm:nixl_num_descriptors_sum"]
)
print(f" Transfer: {transfer_req1 / 1e6:.2f} MB, {descs_req1:.0f} descs")
# Request 2: same prompt, should hit D-side prefix cache
print("\n--- Request 2 (warm, same prompt) ---")
resp2 = proxy_client.completions.create(
model=MODEL, prompt=PROMPT, max_tokens=10, temperature=0, seed=42
)
output2 = resp2.choices[0].text
print(f" Output: {output2!r}")
time.sleep(2)
m_after_req2 = get_all_metrics(DECODE_HOST, DECODE_PORT)
print_metrics("D-side after req2", m_after_req2)
transfer_req2 = (
m_after_req2["vllm:nixl_bytes_transferred_sum"]
- m_after_req1["vllm:nixl_bytes_transferred_sum"]
)
descs_req2 = (
m_after_req2["vllm:nixl_num_descriptors_sum"]
- m_after_req1["vllm:nixl_num_descriptors_sum"]
)
print(f" Transfer: {transfer_req2 / 1e6:.2f} MB, {descs_req2:.0f} descs")
# P-side metrics (informational)
m_prefill = get_all_metrics(PREFILL_HOST, PREFILL_PORT)
print_metrics("P-side final", m_prefill)
# Summary
print("\n--- Summary ---")
print(f" Req 1: {transfer_req1 / 1e6:.2f} MB ({descs_req1:.0f} descs)")
print(f" Req 2: {transfer_req2 / 1e6:.2f} MB ({descs_req2:.0f} descs)")
if transfer_req1 > 0:
reduction_pct = (1 - transfer_req2 / transfer_req1) * 100
print(f" Reduction: {reduction_pct:.1f}%")
# Assertions
assert transfer_req1 > 0, (
f"First request should transfer data, got {transfer_req1} bytes"
)
assert transfer_req2 < transfer_req1, (
f"Second request should transfer fewer bytes due to D-side prefix "
f"cache hits. Got req1={transfer_req1 / 1e6:.2f} MB, "
f"req2={transfer_req2 / 1e6:.2f} MB (no reduction)."
)
assert output1 == output2, f"Outputs differ: {output1!r} vs {output2!r}"
+39
View File
@@ -691,6 +691,45 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
blocks if blocks is not None else [] for blocks in hit_blocks_by_group
), hit_length
def find_longest_cache_hit_per_group(
self,
block_hashes: list[BlockHash],
max_cache_hit_length: int,
) -> tuple[tuple[list[KVCacheBlock], ...], tuple[int, ...]]:
"""Like find_longest_cache_hit but evaluates each group independently.
Returns:
(blocks_per_group, hit_lengths_per_group)
"""
def _get_block_hashes(kv_cache_spec: KVCacheSpec) -> BlockHashList:
if kv_cache_spec.block_size == self.hash_block_size:
return block_hashes
return BlockHashListWithBlockSize(
block_hashes, self.hash_block_size, kv_cache_spec.block_size
)
num_groups = len(self.kv_cache_config.kv_cache_groups)
hit_blocks: list[list[KVCacheBlock]] = [[] for _ in range(num_groups)]
hit_lengths: list[int] = [0] * num_groups
for spec, group_ids, manager_cls, use_eagle in self.attention_groups:
blocks = manager_cls.find_longest_cache_hit(
block_hashes=_get_block_hashes(spec),
max_length=max_cache_hit_length,
kv_cache_group_ids=group_ids,
block_pool=self.block_pool,
kv_cache_spec=spec,
drop_eagle_block=use_eagle,
alignment_tokens=self.scheduler_block_size,
)
group_hit = len(blocks[0]) * spec.block_size
for gid, blks in zip(group_ids, blocks):
hit_blocks[gid] = blks
hit_lengths[gid] = group_hit
return tuple(hit_blocks), tuple(hit_lengths)
def get_kv_cache_coordinator(
kv_cache_config: KVCacheConfig,
+39 -3
View File
@@ -34,6 +34,7 @@ from vllm.v1.core.encoder_cache_manager import (
EncoderCacheManager,
EncoderDecoderCacheManager,
)
from vllm.v1.core.kv_cache_coordinator import HybridKVCacheCoordinator
from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager
from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector
from vllm.v1.core.sched.interface import PauseState, SchedulerInterface
@@ -620,9 +621,44 @@ class Scheduler(SchedulerInterface):
# Get already-cached tokens.
if request.num_computed_tokens == 0:
# Get locally-cached tokens.
new_computed_blocks, num_new_local_computed_tokens = (
self.kv_cache_manager.get_computed_blocks(request)
)
if (
self.connector is not None
and self.has_mamba_layers
and isinstance(
self.kv_cache_manager.coordinator,
HybridKVCacheCoordinator,
)
):
computed, per_group_hits = (
self.kv_cache_manager.coordinator.find_longest_cache_hit_per_group(
request.block_hashes,
request.num_tokens - 1,
)
)
new_computed_blocks = (
self.kv_cache_manager.create_kv_cache_blocks(computed)
)
# NOTE(ZhanqiuHu): For Mamba hybrid models,
# num_new_local_computed_tokens should be the FA hit
# length. This value is passed to the connector's
# get_num_new_matched_tokens which computes:
# external = total - local_computed.
# Using the FA hit skips re-transferring FA blocks
# already cached on D-side. The Mamba state (always
# the last block) is transferred unconditionally by
# _apply_prefix_caching in nixl/worker.py.
num_new_local_computed_tokens = max(per_group_hits)
if self.kv_cache_manager.log_stats:
assert self.kv_cache_manager.prefix_cache_stats is not None
self.kv_cache_manager.prefix_cache_stats.record(
num_tokens=request.num_tokens,
num_hits=num_new_local_computed_tokens,
preempted=request.num_preemptions > 0,
)
else:
new_computed_blocks, num_new_local_computed_tokens = (
self.kv_cache_manager.get_computed_blocks(request)
)
# In case of hybrid models, obtain hint for Marconi-style APC logic
if self.has_mamba_layers: