[Feature][Frontend] Expose effective attention block size for DCP (#56538)

Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Signed-off-by: Julien Darve <jdarve@NVIDIA.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
JulienDarve
2026-09-16 08:29:43 +00:00
committed by GitHub
co-authored by Bugen Zhao OpenAI Codex
parent 0d8173d153
commit f37c550bf6
18 changed files with 159 additions and 11 deletions
@@ -20,6 +20,15 @@ Both approaches are under active development.
Due to the auto-regressive nature of decoding, every decoding step needs to compute a small amount of query tokens w.r.t. a large number of key/value tokens stored in the paged KV cache. The core of decode context parallel is how to shard the KV cache across GPUs.
`engine_client.vllm_config.cache_config.effective_attention_block_size` reports the
initialized full-attention block size in tokens, including DCP: a physical block of
16 tokens with DCP=4 represents 64 tokens. The same value is available through
gRPC `Control.GetServerInfo.effective_attention_block_size`. Physical block-size
fields retain their existing meaning. The new value is unavailable (`None` in
Python, absent in gRPC) if there is no common full-attention size, the scheduler
does not expose it, or an older engine omits it. gRPC also omits the value when
engine replicas disagree. Partial cache events carry their own actual size.
For a model with `H` kv-heads, a request with `T` tokens in the context needs to store `H * T` key/value tensors in the KV cache.
1. If one GPU can hold them all, and the performance is good enough, then no parallelization is needed.
+1 -1
View File
@@ -147,7 +147,7 @@ vllm-llm = { path = "src/llm" }
vllm-managed-engine = { path = "src/managed-engine" }
vllm-metrics = { path = "src/metrics" }
vllm-parser = { path = "src/parser" }
vllm-proto = { version = "0.2.0", path = "proto" }
vllm-proto = { path = "proto" }
vllm-server = { path = "src/server" }
vllm-text = { path = "src/text" }
vllm-tokenizer = { path = "src/tokenizer" }
+4 -2
View File
@@ -2,6 +2,9 @@
This directory is the canonical source for vLLM's gRPC schema.
See [context parallel deployment](../../docs/serving/context_parallel_deployment.md)
for effective attention block-size metadata in Python and `Control.GetServerInfo`.
Schema updates are no longer published to the Buf Schema Registry. Rust consumers
should use `vllm-proto` from crates.io; consumers in other languages can generate
bindings from the `.proto` files in this directory. Buf still builds and lints
@@ -34,8 +37,7 @@ On pull requests and releases, `cargo-semver-checks` compares the crate with its
latest published version. Include any required version bump in the protocol
change PR. This check becomes available after the first manual publication.
1. Update the crate version and the `vllm-proto` workspace dependency together,
and update `rust/Cargo.lock`.
1. Update the crate version in `rust/proto/Cargo.toml` and update `rust/Cargo.lock`.
2. Run `cargo publish --manifest-path rust/proto/Cargo.toml --locked --dry-run`
and the frontend gRPC tests. Record the tested vLLM releases or revisions in
the release notes; matching crate versions alone do not establish runtime compatibility.
+2
View File
@@ -50,6 +50,8 @@ message ServerInfo {
uint64 max_batched_tokens = 9;
uint32 max_loras = 10;
RlCapabilities rl_capabilities = 11;
// Full-attention block size in tokens after initialization; absent if unavailable.
optional uint64 effective_attention_block_size = 14;
}
message RlCapabilities {
@@ -510,6 +510,15 @@ impl EngineCoreClient {
self.engines.iter().map(|engine| engine.ready_response.num_gpu_blocks).sum()
}
/// Return the effective attention block size if all engines report the same value.
pub fn effective_attention_block_size(&self) -> Option<u64> {
let size = self.ready_response().effective_attention_block_size?;
self.engines
.iter()
.all(|engine| engine.ready_response.effective_attention_block_size == Some(size))
.then_some(size)
}
/// Return the minimum engine-reported `max_model_len` across all engines.
///
/// This is the auto-fitted value after KV cache profiling and may differ
@@ -72,6 +72,7 @@ pub fn default_ready_response() -> EngineCoreReadyResponse {
weight_transfer_backend: None,
enable_sleep_mode: false,
supports_draft_weight_updates: false,
effective_attention_block_size: None,
}
}
@@ -104,6 +104,9 @@ pub struct EngineCoreReadyResponse {
/// Whether the engine has a speculative draft model that can be updated.
#[serde(default)]
pub supports_draft_weight_updates: bool,
/// Full-attention block size in tokens after initialization, or unavailable.
#[serde(default)]
pub effective_attention_block_size: Option<u64>,
}
/// Frontend-owned ZMQ addresses that are sent to the engine during startup
@@ -2846,6 +2846,11 @@ fn python_msgpack_fixtures_match_rust_encoding() {
let ready_response: EngineCoreReadyResponse =
rmp_serde::from_slice(&hex::decode(ready_response_hex).unwrap()).unwrap();
let mut legacy_ready = serde_json::to_value(&ready_response).unwrap();
legacy_ready.as_object_mut().unwrap().remove("effective_attention_block_size");
let legacy_ready: EngineCoreReadyResponse =
rmp_serde::from_slice(&rmp_serde::to_vec_named(&legacy_ready).unwrap()).unwrap();
assert!(legacy_ready.effective_attention_block_size.is_none());
assert!(ready_response.supports_lora);
assert_eq!(ready_response.max_loras, 8);
assert_eq!(
@@ -2854,6 +2859,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
);
assert!(ready_response.enable_sleep_mode);
assert!(ready_response.supports_draft_weight_updates);
assert_eq!(ready_response.effective_attention_block_size, Some(64));
let kv_events_config = ready_response.kv_events_config.expect("KV events config should decode");
assert!(kv_events_config.enable_kv_cache_events);
assert_eq!(kv_events_config.publisher, "zmq");
@@ -423,6 +423,7 @@ class EngineCoreReadyResponse:
weight_transfer_backend: str | None = None
enable_sleep_mode: bool = False
supports_draft_weight_updates: bool = False
effective_attention_block_size: int | None = None
ready_response = EngineCoreReadyResponse(
@@ -446,6 +447,7 @@ ready_response = EngineCoreReadyResponse(
weight_transfer_backend="nccl",
enable_sleep_mode=True,
supports_draft_weight_updates=True,
effective_attention_block_size=64,
kv_events_config=KVEventsConfig(
enable_kv_cache_events=True,
publisher="zmq",
+1
View File
@@ -195,6 +195,7 @@ impl pb::control_server::Control for ControlServiceImpl {
max_batched_tokens: ready.max_num_batched_tokens,
max_loras: ready.max_loras,
rl_capabilities: Some(self.rl_capabilities()),
effective_attention_block_size: self.client().effective_attention_block_size(),
}))
}
+12 -1
View File
@@ -1911,8 +1911,16 @@ async fn control_abort_resolves_external_id_and_empty_is_noop() {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn control_reports_server_and_model_info() {
let mut ready = default_ready_response();
ready.effective_attention_block_size = Some(64);
let (generate_service, control_service, engine_health, _engine_task) =
setup_grpc_service(b"engine-grpc-info", default_stream_output_specs()).await;
setup_grpc_service_with_engine_script(
b"engine-grpc-info".to_vec(),
ready,
Arc::new(FakeTextBackend),
|_, _| boxed_test_future(async {}),
)
.await;
let (channel, server_task) = start_grpc_test_server(
generate_service,
control_service,
@@ -1928,6 +1936,7 @@ async fn control_reports_server_and_model_info() {
.expect("get server info")
.into_inner();
assert_eq!(server.engine_version, "test-vllm-version");
assert_eq!(server.effective_attention_block_size, Some(64));
assert_eq!(server.api_version, "vllm");
assert_eq!(server.instance_id, "test-instance");
assert_eq!(server.max_model_len, DEFAULT_MOCK_MAX_MODEL_LEN as u32);
@@ -2195,6 +2204,7 @@ async fn control_aggregates_multi_engine_capacity() {
ready_0.weight_transfer_backend = Some("nccl".to_string());
ready_0.enable_sleep_mode = true;
ready_0.supports_draft_weight_updates = true;
ready_0.effective_attention_block_size = Some(64);
let mut ready_1 = default_ready_response();
ready_1.max_model_len = 4_096;
@@ -2248,6 +2258,7 @@ async fn control_aggregates_multi_engine_capacity() {
.into_inner();
assert_eq!(server.max_model_len, 4_096);
assert_eq!(server.total_kv_blocks, 30);
assert!(server.effective_attention_block_size.is_none());
let rl = server.rl_capabilities.expect("RL capabilities");
assert!(!rl.weight_transfer_enabled);
assert!(rl.weight_transfer_backend.is_empty());
+3
View File
@@ -215,6 +215,9 @@ def test_cache_config_hash_ignores_kv_cache_sizing_knobs():
base_hash = CacheConfig().compute_hash()
assert CacheConfig(kv_cache_memory_bytes=1 << 30).compute_hash() == base_hash
assert CacheConfig(gpu_memory_utilization=0.5).compute_hash() == base_hash
config = CacheConfig()
config.effective_attention_block_size = 64
assert config.compute_hash() == base_hash
def test_scheduler_config_hash_includes_max_num_seqs():
+38
View File
@@ -279,6 +279,44 @@ def make_request(
)
@pytest.mark.parametrize("dcp", [1, 4])
def test_effective_attention_block_size_matches_events(dcp):
from vllm.distributed.kv_events import BlockStored
from vllm.v1.engine.core import EngineCore
config = KVCacheConfig(
num_blocks=32,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(["attention"], new_kv_cache_spec()),
],
)
manager = KVCacheManager(
generate_scheduler_kv_cache_config([config]),
max_model_len=256,
scheduler_block_size=16 * dcp,
hash_block_size=16 * dcp,
dcp_world_size=dcp,
enable_kv_cache_events=True,
)
core = EngineCore.__new__(EngineCore)
core.vllm_config = SimpleNamespace(cache_config=CacheConfig(block_size=16))
core.scheduler = SimpleNamespace(kv_cache_manager=manager)
core._initialize_effective_attention_block_size()
block_size = core.vllm_config.cache_config.effective_attention_block_size
assert block_size == 16 * dcp
request = make_request(
"block-size", list(range(64)), block_size=16 * dcp, hash_fn=sha256
)
assert manager.allocate_slots(request, 64) is not None
assert [
event.block_size
for event in manager.take_events()
if isinstance(event, BlockStored)
] == [block_size]
def new_kv_cache_spec(
block_size=16,
num_kv_heads=2,
+28 -3
View File
@@ -158,7 +158,10 @@ def test_mp_client_uses_env_timeout(monkeypatch: pytest.MonkeyPatch):
local_engines_only=False,
enable_elastic_ep=False,
)
vllm_config = SimpleNamespace(parallel_config=parallel_config)
vllm_config = SimpleNamespace(
parallel_config=parallel_config,
cache_config=SimpleNamespace(effective_attention_block_size=None),
)
client = core_client_mod.MPClient(
asyncio_mode=False,
@@ -368,10 +371,15 @@ def test_dplb_finished_requests_release_inflight():
assert req.request_id not in client.reqs_in_flight
def test_apply_ready_response_syncs_block_size():
@pytest.mark.parametrize(
("effective_size", "other_size"),
[(None, None), (4224, 4224), (4224, 1056), (4224, None)],
)
def test_apply_ready_response_syncs_block_size(effective_size, other_size):
import msgspec
client = object.__new__(MPClient)
client._effective_attention_block_sizes = set()
client.vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
model_config=SimpleNamespace(max_model_len=8192),
@@ -399,14 +407,31 @@ def test_apply_ready_response_syncs_block_size():
max_loras=0,
)
)
client._apply_ready_response(payload)
fields = msgspec.msgpack.decode(payload)
if effective_size is None:
del fields["effective_attention_block_size"]
else:
fields["effective_attention_block_size"] = effective_size
client._apply_ready_response(msgspec.msgpack.encode(fields))
assert client.vllm_config.cache_config.block_size == 1056
cache_config = client.vllm_config.cache_config
assert cache_config.effective_attention_block_size == effective_size
fields["effective_attention_block_size"] = other_size
client._apply_ready_response(msgspec.msgpack.encode(fields))
assert cache_config.effective_attention_block_size == (
effective_size if effective_size == other_size else None
)
client._apply_ready_response(b"")
assert cache_config.effective_attention_block_size is None
def test_apply_ready_response_syncs_mamba_block_size():
import msgspec
client = object.__new__(MPClient)
client._effective_attention_block_sizes = set()
client.vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
model_config=SimpleNamespace(max_model_len=8192),
+3
View File
@@ -211,6 +211,8 @@ class CacheConfig:
"""The number of blocks to allocate for CPU memory."""
# Set after KV cache initialization.
effective_attention_block_size: int | None = field(default=None, init=False)
"""Full-attention block size in tokens, including DCP, or None if unavailable."""
kv_cache_size_tokens: int | None = field(default=None, init=False)
"""Per-DP-engine KV cache capacity in tokens (group-aware). Uses
group-aware capacity since num_gpu_blocks * block_size can be wrong
@@ -277,6 +279,7 @@ class CacheConfig:
# Post-init/derived counters
"num_gpu_blocks",
"num_cpu_blocks",
"effective_attention_block_size",
"kv_cache_size_tokens",
"kv_cache_max_concurrency",
# WIP feature toggle not impacting compiled graph shape
+2
View File
@@ -102,6 +102,8 @@ class EngineCoreReadyResponse:
weight_transfer_backend: str | None = None
enable_sleep_mode: bool = False
supports_draft_weight_updates: bool = False
# Full-attention block size in tokens after initialization, or unavailable.
effective_attention_block_size: int | None = None
class EngineCoreRequest(
+20 -1
View File
@@ -85,7 +85,7 @@ from vllm.v1.fault_tolerance.engine_core_sentinel import (
EngineCoreSentinel,
fault_tolerant_wrapper,
)
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.kv_cache_interface import KVCacheConfig, is_full_attention_spec
from vllm.v1.metrics.stats import SchedulerIterationDetails, SchedulerStats
from vllm.v1.outputs import ModelRunnerOutput
from vllm.v1.request import Request, RequestStatus
@@ -168,6 +168,7 @@ class EngineCore:
block_size=scheduler_block_size,
hash_block_size=hash_block_size,
)
self._initialize_effective_attention_block_size()
self.use_spec_decode = vllm_config.speculative_config is not None
self.check_for_draft_tokens = (
self.use_spec_decode or vllm_config.model_config.is_diffusion
@@ -382,6 +383,21 @@ class EngineCore:
)
return scheduler_kv_cache_config
def _initialize_effective_attention_block_size(self) -> None:
cache_config = self.vllm_config.cache_config
cache_config.effective_attention_block_size = None
cache_manager = getattr(self.scheduler, "kv_cache_manager", None)
if cache_manager is None:
return
block_sizes = {
manager.block_size
for manager in cache_manager.coordinator.single_type_managers
if is_full_attention_spec(manager.kv_cache_spec)
}
cache_config.effective_attention_block_size = (
block_sizes.pop() if len(block_sizes) == 1 else None
)
def get_supported_tasks(self) -> tuple[SupportedTask, ...]:
supported_tasks = self.model_executor.supported_tasks
self._log_pooler_config(supported_tasks)
@@ -1649,6 +1665,9 @@ class EngineCoreProc(EngineCore):
num_gpu_blocks=self.vllm_config.cache_config.num_gpu_blocks or 0,
block_size=self.vllm_config.cache_config.block_size,
mamba_block_size=self.vllm_config.cache_config.mamba_block_size,
effective_attention_block_size=(
self.vllm_config.cache_config.effective_attention_block_size
),
dp_stats_address=self.frontend_stats_publish_address,
dtype=str(self.vllm_config.model_config.dtype).removeprefix("torch."),
vllm_version=VLLM_VERSION,
+15 -3
View File
@@ -566,6 +566,7 @@ class MPClient(EngineCoreClient):
):
self.vllm_config = vllm_config
self._renderer: BaseRenderer | None = renderer
self._effective_attention_block_sizes: set[int | None] = set()
# ZMQ setup.
sync_ctx = zmq.Context(io_threads=2)
@@ -803,10 +804,21 @@ class MPClient(EngineCoreClient):
def _apply_ready_response(self, payload: bytes) -> None:
"""Decode an EngineCoreReadyResponse and sync any post-initialization
config changes (e.g. auto-fitted max_model_len) back to the frontend."""
if not payload:
return
vllm_config = self.vllm_config
response = msgspec.msgpack.decode(payload, type=EngineCoreReadyResponse)
response = (
msgspec.msgpack.decode(payload, type=EngineCoreReadyResponse)
if payload
else None
)
self._effective_attention_block_sizes.add(
response.effective_attention_block_size if response is not None else None
)
sizes = self._effective_attention_block_sizes
vllm_config.cache_config.effective_attention_block_size = (
next(iter(sizes)) if len(sizes) == 1 else None
)
if response is None:
return
vllm_config.model_config.max_model_len = min(
vllm_config.model_config.max_model_len, response.max_model_len
)