[Feature][Frontend] Expose effective attention block size for DCP (#56538)

Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Signed-off-by: Julien Darve <jdarve@NVIDIA.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
JulienDarve
2026-09-16 08:29:43 +00:00
committed by GitHub
co-authored by Bugen Zhao OpenAI Codex
parent 0d8173d153
commit f37c550bf6
18 changed files with 159 additions and 11 deletions
+1 -1
View File
@@ -147,7 +147,7 @@ vllm-llm = { path = "src/llm" }
vllm-managed-engine = { path = "src/managed-engine" }
vllm-metrics = { path = "src/metrics" }
vllm-parser = { path = "src/parser" }
vllm-proto = { version = "0.2.0", path = "proto" }
vllm-proto = { path = "proto" }
vllm-server = { path = "src/server" }
vllm-text = { path = "src/text" }
vllm-tokenizer = { path = "src/tokenizer" }
+4 -2
View File
@@ -2,6 +2,9 @@
This directory is the canonical source for vLLM's gRPC schema.
See [context parallel deployment](../../docs/serving/context_parallel_deployment.md)
for effective attention block-size metadata in Python and `Control.GetServerInfo`.
Schema updates are no longer published to the Buf Schema Registry. Rust consumers
should use `vllm-proto` from crates.io; consumers in other languages can generate
bindings from the `.proto` files in this directory. Buf still builds and lints
@@ -34,8 +37,7 @@ On pull requests and releases, `cargo-semver-checks` compares the crate with its
latest published version. Include any required version bump in the protocol
change PR. This check becomes available after the first manual publication.
1. Update the crate version and the `vllm-proto` workspace dependency together,
and update `rust/Cargo.lock`.
1. Update the crate version in `rust/proto/Cargo.toml` and update `rust/Cargo.lock`.
2. Run `cargo publish --manifest-path rust/proto/Cargo.toml --locked --dry-run`
and the frontend gRPC tests. Record the tested vLLM releases or revisions in
the release notes; matching crate versions alone do not establish runtime compatibility.
+2
View File
@@ -50,6 +50,8 @@ message ServerInfo {
uint64 max_batched_tokens = 9;
uint32 max_loras = 10;
RlCapabilities rl_capabilities = 11;
// Full-attention block size in tokens after initialization; absent if unavailable.
optional uint64 effective_attention_block_size = 14;
}
message RlCapabilities {
@@ -510,6 +510,15 @@ impl EngineCoreClient {
self.engines.iter().map(|engine| engine.ready_response.num_gpu_blocks).sum()
}
/// Return the effective attention block size if all engines report the same value.
pub fn effective_attention_block_size(&self) -> Option<u64> {
let size = self.ready_response().effective_attention_block_size?;
self.engines
.iter()
.all(|engine| engine.ready_response.effective_attention_block_size == Some(size))
.then_some(size)
}
/// Return the minimum engine-reported `max_model_len` across all engines.
///
/// This is the auto-fitted value after KV cache profiling and may differ
@@ -72,6 +72,7 @@ pub fn default_ready_response() -> EngineCoreReadyResponse {
weight_transfer_backend: None,
enable_sleep_mode: false,
supports_draft_weight_updates: false,
effective_attention_block_size: None,
}
}
@@ -104,6 +104,9 @@ pub struct EngineCoreReadyResponse {
/// Whether the engine has a speculative draft model that can be updated.
#[serde(default)]
pub supports_draft_weight_updates: bool,
/// Full-attention block size in tokens after initialization, or unavailable.
#[serde(default)]
pub effective_attention_block_size: Option<u64>,
}
/// Frontend-owned ZMQ addresses that are sent to the engine during startup
@@ -2846,6 +2846,11 @@ fn python_msgpack_fixtures_match_rust_encoding() {
let ready_response: EngineCoreReadyResponse =
rmp_serde::from_slice(&hex::decode(ready_response_hex).unwrap()).unwrap();
let mut legacy_ready = serde_json::to_value(&ready_response).unwrap();
legacy_ready.as_object_mut().unwrap().remove("effective_attention_block_size");
let legacy_ready: EngineCoreReadyResponse =
rmp_serde::from_slice(&rmp_serde::to_vec_named(&legacy_ready).unwrap()).unwrap();
assert!(legacy_ready.effective_attention_block_size.is_none());
assert!(ready_response.supports_lora);
assert_eq!(ready_response.max_loras, 8);
assert_eq!(
@@ -2854,6 +2859,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
);
assert!(ready_response.enable_sleep_mode);
assert!(ready_response.supports_draft_weight_updates);
assert_eq!(ready_response.effective_attention_block_size, Some(64));
let kv_events_config = ready_response.kv_events_config.expect("KV events config should decode");
assert!(kv_events_config.enable_kv_cache_events);
assert_eq!(kv_events_config.publisher, "zmq");
@@ -423,6 +423,7 @@ class EngineCoreReadyResponse:
weight_transfer_backend: str | None = None
enable_sleep_mode: bool = False
supports_draft_weight_updates: bool = False
effective_attention_block_size: int | None = None
ready_response = EngineCoreReadyResponse(
@@ -446,6 +447,7 @@ ready_response = EngineCoreReadyResponse(
weight_transfer_backend="nccl",
enable_sleep_mode=True,
supports_draft_weight_updates=True,
effective_attention_block_size=64,
kv_events_config=KVEventsConfig(
enable_kv_cache_events=True,
publisher="zmq",
+1
View File
@@ -195,6 +195,7 @@ impl pb::control_server::Control for ControlServiceImpl {
max_batched_tokens: ready.max_num_batched_tokens,
max_loras: ready.max_loras,
rl_capabilities: Some(self.rl_capabilities()),
effective_attention_block_size: self.client().effective_attention_block_size(),
}))
}
+12 -1
View File
@@ -1911,8 +1911,16 @@ async fn control_abort_resolves_external_id_and_empty_is_noop() {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn control_reports_server_and_model_info() {
let mut ready = default_ready_response();
ready.effective_attention_block_size = Some(64);
let (generate_service, control_service, engine_health, _engine_task) =
setup_grpc_service(b"engine-grpc-info", default_stream_output_specs()).await;
setup_grpc_service_with_engine_script(
b"engine-grpc-info".to_vec(),
ready,
Arc::new(FakeTextBackend),
|_, _| boxed_test_future(async {}),
)
.await;
let (channel, server_task) = start_grpc_test_server(
generate_service,
control_service,
@@ -1928,6 +1936,7 @@ async fn control_reports_server_and_model_info() {
.expect("get server info")
.into_inner();
assert_eq!(server.engine_version, "test-vllm-version");
assert_eq!(server.effective_attention_block_size, Some(64));
assert_eq!(server.api_version, "vllm");
assert_eq!(server.instance_id, "test-instance");
assert_eq!(server.max_model_len, DEFAULT_MOCK_MAX_MODEL_LEN as u32);
@@ -2195,6 +2204,7 @@ async fn control_aggregates_multi_engine_capacity() {
ready_0.weight_transfer_backend = Some("nccl".to_string());
ready_0.enable_sleep_mode = true;
ready_0.supports_draft_weight_updates = true;
ready_0.effective_attention_block_size = Some(64);
let mut ready_1 = default_ready_response();
ready_1.max_model_len = 4_096;
@@ -2248,6 +2258,7 @@ async fn control_aggregates_multi_engine_capacity() {
.into_inner();
assert_eq!(server.max_model_len, 4_096);
assert_eq!(server.total_kv_blocks, 30);
assert!(server.effective_attention_block_size.is_none());
let rl = server.rl_capabilities.expect("RL capabilities");
assert!(!rl.weight_transfer_enabled);
assert!(rl.weight_transfer_backend.is_empty());