[Feature][Frontend] Expose effective attention block size for DCP (#56538)

Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Signed-off-by: Julien Darve <jdarve@NVIDIA.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
JulienDarve
2026-09-16 08:29:43 +00:00
committed by GitHub
co-authored by Bugen Zhao OpenAI Codex
parent 0d8173d153
commit f37c550bf6
18 changed files with 159 additions and 11 deletions
@@ -510,6 +510,15 @@ impl EngineCoreClient {
self.engines.iter().map(|engine| engine.ready_response.num_gpu_blocks).sum()
}
/// Return the effective attention block size if all engines report the same value.
pub fn effective_attention_block_size(&self) -> Option<u64> {
let size = self.ready_response().effective_attention_block_size?;
self.engines
.iter()
.all(|engine| engine.ready_response.effective_attention_block_size == Some(size))
.then_some(size)
}
/// Return the minimum engine-reported `max_model_len` across all engines.
///
/// This is the auto-fitted value after KV cache profiling and may differ
@@ -72,6 +72,7 @@ pub fn default_ready_response() -> EngineCoreReadyResponse {
weight_transfer_backend: None,
enable_sleep_mode: false,
supports_draft_weight_updates: false,
effective_attention_block_size: None,
}
}
@@ -104,6 +104,9 @@ pub struct EngineCoreReadyResponse {
/// Whether the engine has a speculative draft model that can be updated.
#[serde(default)]
pub supports_draft_weight_updates: bool,
/// Full-attention block size in tokens after initialization, or unavailable.
#[serde(default)]
pub effective_attention_block_size: Option<u64>,
}
/// Frontend-owned ZMQ addresses that are sent to the engine during startup
@@ -2846,6 +2846,11 @@ fn python_msgpack_fixtures_match_rust_encoding() {
let ready_response: EngineCoreReadyResponse =
rmp_serde::from_slice(&hex::decode(ready_response_hex).unwrap()).unwrap();
let mut legacy_ready = serde_json::to_value(&ready_response).unwrap();
legacy_ready.as_object_mut().unwrap().remove("effective_attention_block_size");
let legacy_ready: EngineCoreReadyResponse =
rmp_serde::from_slice(&rmp_serde::to_vec_named(&legacy_ready).unwrap()).unwrap();
assert!(legacy_ready.effective_attention_block_size.is_none());
assert!(ready_response.supports_lora);
assert_eq!(ready_response.max_loras, 8);
assert_eq!(
@@ -2854,6 +2859,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
);
assert!(ready_response.enable_sleep_mode);
assert!(ready_response.supports_draft_weight_updates);
assert_eq!(ready_response.effective_attention_block_size, Some(64));
let kv_events_config = ready_response.kv_events_config.expect("KV events config should decode");
assert!(kv_events_config.enable_kv_cache_events);
assert_eq!(kv_events_config.publisher, "zmq");
@@ -423,6 +423,7 @@ class EngineCoreReadyResponse:
weight_transfer_backend: str | None = None
enable_sleep_mode: bool = False
supports_draft_weight_updates: bool = False
effective_attention_block_size: int | None = None
ready_response = EngineCoreReadyResponse(
@@ -446,6 +447,7 @@ ready_response = EngineCoreReadyResponse(
weight_transfer_backend="nccl",
enable_sleep_mode=True,
supports_draft_weight_updates=True,
effective_attention_block_size=64,
kv_events_config=KVEventsConfig(
enable_kv_cache_events=True,
publisher="zmq",