[Feature][Frontend] Expose effective attention block size for DCP (#56538)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Signed-off-by: Julien Darve <jdarve@NVIDIA.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Bugen Zhao
OpenAI Codex
parent
0d8173d153
commit
f37c550bf6
@@ -510,6 +510,15 @@ impl EngineCoreClient {
|
||||
self.engines.iter().map(|engine| engine.ready_response.num_gpu_blocks).sum()
|
||||
}
|
||||
|
||||
/// Return the effective attention block size if all engines report the same value.
|
||||
pub fn effective_attention_block_size(&self) -> Option<u64> {
|
||||
let size = self.ready_response().effective_attention_block_size?;
|
||||
self.engines
|
||||
.iter()
|
||||
.all(|engine| engine.ready_response.effective_attention_block_size == Some(size))
|
||||
.then_some(size)
|
||||
}
|
||||
|
||||
/// Return the minimum engine-reported `max_model_len` across all engines.
|
||||
///
|
||||
/// This is the auto-fitted value after KV cache profiling and may differ
|
||||
|
||||
@@ -72,6 +72,7 @@ pub fn default_ready_response() -> EngineCoreReadyResponse {
|
||||
weight_transfer_backend: None,
|
||||
enable_sleep_mode: false,
|
||||
supports_draft_weight_updates: false,
|
||||
effective_attention_block_size: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -104,6 +104,9 @@ pub struct EngineCoreReadyResponse {
|
||||
/// Whether the engine has a speculative draft model that can be updated.
|
||||
#[serde(default)]
|
||||
pub supports_draft_weight_updates: bool,
|
||||
/// Full-attention block size in tokens after initialization, or unavailable.
|
||||
#[serde(default)]
|
||||
pub effective_attention_block_size: Option<u64>,
|
||||
}
|
||||
|
||||
/// Frontend-owned ZMQ addresses that are sent to the engine during startup
|
||||
|
||||
@@ -2846,6 +2846,11 @@ fn python_msgpack_fixtures_match_rust_encoding() {
|
||||
|
||||
let ready_response: EngineCoreReadyResponse =
|
||||
rmp_serde::from_slice(&hex::decode(ready_response_hex).unwrap()).unwrap();
|
||||
let mut legacy_ready = serde_json::to_value(&ready_response).unwrap();
|
||||
legacy_ready.as_object_mut().unwrap().remove("effective_attention_block_size");
|
||||
let legacy_ready: EngineCoreReadyResponse =
|
||||
rmp_serde::from_slice(&rmp_serde::to_vec_named(&legacy_ready).unwrap()).unwrap();
|
||||
assert!(legacy_ready.effective_attention_block_size.is_none());
|
||||
assert!(ready_response.supports_lora);
|
||||
assert_eq!(ready_response.max_loras, 8);
|
||||
assert_eq!(
|
||||
@@ -2854,6 +2859,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
|
||||
);
|
||||
assert!(ready_response.enable_sleep_mode);
|
||||
assert!(ready_response.supports_draft_weight_updates);
|
||||
assert_eq!(ready_response.effective_attention_block_size, Some(64));
|
||||
let kv_events_config = ready_response.kv_events_config.expect("KV events config should decode");
|
||||
assert!(kv_events_config.enable_kv_cache_events);
|
||||
assert_eq!(kv_events_config.publisher, "zmq");
|
||||
|
||||
@@ -423,6 +423,7 @@ class EngineCoreReadyResponse:
|
||||
weight_transfer_backend: str | None = None
|
||||
enable_sleep_mode: bool = False
|
||||
supports_draft_weight_updates: bool = False
|
||||
effective_attention_block_size: int | None = None
|
||||
|
||||
|
||||
ready_response = EngineCoreReadyResponse(
|
||||
@@ -446,6 +447,7 @@ ready_response = EngineCoreReadyResponse(
|
||||
weight_transfer_backend="nccl",
|
||||
enable_sleep_mode=True,
|
||||
supports_draft_weight_updates=True,
|
||||
effective_attention_block_size=64,
|
||||
kv_events_config=KVEventsConfig(
|
||||
enable_kv_cache_events=True,
|
||||
publisher="zmq",
|
||||
|
||||
@@ -195,6 +195,7 @@ impl pb::control_server::Control for ControlServiceImpl {
|
||||
max_batched_tokens: ready.max_num_batched_tokens,
|
||||
max_loras: ready.max_loras,
|
||||
rl_capabilities: Some(self.rl_capabilities()),
|
||||
effective_attention_block_size: self.client().effective_attention_block_size(),
|
||||
}))
|
||||
}
|
||||
|
||||
|
||||
@@ -1911,8 +1911,16 @@ async fn control_abort_resolves_external_id_and_empty_is_noop() {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn control_reports_server_and_model_info() {
|
||||
let mut ready = default_ready_response();
|
||||
ready.effective_attention_block_size = Some(64);
|
||||
let (generate_service, control_service, engine_health, _engine_task) =
|
||||
setup_grpc_service(b"engine-grpc-info", default_stream_output_specs()).await;
|
||||
setup_grpc_service_with_engine_script(
|
||||
b"engine-grpc-info".to_vec(),
|
||||
ready,
|
||||
Arc::new(FakeTextBackend),
|
||||
|_, _| boxed_test_future(async {}),
|
||||
)
|
||||
.await;
|
||||
let (channel, server_task) = start_grpc_test_server(
|
||||
generate_service,
|
||||
control_service,
|
||||
@@ -1928,6 +1936,7 @@ async fn control_reports_server_and_model_info() {
|
||||
.expect("get server info")
|
||||
.into_inner();
|
||||
assert_eq!(server.engine_version, "test-vllm-version");
|
||||
assert_eq!(server.effective_attention_block_size, Some(64));
|
||||
assert_eq!(server.api_version, "vllm");
|
||||
assert_eq!(server.instance_id, "test-instance");
|
||||
assert_eq!(server.max_model_len, DEFAULT_MOCK_MAX_MODEL_LEN as u32);
|
||||
@@ -2195,6 +2204,7 @@ async fn control_aggregates_multi_engine_capacity() {
|
||||
ready_0.weight_transfer_backend = Some("nccl".to_string());
|
||||
ready_0.enable_sleep_mode = true;
|
||||
ready_0.supports_draft_weight_updates = true;
|
||||
ready_0.effective_attention_block_size = Some(64);
|
||||
|
||||
let mut ready_1 = default_ready_response();
|
||||
ready_1.max_model_len = 4_096;
|
||||
@@ -2248,6 +2258,7 @@ async fn control_aggregates_multi_engine_capacity() {
|
||||
.into_inner();
|
||||
assert_eq!(server.max_model_len, 4_096);
|
||||
assert_eq!(server.total_kv_blocks, 30);
|
||||
assert!(server.effective_attention_block_size.is_none());
|
||||
let rl = server.rl_capabilities.expect("RL capabilities");
|
||||
assert!(!rl.weight_transfer_enabled);
|
||||
assert!(rl.weight_transfer_backend.is_empty());
|
||||
|
||||
Reference in New Issue
Block a user