[Rust Frontend][gRPC] Advertise LoRA capabilities (#52031)
Signed-off-by: Connor Carpenter <connorc@nvidia.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
d1e3eee6fb
commit
f08a95f8d8
@@ -38,6 +38,7 @@ message ServerInfo {
|
|||||||
uint64 total_kv_blocks = 7;
|
uint64 total_kv_blocks = 7;
|
||||||
uint64 max_running_requests = 8;
|
uint64 max_running_requests = 8;
|
||||||
uint64 max_batched_tokens = 9;
|
uint64 max_batched_tokens = 9;
|
||||||
|
uint32 max_loras = 10;
|
||||||
RlCapabilities rl_capabilities = 11;
|
RlCapabilities rl_capabilities = 11;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -65,6 +66,7 @@ message ModelInfo {
|
|||||||
|
|
||||||
bool supports_text_input = 20;
|
bool supports_text_input = 20;
|
||||||
bool supports_token_ids_input = 21;
|
bool supports_token_ids_input = 21;
|
||||||
|
bool supports_lora = 22;
|
||||||
bool supports_multimodal = 23;
|
bool supports_multimodal = 23;
|
||||||
string reasoning_parser = 24;
|
string reasoning_parser = 24;
|
||||||
string tool_call_parser = 25;
|
string tool_call_parser = 25;
|
||||||
|
|||||||
@@ -286,6 +286,7 @@ impl EngineCoreClient {
|
|||||||
config: EngineCoreClientConfig,
|
config: EngineCoreClientConfig,
|
||||||
connected: transport::ConnectedTransport,
|
connected: transport::ConnectedTransport,
|
||||||
) -> Result<Self> {
|
) -> Result<Self> {
|
||||||
|
validate_lora_capabilities(&connected.engines)?;
|
||||||
let (output_tx, output_rx) = mpsc::channel(64);
|
let (output_tx, output_rx) = mpsc::channel(64);
|
||||||
let (abort_tx, abort_rx) = mpsc::unbounded_channel();
|
let (abort_tx, abort_rx) = mpsc::unbounded_channel();
|
||||||
let engines = connected.engines;
|
let engines = connected.engines;
|
||||||
@@ -476,6 +477,32 @@ impl EngineCoreClient {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn validate_lora_capabilities(engines: &[ConnectedEngine]) -> Result<()> {
|
||||||
|
let first = engines.first().expect("engine core client requires at least one engine");
|
||||||
|
for engine in engines {
|
||||||
|
let ready = &engine.ready_response;
|
||||||
|
if ready.supports_lora != (ready.max_loras > 0) {
|
||||||
|
return Err(Error::UnexpectedHandshakeMessage {
|
||||||
|
message: format!(
|
||||||
|
"engine {:?} reported inconsistent LoRA capability (supports_lora={}, max_loras={})",
|
||||||
|
engine.engine_id, ready.supports_lora, ready.max_loras
|
||||||
|
),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if ready.supports_lora != first.ready_response.supports_lora
|
||||||
|
|| ready.max_loras != first.ready_response.max_loras
|
||||||
|
{
|
||||||
|
return Err(Error::UnexpectedHandshakeMessage {
|
||||||
|
message: format!(
|
||||||
|
"engine {:?} reported LoRA capability inconsistent with engine {:?}",
|
||||||
|
engine.engine_id, first.engine_id
|
||||||
|
),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
// Client API implementation.
|
// Client API implementation.
|
||||||
impl EngineCoreClient {
|
impl EngineCoreClient {
|
||||||
/// Add a new request to the engine and return a per-request raw output
|
/// Add a new request to the engine and return a per-request raw output
|
||||||
|
|||||||
@@ -64,6 +64,8 @@ pub fn default_ready_response() -> EngineCoreReadyResponse {
|
|||||||
max_num_seqs: 256,
|
max_num_seqs: 256,
|
||||||
max_num_batched_tokens: 8192,
|
max_num_batched_tokens: 8192,
|
||||||
instance_id: "test-instance".to_string(),
|
instance_id: "test-instance".to_string(),
|
||||||
|
supports_lora: false,
|
||||||
|
max_loras: 0,
|
||||||
kv_cache_size_tokens: None,
|
kv_cache_size_tokens: None,
|
||||||
kv_cache_max_concurrency: None,
|
kv_cache_max_concurrency: None,
|
||||||
kv_events_config: None,
|
kv_events_config: None,
|
||||||
|
|||||||
@@ -80,6 +80,10 @@ pub struct EngineCoreReadyResponse {
|
|||||||
pub max_num_batched_tokens: u64,
|
pub max_num_batched_tokens: u64,
|
||||||
/// Unique identifier for this server instance.
|
/// Unique identifier for this server instance.
|
||||||
pub instance_id: String,
|
pub instance_id: String,
|
||||||
|
/// Whether the engine was started with LoRA support enabled.
|
||||||
|
pub supports_lora: bool,
|
||||||
|
/// Maximum number of LoRA adapters the engine may keep active.
|
||||||
|
pub max_loras: u32,
|
||||||
/// Total KV cache capacity in tokens, if reported.
|
/// Total KV cache capacity in tokens, if reported.
|
||||||
pub kv_cache_size_tokens: Option<u64>,
|
pub kv_cache_size_tokens: Option<u64>,
|
||||||
/// Maximum achievable request concurrency given the KV cache, if reported.
|
/// Maximum achievable request concurrency given the KV cache, if reported.
|
||||||
|
|||||||
@@ -2686,6 +2686,8 @@ fn python_msgpack_fixtures_match_rust_encoding() {
|
|||||||
|
|
||||||
let ready_response: EngineCoreReadyResponse =
|
let ready_response: EngineCoreReadyResponse =
|
||||||
rmp_serde::from_slice(&hex::decode(ready_response_hex).unwrap()).unwrap();
|
rmp_serde::from_slice(&hex::decode(ready_response_hex).unwrap()).unwrap();
|
||||||
|
assert!(ready_response.supports_lora);
|
||||||
|
assert_eq!(ready_response.max_loras, 8);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
ready_response.weight_transfer_backend.as_deref(),
|
ready_response.weight_transfer_backend.as_deref(),
|
||||||
Some("nccl")
|
Some("nccl")
|
||||||
|
|||||||
@@ -409,6 +409,8 @@ class EngineCoreReadyResponse:
|
|||||||
max_num_seqs: int
|
max_num_seqs: int
|
||||||
max_num_batched_tokens: int
|
max_num_batched_tokens: int
|
||||||
instance_id: str
|
instance_id: str
|
||||||
|
supports_lora: bool
|
||||||
|
max_loras: int
|
||||||
kv_cache_size_tokens: int | None = None
|
kv_cache_size_tokens: int | None = None
|
||||||
kv_cache_max_concurrency: float | None = None
|
kv_cache_max_concurrency: float | None = None
|
||||||
kv_events_config: KVEventsConfig | None = None
|
kv_events_config: KVEventsConfig | None = None
|
||||||
@@ -433,6 +435,8 @@ ready_response = EngineCoreReadyResponse(
|
|||||||
max_num_seqs=256,
|
max_num_seqs=256,
|
||||||
max_num_batched_tokens=8192,
|
max_num_batched_tokens=8192,
|
||||||
instance_id="test-instance",
|
instance_id="test-instance",
|
||||||
|
supports_lora=True,
|
||||||
|
max_loras=8,
|
||||||
weight_transfer_backend="nccl",
|
weight_transfer_backend="nccl",
|
||||||
enable_sleep_mode=True,
|
enable_sleep_mode=True,
|
||||||
supports_draft_weight_updates=True,
|
supports_draft_weight_updates=True,
|
||||||
|
|||||||
@@ -150,6 +150,7 @@ impl pb::control_server::Control for ControlServiceImpl {
|
|||||||
total_kv_blocks: self.state.engine_core_client().total_num_gpu_blocks(),
|
total_kv_blocks: self.state.engine_core_client().total_num_gpu_blocks(),
|
||||||
max_running_requests: ready.max_num_seqs,
|
max_running_requests: ready.max_num_seqs,
|
||||||
max_batched_tokens: ready.max_num_batched_tokens,
|
max_batched_tokens: ready.max_num_batched_tokens,
|
||||||
|
max_loras: ready.max_loras,
|
||||||
rl_capabilities: Some(self.rl_capabilities()),
|
rl_capabilities: Some(self.rl_capabilities()),
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
@@ -166,6 +167,7 @@ impl pb::control_server::Control for ControlServiceImpl {
|
|||||||
// GenerateRequest accepts both prompt representations.
|
// GenerateRequest accepts both prompt representations.
|
||||||
supports_text_input: true,
|
supports_text_input: true,
|
||||||
supports_token_ids_input: true,
|
supports_token_ids_input: true,
|
||||||
|
supports_lora: self.ready().supports_lora,
|
||||||
supports_multimodal: self.state.chat.supports_multimodal(),
|
supports_multimodal: self.state.chat.supports_multimodal(),
|
||||||
reasoning_parser: self
|
reasoning_parser: self
|
||||||
.state
|
.state
|
||||||
|
|||||||
@@ -335,6 +335,8 @@ def test_apply_ready_response_syncs_block_size():
|
|||||||
max_num_seqs=256,
|
max_num_seqs=256,
|
||||||
max_num_batched_tokens=8192,
|
max_num_batched_tokens=8192,
|
||||||
instance_id="test-instance",
|
instance_id="test-instance",
|
||||||
|
supports_lora=False,
|
||||||
|
max_loras=0,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
client._apply_ready_response(payload)
|
client._apply_ready_response(payload)
|
||||||
|
|||||||
@@ -88,6 +88,8 @@ class EngineCoreReadyResponse:
|
|||||||
max_num_seqs: int
|
max_num_seqs: int
|
||||||
max_num_batched_tokens: int
|
max_num_batched_tokens: int
|
||||||
instance_id: str
|
instance_id: str
|
||||||
|
supports_lora: bool
|
||||||
|
max_loras: int
|
||||||
# KV cache capacity (None for encoder-only/attention-free models).
|
# KV cache capacity (None for encoder-only/attention-free models).
|
||||||
kv_cache_size_tokens: int | None = None
|
kv_cache_size_tokens: int | None = None
|
||||||
kv_cache_max_concurrency: float | None = None
|
kv_cache_max_concurrency: float | None = None
|
||||||
|
|||||||
@@ -1636,6 +1636,12 @@ class EngineCoreProc(EngineCore):
|
|||||||
max_num_seqs=scheduler_config.max_num_seqs,
|
max_num_seqs=scheduler_config.max_num_seqs,
|
||||||
max_num_batched_tokens=scheduler_config.max_num_batched_tokens,
|
max_num_batched_tokens=scheduler_config.max_num_batched_tokens,
|
||||||
instance_id=self.vllm_config.instance_id,
|
instance_id=self.vllm_config.instance_id,
|
||||||
|
supports_lora=self.vllm_config.lora_config is not None,
|
||||||
|
max_loras=(
|
||||||
|
self.vllm_config.lora_config.max_loras
|
||||||
|
if self.vllm_config.lora_config is not None
|
||||||
|
else 0
|
||||||
|
),
|
||||||
kv_events_config=self.scheduler.get_kv_event_publisher_config(),
|
kv_events_config=self.scheduler.get_kv_event_publisher_config(),
|
||||||
weight_transfer_backend=(
|
weight_transfer_backend=(
|
||||||
self.vllm_config.weight_transfer_config.backend
|
self.vllm_config.weight_transfer_config.backend
|
||||||
|
|||||||
Reference in New Issue
Block a user