[Rust Frontend][RL] add routed expert prompt offset (#52703)
Signed-off-by: Biswa Panda <biswa.panda@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
8d6b18329a
commit
6a391a931a
@@ -145,6 +145,9 @@ pub struct EngineCoreSamplingParams {
|
||||
pub skip_reading_prefix_cache: Option<bool>,
|
||||
/// Additional request parameters for custom extensions (from `vllm_xargs`).
|
||||
pub extra_args: Option<HashMap<String, serde_json::Value>>,
|
||||
/// Number of prompt tokens to skip from returned routed-expert data.
|
||||
/// A value of zero returns routing data for the entire prompt.
|
||||
pub routed_experts_prompt_start: u32,
|
||||
}
|
||||
|
||||
impl EngineCoreSamplingParams {
|
||||
@@ -175,6 +178,7 @@ impl EngineCoreSamplingParams {
|
||||
logprob_token_ids: None,
|
||||
skip_reading_prefix_cache: None,
|
||||
extra_args: None,
|
||||
routed_experts_prompt_start: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -159,6 +159,7 @@ fn sample_request_with_id(request_id: &str) -> EngineCoreRequest {
|
||||
stop_token_ids: vec![151643],
|
||||
eos_token_id: Some(151645),
|
||||
all_stop_token_ids: BTreeSet::from([151643, 151645]),
|
||||
routed_experts_prompt_start: 1,
|
||||
..EngineCoreSamplingParams::for_test()
|
||||
}),
|
||||
arrival_time: 42.5,
|
||||
@@ -2680,6 +2681,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
|
||||
logprob_token_ids: None,
|
||||
skip_reading_prefix_cache: None,
|
||||
extra_args: None,
|
||||
routed_experts_prompt_start: 0,
|
||||
},
|
||||
);
|
||||
|
||||
|
||||
@@ -47,6 +47,7 @@ class EngineCoreSamplingParams(msgspec.Struct, dict=True, omit_defaults=True):
|
||||
stop_token_ids: list[int] = []
|
||||
_eos_token_id: int | None = None
|
||||
_all_stop_token_ids: set[int] = set()
|
||||
routed_experts_prompt_start: int = 0
|
||||
output_kind: RequestOutputKind = RequestOutputKind.DELTA
|
||||
|
||||
|
||||
@@ -135,6 +136,7 @@ request = EngineCoreRequest(
|
||||
stop_token_ids=[151643],
|
||||
_eos_token_id=151645,
|
||||
_all_stop_token_ids={151643, 151645},
|
||||
routed_experts_prompt_start=1,
|
||||
output_kind=RequestOutputKind.FINAL_ONLY,
|
||||
),
|
||||
pooling_params=None,
|
||||
|
||||
@@ -188,6 +188,7 @@ pub fn lower_sampling_params(
|
||||
logprob_token_ids,
|
||||
skip_reading_prefix_cache,
|
||||
extra_args: vllm_xargs,
|
||||
routed_experts_prompt_start: 0,
|
||||
};
|
||||
validate_resolved_sampling_params(¶ms)?;
|
||||
validate_vocab_range(¶ms, &sampling_limits)?;
|
||||
@@ -645,6 +646,7 @@ mod tests {
|
||||
logprob_token_ids: None,
|
||||
skip_reading_prefix_cache: None,
|
||||
extra_args: None,
|
||||
routed_experts_prompt_start: 0,
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(¶ms);
|
||||
@@ -694,6 +696,7 @@ mod tests {
|
||||
logprob_token_ids: None,
|
||||
skip_reading_prefix_cache: None,
|
||||
extra_args: None,
|
||||
routed_experts_prompt_start: 0,
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(¶ms);
|
||||
@@ -859,6 +862,7 @@ mod tests {
|
||||
logprob_token_ids: None,
|
||||
skip_reading_prefix_cache: None,
|
||||
extra_args: None,
|
||||
routed_experts_prompt_start: 0,
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(¶ms);
|
||||
@@ -926,6 +930,7 @@ mod tests {
|
||||
logprob_token_ids: None,
|
||||
skip_reading_prefix_cache: None,
|
||||
extra_args: None,
|
||||
routed_experts_prompt_start: 0,
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(¶ms);
|
||||
@@ -986,6 +991,7 @@ mod tests {
|
||||
logprob_token_ids: None,
|
||||
skip_reading_prefix_cache: None,
|
||||
extra_args: None,
|
||||
routed_experts_prompt_start: 0,
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(¶ms);
|
||||
@@ -1235,6 +1241,7 @@ mod tests {
|
||||
logprob_token_ids: None,
|
||||
skip_reading_prefix_cache: None,
|
||||
extra_args: None,
|
||||
routed_experts_prompt_start: 0,
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(¶ms);
|
||||
|
||||
Reference in New Issue
Block a user