[Feature] Mask Replay (#49577)
Signed-off-by: vx120 <893600387@qq.com> Signed-off-by: vx120 <57470515+vx120@users.noreply.github.com> Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
aoshen02
Codex
Claude Opus 4.6
OpenAI Codex
parent
10bcad2523
commit
50ba4bc6b2
@@ -123,6 +123,8 @@ pub struct EngineCoreOutput {
|
||||
/// frontend grows one.
|
||||
#[serde(default)]
|
||||
pub mm_cache_miss_hashes: Option<Vec<String>>,
|
||||
#[serde(default)]
|
||||
pub new_sampling_mask: Option<OpaqueValue>,
|
||||
}
|
||||
|
||||
impl EngineCoreOutput {
|
||||
@@ -440,6 +442,7 @@ mod tests {
|
||||
routed_experts: None,
|
||||
num_nans_in_logits: 0,
|
||||
mm_cache_miss_hashes: None,
|
||||
new_sampling_mask: None,
|
||||
},
|
||||
],
|
||||
scheduler_stats: None,
|
||||
|
||||
@@ -2498,6 +2498,8 @@ fn python_msgpack_fixtures_match_rust_encoding() {
|
||||
let defaults_request_hex = lines.next().expect("missing defaults request fixture line");
|
||||
let multimodal_request_hex = lines.next().expect("missing multimodal request fixture line");
|
||||
let outputs_hex = lines.next().expect("missing outputs fixture line");
|
||||
let sampling_mask_outputs_hex =
|
||||
lines.next().expect("missing sampling mask outputs fixture line");
|
||||
let inline_logprobs_frames = lines.next().expect("missing inline logprobs fixture line");
|
||||
let multipart_logprobs_frames = lines.next().expect("missing multipart logprobs fixture line");
|
||||
let inline_prompt_frames = lines.next().expect("missing inline prompt logprobs fixture line");
|
||||
@@ -2508,6 +2510,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
|
||||
let request_bytes = hex::decode(request_hex).unwrap();
|
||||
let multimodal_request_bytes = hex::decode(multimodal_request_hex).unwrap();
|
||||
let outputs_bytes = hex::decode(outputs_hex).unwrap();
|
||||
let sampling_mask_outputs_bytes = hex::decode(sampling_mask_outputs_hex).unwrap();
|
||||
|
||||
let decoded_request: EngineCoreRequest = rmp_serde::from_slice(&request_bytes).unwrap();
|
||||
let expected_request = sample_request();
|
||||
@@ -2571,6 +2574,16 @@ fn python_msgpack_fixtures_match_rust_encoding() {
|
||||
decode_value(&rmp_serde::to_vec_named(&expected_multimodal_request.mm_features).unwrap());
|
||||
assert_eq!(python_mm_features, rust_mm_features);
|
||||
|
||||
let decoded_sampling_mask_outputs: EngineCoreOutputs =
|
||||
rmp_serde::from_slice(&sampling_mask_outputs_bytes).unwrap();
|
||||
let sampling_mask_output =
|
||||
&decoded_sampling_mask_outputs.as_request_batch().unwrap().outputs[0];
|
||||
assert!(sampling_mask_output.mm_cache_miss_hashes.is_none());
|
||||
assert!(matches!(
|
||||
sampling_mask_output.new_sampling_mask.as_ref(),
|
||||
Some(Value::Array(fields)) if fields.len() == 3
|
||||
));
|
||||
|
||||
let decoded_outputs: EngineCoreOutputs = rmp_serde::from_slice(&outputs_bytes).unwrap();
|
||||
expect_test::expect![[r#"
|
||||
RequestBatch(
|
||||
@@ -2598,6 +2611,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
|
||||
routed_experts: None,
|
||||
num_nans_in_logits: 0,
|
||||
mm_cache_miss_hashes: None,
|
||||
new_sampling_mask: None,
|
||||
},
|
||||
],
|
||||
scheduler_stats: None,
|
||||
|
||||
@@ -98,6 +98,7 @@ class EngineCoreOutput(
|
||||
routed_experts: object | None = None
|
||||
num_nans_in_logits: int = 0
|
||||
mm_cache_miss_hashes: list[str] | None = None
|
||||
new_sampling_mask: object | None = None
|
||||
|
||||
|
||||
class EngineCoreOutputs(
|
||||
@@ -202,6 +203,29 @@ outputs = EngineCoreOutputs(
|
||||
finished_requests={"req-1"},
|
||||
)
|
||||
|
||||
sampling_mask_wire = [
|
||||
[
|
||||
"<i4",
|
||||
[5],
|
||||
msgpack.ExtType(3, np.array([2, 12, 16, 17, 18], dtype=np.int32).tobytes()),
|
||||
],
|
||||
[
|
||||
"<i8",
|
||||
[2],
|
||||
msgpack.ExtType(3, np.array([0, 5], dtype=np.int64).tobytes()),
|
||||
],
|
||||
None,
|
||||
]
|
||||
outputs_with_sampling_mask = EngineCoreOutputs(
|
||||
outputs=[
|
||||
EngineCoreOutput(
|
||||
request_id="req-mask",
|
||||
new_token_ids=[16],
|
||||
new_sampling_mask=sampling_mask_wire,
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def encode_ndarray(
|
||||
array: np.ndarray,
|
||||
@@ -422,6 +446,7 @@ print(msgspec.msgpack.encode(request).hex())
|
||||
print(msgspec.msgpack.encode(defaults_request).hex())
|
||||
print(msgpack.packb(multimodal_request_wire, use_bin_type=True).hex())
|
||||
print(msgspec.msgpack.encode(outputs).hex())
|
||||
print(msgspec.msgpack.encode(outputs_with_sampling_mask).hex())
|
||||
print(" ".join(frame.hex() for frame in encode_output_frames(inline_logprobs)))
|
||||
print(
|
||||
" ".join(
|
||||
|
||||
Reference in New Issue
Block a user