[Feature] Mask Replay (#49577)

Signed-off-by: vx120 <893600387@qq.com>
Signed-off-by: vx120 <57470515+vx120@users.noreply.github.com>
Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
vx120
2026-08-13 08:44:31 +00:00
committed by GitHub
co-authored by aoshen02 Codex Claude Opus 4.6 OpenAI Codex
parent 10bcad2523
commit 50ba4bc6b2
24 changed files with 557 additions and 6 deletions
@@ -123,6 +123,8 @@ pub struct EngineCoreOutput {
/// frontend grows one.
#[serde(default)]
pub mm_cache_miss_hashes: Option<Vec<String>>,
#[serde(default)]
pub new_sampling_mask: Option<OpaqueValue>,
}
impl EngineCoreOutput {
@@ -440,6 +442,7 @@ mod tests {
routed_experts: None,
num_nans_in_logits: 0,
mm_cache_miss_hashes: None,
new_sampling_mask: None,
},
],
scheduler_stats: None,
@@ -2498,6 +2498,8 @@ fn python_msgpack_fixtures_match_rust_encoding() {
let defaults_request_hex = lines.next().expect("missing defaults request fixture line");
let multimodal_request_hex = lines.next().expect("missing multimodal request fixture line");
let outputs_hex = lines.next().expect("missing outputs fixture line");
let sampling_mask_outputs_hex =
lines.next().expect("missing sampling mask outputs fixture line");
let inline_logprobs_frames = lines.next().expect("missing inline logprobs fixture line");
let multipart_logprobs_frames = lines.next().expect("missing multipart logprobs fixture line");
let inline_prompt_frames = lines.next().expect("missing inline prompt logprobs fixture line");
@@ -2508,6 +2510,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
let request_bytes = hex::decode(request_hex).unwrap();
let multimodal_request_bytes = hex::decode(multimodal_request_hex).unwrap();
let outputs_bytes = hex::decode(outputs_hex).unwrap();
let sampling_mask_outputs_bytes = hex::decode(sampling_mask_outputs_hex).unwrap();
let decoded_request: EngineCoreRequest = rmp_serde::from_slice(&request_bytes).unwrap();
let expected_request = sample_request();
@@ -2571,6 +2574,16 @@ fn python_msgpack_fixtures_match_rust_encoding() {
decode_value(&rmp_serde::to_vec_named(&expected_multimodal_request.mm_features).unwrap());
assert_eq!(python_mm_features, rust_mm_features);
let decoded_sampling_mask_outputs: EngineCoreOutputs =
rmp_serde::from_slice(&sampling_mask_outputs_bytes).unwrap();
let sampling_mask_output =
&decoded_sampling_mask_outputs.as_request_batch().unwrap().outputs[0];
assert!(sampling_mask_output.mm_cache_miss_hashes.is_none());
assert!(matches!(
sampling_mask_output.new_sampling_mask.as_ref(),
Some(Value::Array(fields)) if fields.len() == 3
));
let decoded_outputs: EngineCoreOutputs = rmp_serde::from_slice(&outputs_bytes).unwrap();
expect_test::expect![[r#"
RequestBatch(
@@ -2598,6 +2611,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
routed_experts: None,
num_nans_in_logits: 0,
mm_cache_miss_hashes: None,
new_sampling_mask: None,
},
],
scheduler_stats: None,
@@ -98,6 +98,7 @@ class EngineCoreOutput(
routed_experts: object | None = None
num_nans_in_logits: int = 0
mm_cache_miss_hashes: list[str] | None = None
new_sampling_mask: object | None = None
class EngineCoreOutputs(
@@ -202,6 +203,29 @@ outputs = EngineCoreOutputs(
finished_requests={"req-1"},
)
sampling_mask_wire = [
[
"<i4",
[5],
msgpack.ExtType(3, np.array([2, 12, 16, 17, 18], dtype=np.int32).tobytes()),
],
[
"<i8",
[2],
msgpack.ExtType(3, np.array([0, 5], dtype=np.int64).tobytes()),
],
None,
]
outputs_with_sampling_mask = EngineCoreOutputs(
outputs=[
EngineCoreOutput(
request_id="req-mask",
new_token_ids=[16],
new_sampling_mask=sampling_mask_wire,
)
]
)
def encode_ndarray(
array: np.ndarray,
@@ -422,6 +446,7 @@ print(msgspec.msgpack.encode(request).hex())
print(msgspec.msgpack.encode(defaults_request).hex())
print(msgpack.packb(multimodal_request_wire, use_bin_type=True).hex())
print(msgspec.msgpack.encode(outputs).hex())
print(msgspec.msgpack.encode(outputs_with_sampling_mask).hex())
print(" ".join(frame.hex() for frame in encode_output_frames(inline_logprobs)))
print(
" ".join(