[ModelRunner V2] Support realtime embeddings (#46762)

This commit is contained in:
Nick Hill
2026-06-26 19:42:27 -07:00
committed by GitHub
parent af16446bf3
commit c6dd32a810
6 changed files with 52 additions and 36 deletions
+3 -3
View File
@@ -53,14 +53,14 @@ def _make_runner(
def _gather(runner: EncoderRunner, *, num_scheduled: int, draft_lookahead: int):
# Single prefilling request, computed_prefill=0, prefill_len large.
# Single prefilling request, num_computed_tokens=0, prefill_len large.
return runner.gather_mm_embeddings(
req_ids=["req0"],
total_num_scheduled_tokens=num_scheduled,
num_scheduled_tokens=np.array([num_scheduled]),
query_start_loc=np.array([0]),
prefill_lens=np.array([1000]),
computed_prefill_lens=np.array([0]),
num_computed_tokens=np.array([0]),
draft_lookahead=draft_lookahead,
)
@@ -147,7 +147,7 @@ def test_multi_request_batch_gathers_per_request(draft_lookahead):
num_scheduled_tokens=np.array([8, 8]),
query_start_loc=np.array([0, 8]),
prefill_lens=np.array([1000, 1000]),
computed_prefill_lens=np.array([0, 0]),
num_computed_tokens=np.array([0, 0]),
draft_lookahead=draft_lookahead,
)