[MRV2] Enable routed-experts capture (#50721)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
aoshen02
2026-08-03 21:00:13 +00:00
committed by GitHub
co-authored by Nick Hill
parent e279f71583
commit 42ab184ea7
10 changed files with 335 additions and 227 deletions
+28 -18
View File
@@ -202,10 +202,10 @@ def detach_zero_copy_from_model_runner_output(output: "ModelRunnerOutput") -> No
backed by Ray's shared-memory object store. Ray's channel docs explicitly
warn that subsequent reads may block if such an object is still in scope.
vLLM can return numpy-backed logprobs in `ModelRunnerOutput.logprobs`. If
those arrays are backed by Ray SHM (commonly read-only), retaining them in
scope across scheduler iterations can stall the channel and eventually hit
`RAY_CGRAPH_get_timeout`.
vLLM can return numpy-backed logprobs and routed experts in
`ModelRunnerOutput`. If those arrays are backed by Ray SHM (commonly
read-only), retaining them in scope across scheduler iterations can stall
the channel and eventually hit `RAY_CGRAPH_get_timeout`.
Copy read-only numpy arrays so the returned output no longer retains
references to Ray's shared-memory buffers.
@@ -214,27 +214,37 @@ def detach_zero_copy_from_model_runner_output(output: "ModelRunnerOutput") -> No
`LogprobsTensors` backed by PyTorch-owned CPU tensors (`to_cpu_nonblocking`
or `empty_cpu`), not NumPy views decoded from Ray channels.
"""
if output.logprobs is None:
return
token_ids, logprobs, ranks, cu_num_generated_tokens = output.logprobs
def _copy_if_readonly(arr):
if isinstance(arr, np.ndarray) and not arr.flags.writeable:
return arr.copy()
return arr
# `cu_num_generated_tokens` is already a plain Python list (or None), so it
# never aliases Ray SHM buffers and can be reused as-is.
token_ids_c = _copy_if_readonly(token_ids)
logprobs_c = _copy_if_readonly(logprobs)
ranks_c = _copy_if_readonly(ranks)
if token_ids_c is token_ids and logprobs_c is logprobs and ranks_c is ranks:
return
if output.logprobs is not None:
token_ids, logprobs, ranks, cu_num_generated_tokens = output.logprobs
output.logprobs = type(output.logprobs)(
token_ids_c, logprobs_c, ranks_c, cu_num_generated_tokens
)
# `cu_num_generated_tokens` is already a plain Python list (or None),
# so it never aliases Ray SHM buffers and can be reused as-is.
token_ids_c = _copy_if_readonly(token_ids)
logprobs_c = _copy_if_readonly(logprobs)
ranks_c = _copy_if_readonly(ranks)
if (
token_ids_c is not token_ids
or logprobs_c is not logprobs
or ranks_c is not ranks
):
output.logprobs = type(output.logprobs)(
token_ids_c, logprobs_c, ranks_c, cu_num_generated_tokens
)
if output.routed_experts is not None:
routing_data, slot_mapping = output.routed_experts
routing_data_c = _copy_if_readonly(routing_data)
slot_mapping_c = _copy_if_readonly(slot_mapping)
if routing_data_c is not routing_data or slot_mapping_c is not slot_mapping:
output.routed_experts = type(output.routed_experts)(
routing_data_c, slot_mapping_c
)
class FutureWrapper(Future):