[MRV2] Enable routed-experts capture (#50721)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -202,10 +202,10 @@ def detach_zero_copy_from_model_runner_output(output: "ModelRunnerOutput") -> No
|
||||
backed by Ray's shared-memory object store. Ray's channel docs explicitly
|
||||
warn that subsequent reads may block if such an object is still in scope.
|
||||
|
||||
vLLM can return numpy-backed logprobs in `ModelRunnerOutput.logprobs`. If
|
||||
those arrays are backed by Ray SHM (commonly read-only), retaining them in
|
||||
scope across scheduler iterations can stall the channel and eventually hit
|
||||
`RAY_CGRAPH_get_timeout`.
|
||||
vLLM can return numpy-backed logprobs and routed experts in
|
||||
`ModelRunnerOutput`. If those arrays are backed by Ray SHM (commonly
|
||||
read-only), retaining them in scope across scheduler iterations can stall
|
||||
the channel and eventually hit `RAY_CGRAPH_get_timeout`.
|
||||
|
||||
Copy read-only numpy arrays so the returned output no longer retains
|
||||
references to Ray's shared-memory buffers.
|
||||
@@ -214,27 +214,37 @@ def detach_zero_copy_from_model_runner_output(output: "ModelRunnerOutput") -> No
|
||||
`LogprobsTensors` backed by PyTorch-owned CPU tensors (`to_cpu_nonblocking`
|
||||
or `empty_cpu`), not NumPy views decoded from Ray channels.
|
||||
"""
|
||||
if output.logprobs is None:
|
||||
return
|
||||
|
||||
token_ids, logprobs, ranks, cu_num_generated_tokens = output.logprobs
|
||||
|
||||
def _copy_if_readonly(arr):
|
||||
if isinstance(arr, np.ndarray) and not arr.flags.writeable:
|
||||
return arr.copy()
|
||||
return arr
|
||||
|
||||
# `cu_num_generated_tokens` is already a plain Python list (or None), so it
|
||||
# never aliases Ray SHM buffers and can be reused as-is.
|
||||
token_ids_c = _copy_if_readonly(token_ids)
|
||||
logprobs_c = _copy_if_readonly(logprobs)
|
||||
ranks_c = _copy_if_readonly(ranks)
|
||||
if token_ids_c is token_ids and logprobs_c is logprobs and ranks_c is ranks:
|
||||
return
|
||||
if output.logprobs is not None:
|
||||
token_ids, logprobs, ranks, cu_num_generated_tokens = output.logprobs
|
||||
|
||||
output.logprobs = type(output.logprobs)(
|
||||
token_ids_c, logprobs_c, ranks_c, cu_num_generated_tokens
|
||||
)
|
||||
# `cu_num_generated_tokens` is already a plain Python list (or None),
|
||||
# so it never aliases Ray SHM buffers and can be reused as-is.
|
||||
token_ids_c = _copy_if_readonly(token_ids)
|
||||
logprobs_c = _copy_if_readonly(logprobs)
|
||||
ranks_c = _copy_if_readonly(ranks)
|
||||
if (
|
||||
token_ids_c is not token_ids
|
||||
or logprobs_c is not logprobs
|
||||
or ranks_c is not ranks
|
||||
):
|
||||
output.logprobs = type(output.logprobs)(
|
||||
token_ids_c, logprobs_c, ranks_c, cu_num_generated_tokens
|
||||
)
|
||||
|
||||
if output.routed_experts is not None:
|
||||
routing_data, slot_mapping = output.routed_experts
|
||||
routing_data_c = _copy_if_readonly(routing_data)
|
||||
slot_mapping_c = _copy_if_readonly(slot_mapping)
|
||||
if routing_data_c is not routing_data or slot_mapping_c is not slot_mapping:
|
||||
output.routed_experts = type(output.routed_experts)(
|
||||
routing_data_c, slot_mapping_c
|
||||
)
|
||||
|
||||
|
||||
class FutureWrapper(Future):
|
||||
|
||||
Reference in New Issue
Block a user