diff --git a/vllm/entrypoints/serve/disagg/protocol.py b/vllm/entrypoints/serve/disagg/protocol.py index 7e776ae717..2e98f5e811 100644 --- a/vllm/entrypoints/serve/disagg/protocol.py +++ b/vllm/entrypoints/serve/disagg/protocol.py @@ -181,6 +181,7 @@ class GenerateResponseStreamChoice(BaseModel): logprobs: ChatCompletionLogProbs | None = None finish_reason: str | None = None token_ids: list[int] | None = None + routed_experts: str | None = None class GenerateStreamResponse(BaseModel): diff --git a/vllm/entrypoints/serve/disagg/serving.py b/vllm/entrypoints/serve/disagg/serving.py index 0bb29c68d0..5031627ea0 100644 --- a/vllm/entrypoints/serve/disagg/serving.py +++ b/vllm/entrypoints/serve/disagg/serving.py @@ -400,6 +400,14 @@ class ServingTokens(OpenAIServing): else: logprobs = None + routed_experts_b64 = None + if output.routed_experts is not None: + buf = io.BytesIO() + np.save(buf, output.routed_experts) + routed_experts_b64 = base64.b64encode(buf.getvalue()).decode( + "ascii" + ) + chunk = GenerateStreamResponse( request_id=request_id, choices=[ @@ -408,6 +416,7 @@ class ServingTokens(OpenAIServing): logprobs=logprobs, finish_reason=finish_reason, token_ids=as_list(delta_token_ids), + routed_experts=routed_experts_b64, ) ], )