[RFC] Replace shared-memory routed experts with ModelRunnerOutput transfer and HTTP support (#39568)

Signed-off-by: xhx1022 <1737006628@qq.com>
Signed-off-by: arlenxu <arlenxu@tencent.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Co-authored-by: arlenxu <arlenxu@tencent.com>
Co-authored-by: Junjie Zhang <junj.jay.zhang@gmail.com>
This commit is contained in:
Hongxin Xu
2026-05-14 14:12:30 +00:00
committed by GitHub
co-authored by Claude Opus 4.6 arlenxu Junjie Zhang
parent 2317682f95
commit c7560af424
12 changed files with 605 additions and 369 deletions
+10
View File
@@ -160,6 +160,16 @@ class GenerateResponseChoice(BaseModel):
# per OpenAI spec this is the default
finish_reason: str | None = "stop"
token_ids: list[int] | None = None
# Per-token expert routing decisions, base64-encoded ``.npy`` bytes
# (numpy serialization). Shape after decode:
# (num_tokens - 1, num_layers, num_experts_per_tok) dtype uint8/uint16
# ``num_tokens - 1`` because the last sampled token has not been
# forwarded yet and therefore has no routing data.
# Decode:
# np.load(io.BytesIO(base64.b64decode(s)))
# ``None`` if (a) the request was aborted before any forward pass,
# or (b) ``enable_return_routed_experts`` is off server-side.
routed_experts: str | None = None
class GenerateResponseStreamChoice(BaseModel):
+16
View File
@@ -3,10 +3,13 @@
import asyncio
import io
import time
from collections.abc import AsyncGenerator
from collections.abc import Sequence as GenericSequence
import numpy as np
import pybase64 as base64
from fastapi import Request
from vllm.engine.protocol import EngineClient
@@ -255,11 +258,24 @@ class ServingTokens(OpenAIServing):
else:
logprobs = None
# Encode routed_experts for transport. JSON can't carry raw
# bytes, so we write the ndarray as a ``.npy`` byte stream
# and base64-encode it. ``pybase64`` is ~3x faster than the
# stdlib ``base64`` on large payloads thanks to SIMD.
# This is the only base64 hop in the pipeline -- the
# engine<->API-server link is binary msgpack + zmq.
routed_experts_b64 = None
if output.routed_experts is not None:
buf = io.BytesIO()
np.save(buf, output.routed_experts)
routed_experts_b64 = base64.b64encode(buf.getvalue()).decode("ascii")
choice_data = GenerateResponseChoice(
index=output.index,
logprobs=logprobs,
finish_reason=output.finish_reason if output.finish_reason else "stop",
token_ids=as_list(output.token_ids),
routed_experts=routed_experts_b64,
)
choices.append(choice_data)