[RFC] Replace shared-memory routed experts with ModelRunnerOutput transfer and HTTP support (#39568)

Signed-off-by: xhx1022 <1737006628@qq.com>
Signed-off-by: arlenxu <arlenxu@tencent.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Co-authored-by: arlenxu <arlenxu@tencent.com>
Co-authored-by: Junjie Zhang <junj.jay.zhang@gmail.com>
This commit is contained in:
Hongxin Xu
2026-05-14 14:12:30 +00:00
committed by GitHub
co-authored by Claude Opus 4.6 arlenxu Junjie Zhang
parent 2317682f95
commit c7560af424
12 changed files with 605 additions and 369 deletions
+13 -7
View File
@@ -175,6 +175,9 @@ class RequestState:
self.stats = RequestStateStats(arrival_time=arrival_time) if log_stats else None
# Routed experts accumulation (prompt + sample chunks)
self.routed_experts_chunks: list[np.ndarray] = []
# Stream Interval
self.stream_interval = stream_interval
self.sent_tokens_offset = 0 # Offset of sent tokens
@@ -273,7 +276,6 @@ class RequestState:
finish_reason: FinishReason | None,
stop_reason: int | str | None,
kv_transfer_params: dict[str, Any] | None = None,
routed_experts: np.ndarray | None = None,
) -> RequestOutput | PoolingRequestOutput | None:
finished = finish_reason is not None
final_only = self.output_kind == RequestOutputKind.FINAL_ONLY
@@ -314,9 +316,7 @@ class RequestState:
finished,
)
output = self._new_completion_output(
new_token_ids, finish_reason, stop_reason, routed_experts
)
output = self._new_completion_output(new_token_ids, finish_reason, stop_reason)
if self.parent_req is None:
outputs = [output]
@@ -378,7 +378,6 @@ class RequestState:
token_ids: list[int],
finish_reason: FinishReason | None,
stop_reason: int | str | None,
routed_experts: np.ndarray | None = None,
) -> CompletionOutput:
assert self.detokenizer is not None
assert self.logprobs_processor is not None
@@ -395,6 +394,11 @@ class RequestState:
if delta and logprobs:
logprobs = logprobs[-len(token_ids) :]
# Concatenate routed experts on finish
routed_experts = None
if finished and self.routed_experts_chunks:
routed_experts = np.concatenate(self.routed_experts_chunks, axis=0)
return CompletionOutput(
index=self.request_index,
text=text,
@@ -616,7 +620,10 @@ class OutputProcessor:
finish_reason = engine_core_output.finish_reason
stop_reason = engine_core_output.stop_reason
kv_transfer_params = engine_core_output.kv_transfer_params
routed_experts = engine_core_output.routed_experts
if engine_core_output.routed_experts is not None:
req_state.routed_experts_chunks.append(
engine_core_output.routed_experts
)
if req_state.is_prefilling:
if engine_core_output.prefill_stats is not None:
@@ -647,7 +654,6 @@ class OutputProcessor:
finish_reason,
stop_reason,
kv_transfer_params,
routed_experts,
):
if req_state.streaming_input:
request_output.finished = False