[RFC] Replace shared-memory routed experts with ModelRunnerOutput transfer and HTTP support (#39568)
Signed-off-by: xhx1022 <1737006628@qq.com> Signed-off-by: arlenxu <arlenxu@tencent.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Co-authored-by: arlenxu <arlenxu@tencent.com> Co-authored-by: Junjie Zhang <junj.jay.zhang@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
arlenxu
Junjie Zhang
parent
2317682f95
commit
c7560af424
@@ -175,6 +175,9 @@ class RequestState:
|
||||
|
||||
self.stats = RequestStateStats(arrival_time=arrival_time) if log_stats else None
|
||||
|
||||
# Routed experts accumulation (prompt + sample chunks)
|
||||
self.routed_experts_chunks: list[np.ndarray] = []
|
||||
|
||||
# Stream Interval
|
||||
self.stream_interval = stream_interval
|
||||
self.sent_tokens_offset = 0 # Offset of sent tokens
|
||||
@@ -273,7 +276,6 @@ class RequestState:
|
||||
finish_reason: FinishReason | None,
|
||||
stop_reason: int | str | None,
|
||||
kv_transfer_params: dict[str, Any] | None = None,
|
||||
routed_experts: np.ndarray | None = None,
|
||||
) -> RequestOutput | PoolingRequestOutput | None:
|
||||
finished = finish_reason is not None
|
||||
final_only = self.output_kind == RequestOutputKind.FINAL_ONLY
|
||||
@@ -314,9 +316,7 @@ class RequestState:
|
||||
finished,
|
||||
)
|
||||
|
||||
output = self._new_completion_output(
|
||||
new_token_ids, finish_reason, stop_reason, routed_experts
|
||||
)
|
||||
output = self._new_completion_output(new_token_ids, finish_reason, stop_reason)
|
||||
|
||||
if self.parent_req is None:
|
||||
outputs = [output]
|
||||
@@ -378,7 +378,6 @@ class RequestState:
|
||||
token_ids: list[int],
|
||||
finish_reason: FinishReason | None,
|
||||
stop_reason: int | str | None,
|
||||
routed_experts: np.ndarray | None = None,
|
||||
) -> CompletionOutput:
|
||||
assert self.detokenizer is not None
|
||||
assert self.logprobs_processor is not None
|
||||
@@ -395,6 +394,11 @@ class RequestState:
|
||||
if delta and logprobs:
|
||||
logprobs = logprobs[-len(token_ids) :]
|
||||
|
||||
# Concatenate routed experts on finish
|
||||
routed_experts = None
|
||||
if finished and self.routed_experts_chunks:
|
||||
routed_experts = np.concatenate(self.routed_experts_chunks, axis=0)
|
||||
|
||||
return CompletionOutput(
|
||||
index=self.request_index,
|
||||
text=text,
|
||||
@@ -616,7 +620,10 @@ class OutputProcessor:
|
||||
finish_reason = engine_core_output.finish_reason
|
||||
stop_reason = engine_core_output.stop_reason
|
||||
kv_transfer_params = engine_core_output.kv_transfer_params
|
||||
routed_experts = engine_core_output.routed_experts
|
||||
if engine_core_output.routed_experts is not None:
|
||||
req_state.routed_experts_chunks.append(
|
||||
engine_core_output.routed_experts
|
||||
)
|
||||
|
||||
if req_state.is_prefilling:
|
||||
if engine_core_output.prefill_stats is not None:
|
||||
@@ -647,7 +654,6 @@ class OutputProcessor:
|
||||
finish_reason,
|
||||
stop_reason,
|
||||
kv_transfer_params,
|
||||
routed_experts,
|
||||
):
|
||||
if req_state.streaming_input:
|
||||
request_output.finished = False
|
||||
|
||||
Reference in New Issue
Block a user