MOVZXandHinata
2066d3143f
- Remove Anthropic, Cohere, SageMaker, decisions, structured-decisions, rl, render-launcher and batch entrypoints; keep scale-out token routes - Drop the Swagger static bundle and the /docs, /redoc routes - Delete device-keyed tuning configs for fused_moe, quant utils, mamba select-state and helion; no sm86 entry ever existed for any of them - Remove ten unused rope variants plus the fbgemm_fp8 and fp_quant quantizers and their config, metrics and loader plumbing - Cut the CPU device build: csrc/cpu, cpu_extension.cmake, cpu requirements and the setup.py cpu target path - Delete 29 environment variables nothing reads and 14 chat-template fallbacks for models this fork no longer registers - Remove nine dead CUDA ops (deepseek_v4, kimi_k3, minimax, muse_spark, dsv3 gemm) with their kernels, bindings, wrappers and CMake entries - Delete the connector test suites that targeted removed connectors - Cut the Mistral tokenizer and parser stack, its tests, and the mistral_common and ijson dependency pins - Restore the Qwen4Exp registry rows and nvidia/mtp.py that an earlier cut deleted while keeping the model tree Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
101 lines
3.4 KiB
Python
101 lines
3.4 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
|
|
from typing import cast
|
|
|
|
from transformers import PreTrainedTokenizerBase
|
|
|
|
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
|
from vllm.reasoning import ReasoningParser
|
|
|
|
|
|
class StreamingReasoningReconstructor:
|
|
def __init__(self):
|
|
self.reasoning = None
|
|
self.other_content = None
|
|
|
|
def append_delta(self, delta: DeltaMessage):
|
|
# content and the reasoning content should not be present
|
|
# at the same time
|
|
assert delta.content is None or delta.reasoning is None, (
|
|
"Both content and reasoning content are present in the delta message"
|
|
)
|
|
if delta.content is not None:
|
|
if self.other_content is None:
|
|
self.other_content = delta.content
|
|
else:
|
|
self.other_content += delta.content
|
|
else:
|
|
if self.reasoning is None:
|
|
self.reasoning = delta.reasoning
|
|
else:
|
|
self.reasoning += delta.reasoning
|
|
|
|
|
|
def run_reasoning_extraction(
|
|
reasoning_parser: ReasoningParser,
|
|
model_output: list[str],
|
|
request: ChatCompletionRequest | None = None,
|
|
streaming: bool = False,
|
|
) -> tuple[str | None, str | None]:
|
|
if streaming:
|
|
reconstructor = run_reasoning_extraction_streaming(
|
|
reasoning_parser,
|
|
model_output,
|
|
request,
|
|
)
|
|
return (
|
|
reconstructor.reasoning,
|
|
reconstructor.other_content or None,
|
|
)
|
|
else:
|
|
reasoning, content = run_reasoning_extraction_nonstreaming(
|
|
reasoning_parser, model_output, request
|
|
)
|
|
return reasoning, content
|
|
|
|
|
|
def run_reasoning_extraction_nonstreaming(
|
|
reasoning_parser: ReasoningParser,
|
|
model_output: list[str],
|
|
request: ChatCompletionRequest | None = None,
|
|
) -> tuple[str | None, str | None]:
|
|
request = request or ChatCompletionRequest(messages=[], model="test-model")
|
|
return reasoning_parser.extract_reasoning(
|
|
model_output="".join(model_output), request=request
|
|
)
|
|
|
|
|
|
def run_reasoning_extraction_streaming(
|
|
reasoning_parser: ReasoningParser,
|
|
model_deltas: list[str],
|
|
request: ChatCompletionRequest | None = None,
|
|
) -> StreamingReasoningReconstructor:
|
|
request = request or ChatCompletionRequest(messages=[], model="test-model")
|
|
reconstructor = StreamingReasoningReconstructor()
|
|
previous_text = ""
|
|
previous_tokens: list[int] = []
|
|
tokenizer = cast(PreTrainedTokenizerBase, reasoning_parser.model_tokenizer)
|
|
for delta in model_deltas:
|
|
token_delta = [
|
|
reasoning_parser.vocab[token]
|
|
for token in tokenizer.tokenize(delta)
|
|
if token in reasoning_parser.vocab
|
|
]
|
|
current_text = previous_text + delta
|
|
current_tokens = previous_tokens + token_delta
|
|
delta_message = reasoning_parser.extract_reasoning_streaming(
|
|
previous_text,
|
|
current_text,
|
|
delta,
|
|
previous_tokens,
|
|
current_tokens,
|
|
token_delta,
|
|
)
|
|
if delta_message is not None:
|
|
reconstructor.append_delta(delta_message)
|
|
previous_text = current_text
|
|
previous_tokens = current_tokens
|
|
return reconstructor
|