[Model] Support every Decision 3.0 size (d3-flash, d3-mini, d3-nano, d3-lite, d3-edge) (#61778)
Signed-off-by: Xunzhuo Liu <xunzhuo.liu@amd.com>
This commit is contained in:
@@ -304,12 +304,16 @@ vllm serve Cloudflare/clef-omni
|
||||
|
||||
### Decision 3.0
|
||||
|
||||
The Decision 3.0 models ([`vllm-sr/d3`](https://huggingface.co/vllm-sr/d3) and
|
||||
its smaller variants such as `vllm-sr/d3-mini` and `vllm-sr/d3-edge`) are a
|
||||
Qwen3.5 backbone with an answer-code readout: one forward pass per question,
|
||||
and the readout scores the question's options at the last token. vLLM detects
|
||||
the checkpoint's `decision_config.json` and serves it as a classification
|
||||
model.
|
||||
The Decision 3.0 models are a Qwen3.5 backbone with an answer-code readout:
|
||||
one forward pass per question, and the readout scores the question's options
|
||||
at the last token. vLLM detects the checkpoint's `decision_config.json` and
|
||||
serves it as a classification model. All six sizes take the same command:
|
||||
[`vllm-sr/d3`](https://huggingface.co/vllm-sr/d3) (27B),
|
||||
[`vllm-sr/d3-flash`](https://huggingface.co/vllm-sr/d3-flash) (9B),
|
||||
[`vllm-sr/d3-mini`](https://huggingface.co/vllm-sr/d3-mini) (4B),
|
||||
[`vllm-sr/d3-nano`](https://huggingface.co/vllm-sr/d3-nano) (2B),
|
||||
[`vllm-sr/d3-lite`](https://huggingface.co/vllm-sr/d3-lite) (0.8B) and
|
||||
[`vllm-sr/d3-edge`](https://huggingface.co/vllm-sr/d3-edge) (0.6B).
|
||||
|
||||
```bash
|
||||
vllm serve vllm-sr/d3
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""d3-edge against the checkpoint's own runtime (`trust_remote_code`).
|
||||
"""Decision 3.0 against the checkpoint's own runtime (`trust_remote_code`).
|
||||
|
||||
The runtime reads a question's options from the 255 code logits of its readout
|
||||
at the last prompt token. vLLM's classify head must give the same code logits
|
||||
for the same prompt tokens, and for an image prompt that vLLM processes itself.
|
||||
d3-edge has an irregular layer layout; d3-nano ships its backbone as a single
|
||||
weight file without an index, next to the readout file.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
@@ -14,7 +16,7 @@ from transformers import AutoModel
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
|
||||
MODEL = "vllm-sr/d3-edge"
|
||||
MODELS = ["vllm-sr/d3-edge", "vllm-sr/d3-nano"]
|
||||
STATE = (
|
||||
"The order arrived damaged yesterday. The customer has a receipt and asks "
|
||||
"for a replacement today."
|
||||
@@ -47,7 +49,7 @@ def option_probs(logits: torch.Tensor, count: int) -> torch.Tensor:
|
||||
return torch.softmax(logits[:count].float(), dim=-1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", [MODEL])
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
def test_d3_matches_the_checkpoint_runtime(vllm_runner, model: str) -> None:
|
||||
reference = AutoModel.from_pretrained(model, trust_remote_code=True)
|
||||
runtime = reference.runtime
|
||||
|
||||
@@ -731,7 +731,13 @@ _SEQUENCE_CLASSIFICATION_EXAMPLE_MODELS = {
|
||||
),
|
||||
"D3ForSequenceClassification": _HfExamplesInfo(
|
||||
"vllm-sr/d3-edge",
|
||||
extras={"d3": "vllm-sr/d3", "d3-mini": "vllm-sr/d3-mini"},
|
||||
extras={
|
||||
"d3": "vllm-sr/d3",
|
||||
"d3-flash": "vllm-sr/d3-flash",
|
||||
"d3-mini": "vllm-sr/d3-mini",
|
||||
"d3-nano": "vllm-sr/d3-nano",
|
||||
"d3-lite": "vllm-sr/d3-lite",
|
||||
},
|
||||
max_model_len=4096,
|
||||
),
|
||||
"GPT2ForSequenceClassification": _HfExamplesInfo(
|
||||
|
||||
@@ -81,3 +81,8 @@ class D3ForSequenceClassification(_D3ForSequenceClassificationBase):
|
||||
allow_patterns_overrides=[READOUT_WEIGHTS],
|
||||
)
|
||||
]
|
||||
|
||||
def is_unused_checkpoint_weight(self, name: str) -> bool:
|
||||
# The readout's only tensor, loaded through `secondary_weights`. A
|
||||
# checkpoint without a weight index lists its file with the backbone's.
|
||||
return name == "weight"
|
||||
|
||||
Reference in New Issue
Block a user