vLLM/vllm/lora/layers/classifier.py

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project


import torch
from transformers import PreTrainedConfig

from vllm.config.lora import LoRAConfig
from vllm.platforms import current_platform

from .replicated_linear import ReplicatedLinearWithLoRA


class ClassificationHeadWithLoRA(ReplicatedLinearWithLoRA):
    def create_lora_weights(
        self,
        max_loras: int,
        lora_config: LoRAConfig,
        model_config: PreTrainedConfig | None = None,
    ) -> None:
        # Preserve ordinary LoRA A/B support for classification heads.
        super().create_lora_weights(max_loras, lora_config, model_config)

        self.max_lora_cls_labels = lora_config.max_lora_cls_labels or self.output_size
        self.padded_num_labels = max(self.output_size, self.max_lora_cls_labels)
        self.full_weight_stacked = torch.zeros(
            max_loras,
            1,
            self.padded_num_labels,
            self.input_size,
            dtype=lora_config.lora_dtype,
            device=self.device,
        )
        self.full_bias_stacked = torch.zeros(
            max_loras,
            self.padded_num_labels,
            dtype=self.base_layer.params_dtype,
            device=self.device,
        )
        self.full_module_enabled = torch.zeros(
            max_loras, dtype=torch.bool, device=self.device
        )
        # Zero marks an unused slot; positive values are unpadded label counts.
        self.full_module_num_labels = [0] * max_loras
        self._output_lora_indices: tuple[int, ...] = ()

    def set_output_mapping(self, slot_indices: tuple[int, ...]) -> None:
        self._output_lora_indices = slot_indices

    def reset_module_to_save(self, index: int) -> None:
        self.full_weight_stacked[index].zero_()
        self.full_bias_stacked[index].zero_()
        self.full_module_enabled[index] = False
        self.full_module_num_labels[index] = 0

    def set_module_to_save(
        self,
        index: int,
        weight: torch.Tensor,
        bias: torch.Tensor | None,
    ) -> None:
        self.reset_module_to_save(index)
        num_labels = weight.size(0)
        self.full_weight_stacked[index, 0, :num_labels].copy_(weight, non_blocking=True)
        if bias is not None:
            self.full_bias_stacked[index, :num_labels].copy_(bias, non_blocking=True)
        self.full_module_num_labels[index] = num_labels
        self.full_module_enabled[index] = True

    def forward(self, input_: torch.Tensor) -> torch.Tensor | list[torch.Tensor]:
        # TODO base_result maybe don't need compute truly.
        base_result = super().forward(input_)
        base_result = base_result[0] if isinstance(base_result, tuple) else base_result

        slot_indices = self._output_lora_indices
        assert len(slot_indices) == base_result.size(0), (
            "Classification rows do not match LoRA request mapping"
        )
        num_labels = [
            self.full_module_num_labels[index] if index >= 0 else 0
            for index in slot_indices
        ]

        if not any(num_labels):
            return base_result

        output = base_result.new_zeros(base_result.size(0), self.padded_num_labels)
        output[:, : self.output_size].copy_(base_result)
        self.punica_wrapper.apply_lora_full_linear(
            output,
            input_.to(self.full_weight_stacked.dtype),
            self.full_weight_stacked,
            self.full_bias_stacked,
            self.full_module_enabled,
        )

        # Rows without a full head keep the base classifier output size.
        num_labels = [size or self.output_size for size in num_labels]
        if len(set(num_labels)) == 1:
            return output[:, : num_labels[0]]

        output = [output[row, :size] for row, size in enumerate(num_labels)]
        return output

    def _apply_lora_to_output(
        self, x: torch.Tensor, output: torch.Tensor
    ) -> torch.Tensor:
        original_shape = output.shape if output.ndim == 3 else None
        if x.ndim == 3 and output.ndim == 3:
            output = output.flatten(0, 1)
            x = x.flatten(0, 1)

        # Classification outputs are request-level, so use prompt-based LoRA routing.
        lora_output: torch.Tensor | None = self.punica_wrapper.add_lora_logits(
            output,
            x.to(self.lora_a_stacked[0].dtype),
            self.lora_a_stacked[0],
            self.lora_b_stacked[0],
            1.0,
        )
        if not current_platform.can_update_inplace():
            output = lora_output

        if original_shape is not None:
            output = output.reshape(original_shape)

        return output