Support quantized DSpark Markov heads (#50424)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: Benjamin Chislett <bchislett@nvidia.com>
This commit is contained in:
co-authored by
Andrii Skliar
Benjamin Chislett
parent
ad0bf3963e
commit
b977407d8b
@@ -22,6 +22,7 @@ import torch.nn as nn
|
|||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||||
|
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||||
ParallelLMHead,
|
ParallelLMHead,
|
||||||
)
|
)
|
||||||
@@ -51,6 +52,7 @@ class DSparkMarkovHead(nn.Module):
|
|||||||
draft_vocab_size: int,
|
draft_vocab_size: int,
|
||||||
markov_rank: int,
|
markov_rank: int,
|
||||||
prefix: str,
|
prefix: str,
|
||||||
|
quant_config: QuantizationConfig | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.markov_w1 = nn.Embedding(vocab_size, markov_rank)
|
self.markov_w1 = nn.Embedding(vocab_size, markov_rank)
|
||||||
@@ -58,6 +60,7 @@ class DSparkMarkovHead(nn.Module):
|
|||||||
draft_vocab_size,
|
draft_vocab_size,
|
||||||
markov_rank,
|
markov_rank,
|
||||||
bias=False,
|
bias=False,
|
||||||
|
quant_config=quant_config,
|
||||||
prefix=maybe_prefix(prefix, "markov_w2"),
|
prefix=maybe_prefix(prefix, "markov_w2"),
|
||||||
disable_tp=True,
|
disable_tp=True,
|
||||||
)
|
)
|
||||||
@@ -97,6 +100,7 @@ class Qwen3DSparkModel(DFlashQwen3Model):
|
|||||||
draft_vocab_size,
|
draft_vocab_size,
|
||||||
config.markov_rank,
|
config.markov_rank,
|
||||||
prefix=maybe_prefix(prefix, "markov_head"),
|
prefix=maybe_prefix(prefix, "markov_head"),
|
||||||
|
quant_config=self.quant_config,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user