[SpecDecode] Add LiLiCorr drafter (#57934)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Signed-off-by: Andrii Skliar <andreyws96@gmail.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
This commit is contained in:
co-authored by
Andrii Skliar
Codex
Benjamin Chislett
parent
25b0add7b8
commit
73a78e6f1f
@@ -176,6 +176,7 @@ class DFlashGroupedConv(nn.Module):
|
||||
block_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
prefix: str,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if hidden_size % group_size:
|
||||
@@ -195,7 +196,7 @@ class DFlashGroupedConv(nn.Module):
|
||||
2 * taps * self.num_groups,
|
||||
bias=False,
|
||||
params_dtype=params_dtype,
|
||||
quant_config=None,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "kernel_projection"),
|
||||
return_bias=False,
|
||||
)
|
||||
@@ -254,6 +255,7 @@ class DFlash2Qwen3DecoderLayer(DFlashQwen3DecoderLayer):
|
||||
# Query tokens per request: the bonus token plus the mask tokens.
|
||||
block_size=1 + speculative_config.num_speculative_tokens,
|
||||
params_dtype=vllm_config.model_config.dtype,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
self.attention_conv = DFlashGroupedConv(
|
||||
**conv_args, prefix=maybe_prefix(prefix, "attention_conv")
|
||||
|
||||
Reference in New Issue
Block a user