[Bugfix] Propagate quant_config to LFM2 ShortConv projections (#48917)
Signed-off-by: Alex Yuan <alex.yuan@liquid.ai> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
616c9bd0f4
commit
8def3cdde2
@@ -23,6 +23,7 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
|
|||||||
causal_conv1d_fn,
|
causal_conv1d_fn,
|
||||||
causal_conv1d_update,
|
causal_conv1d_update,
|
||||||
)
|
)
|
||||||
|
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
from vllm.utils.torch_utils import direct_register_custom_op
|
from vllm.utils.torch_utils import direct_register_custom_op
|
||||||
from vllm.v1.attention.backend import AttentionMetadata
|
from vllm.v1.attention.backend import AttentionMetadata
|
||||||
@@ -42,6 +43,7 @@ class ShortConv(MambaBase, CustomOp):
|
|||||||
layer_idx: int,
|
layer_idx: int,
|
||||||
model_config: ModelConfig | None = None,
|
model_config: ModelConfig | None = None,
|
||||||
cache_config: CacheConfig | None = None,
|
cache_config: CacheConfig | None = None,
|
||||||
|
quant_config: QuantizationConfig | None = None,
|
||||||
prefix: str = "",
|
prefix: str = "",
|
||||||
):
|
):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
@@ -67,12 +69,14 @@ class ShortConv(MambaBase, CustomOp):
|
|||||||
input_size=dim,
|
input_size=dim,
|
||||||
output_sizes=[dim] * 3,
|
output_sizes=[dim] * 3,
|
||||||
bias=self.bias,
|
bias=self.bias,
|
||||||
|
quant_config=quant_config,
|
||||||
prefix=f"{prefix}.in_proj",
|
prefix=f"{prefix}.in_proj",
|
||||||
)
|
)
|
||||||
self.out_proj = RowParallelLinear(
|
self.out_proj = RowParallelLinear(
|
||||||
input_size=dim,
|
input_size=dim,
|
||||||
output_size=dim,
|
output_size=dim,
|
||||||
bias=self.bias,
|
bias=self.bias,
|
||||||
|
quant_config=quant_config,
|
||||||
prefix=f"{prefix}.out_proj",
|
prefix=f"{prefix}.out_proj",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -257,6 +257,7 @@ class Lfm2ShortConvDecoderLayer(nn.Module):
|
|||||||
layer_idx=layer_idx,
|
layer_idx=layer_idx,
|
||||||
model_config=model_config,
|
model_config=model_config,
|
||||||
cache_config=cache_config,
|
cache_config=cache_config,
|
||||||
|
quant_config=quant_config,
|
||||||
prefix=f"{prefix}.conv",
|
prefix=f"{prefix}.conv",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -351,6 +351,7 @@ class Lfm2MoeShortConvDecoderLayer(nn.Module):
|
|||||||
layer_idx=layer_idx,
|
layer_idx=layer_idx,
|
||||||
model_config=model_config,
|
model_config=model_config,
|
||||||
cache_config=cache_config,
|
cache_config=cache_config,
|
||||||
|
quant_config=quant_config,
|
||||||
prefix=f"{prefix}.conv",
|
prefix=f"{prefix}.conv",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user