[Feature] TurboQuant: support hybrid models and uniform quantization (#39931)
Signed-off-by: JartX <sagformas@epdcenter.es> Signed-off-by: Jim Smith <jhsmith0@me.com> Co-authored-by: Jim Smith <jhsmith0@me.com> Co-authored-by: Sandermage <sandermage@users.noreply.github.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Jim Smith
Sandermage
Claude
parent
577b9623e6
commit
4f2af1a7c0
@@ -1699,29 +1699,15 @@ class EngineArgs:
|
||||
kv_offloading_backend=self.kv_offloading_backend,
|
||||
)
|
||||
|
||||
# TurboQuant: auto-skip first/last 2 layers (boundary protection).
|
||||
# These layers are most sensitive to quantization error.
|
||||
# Users can add extra layers via --kv-cache-dtype-skip-layers.
|
||||
if resolved_cache_dtype.startswith("turboquant_"):
|
||||
if model_config.is_hybrid:
|
||||
raise NotImplementedError(
|
||||
"TurboQuant KV cache is not supported for hybrid "
|
||||
"(attention + Mamba) models. Boundary layer protection "
|
||||
"requires uniform attention layers."
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.turboquant.config import (
|
||||
TurboQuantConfig,
|
||||
)
|
||||
|
||||
num_layers = model_config.hf_text_config.num_hidden_layers
|
||||
boundary = TurboQuantConfig.get_boundary_skip_layers(num_layers)
|
||||
boundary = TurboQuantConfig.get_boundary_skip_layers(model_config)
|
||||
existing = set(cache_config.kv_cache_dtype_skip_layers)
|
||||
merged = sorted(existing | set(boundary), key=lambda x: int(x))
|
||||
cache_config.kv_cache_dtype_skip_layers = merged
|
||||
logger.info(
|
||||
"TQ: skipping layers %s for boundary protection (num_layers=%d)",
|
||||
merged,
|
||||
num_layers,
|
||||
cache_config.kv_cache_dtype_skip_layers = sorted(
|
||||
existing | set(boundary), key=int
|
||||
)
|
||||
|
||||
ray_runtime_env = None
|
||||
|
||||
Reference in New Issue
Block a user