[Feature] TurboQuant: support hybrid models and uniform quantization (#39931)

Signed-off-by: JartX <sagformas@epdcenter.es>
Signed-off-by: Jim Smith <jhsmith0@me.com>
Co-authored-by: Jim Smith <jhsmith0@me.com>
Co-authored-by: Sandermage <sandermage@users.noreply.github.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
JartX
2026-05-04 20:14:01 -04:00
committed by GitHub
co-authored by Jim Smith Sandermage Claude
parent 577b9623e6
commit 4f2af1a7c0
4 changed files with 185 additions and 26 deletions
+82 -4
View File
@@ -182,22 +182,100 @@ class TestTurboQuantConfig:
# ---- Boundary skip layers ----
@staticmethod
def _dense_model_config(num_layers):
from types import SimpleNamespace
return SimpleNamespace(
is_hybrid=False,
hf_text_config=SimpleNamespace(num_hidden_layers=num_layers),
)
def test_boundary_skip_layers_basic(self):
layers = TurboQuantConfig.get_boundary_skip_layers(32)
mc = self._dense_model_config(32)
layers = TurboQuantConfig.get_boundary_skip_layers(mc)
assert layers == ["0", "1", "30", "31"]
def test_boundary_skip_layers_zero(self):
assert TurboQuantConfig.get_boundary_skip_layers(32, 0) == []
mc = self._dense_model_config(32)
assert TurboQuantConfig.get_boundary_skip_layers(mc, 0) == []
def test_boundary_skip_layers_small_model(self):
layers = TurboQuantConfig.get_boundary_skip_layers(4)
mc = self._dense_model_config(4)
layers = TurboQuantConfig.get_boundary_skip_layers(mc)
assert layers == ["0", "1", "2", "3"]
def test_boundary_skip_layers_cap_at_half(self):
layers = TurboQuantConfig.get_boundary_skip_layers(8, 10)
mc = self._dense_model_config(8)
layers = TurboQuantConfig.get_boundary_skip_layers(mc, 10)
assert len(layers) == 8
class TestHybridAttentionIndices:
"""Regression tests for boundary protection on hybrid models.
Hybrid models (attention + Mamba / linear-attention) identify KV-carrying
layers via layer_types / layers_block_type / attn_type_list. The helper
must return the *global* layer indices of the full-attention layers so
that kv_cache_dtype_skip_layers matches what extract_layer_index(prefix)
reports on the Attention layers at runtime.
"""
@staticmethod
def _fake_model_config(text_cfg=None, hf_cfg=None):
from types import SimpleNamespace
return SimpleNamespace(
hf_text_config=text_cfg if text_cfg is not None else SimpleNamespace(),
hf_config=hf_cfg if hf_cfg is not None else SimpleNamespace(),
)
def test_layer_types_full_attention(self):
from vllm.model_executor.layers.quantization.turboquant.config import (
_get_full_attention_layer_indices,
)
cfg = type("C", (), {})()
cfg.layer_types = [
"linear_attention",
"linear_attention",
"full_attention",
"linear_attention",
"full_attention",
"full_attention",
]
mc = self._fake_model_config(text_cfg=cfg)
assert _get_full_attention_layer_indices(mc) == [2, 4, 5]
def test_layers_block_type_jamba(self):
from vllm.model_executor.layers.quantization.turboquant.config import (
_get_full_attention_layer_indices,
)
cfg = type("C", (), {})()
cfg.layers_block_type = ["mamba", "attention", "mamba", "attention"]
mc = self._fake_model_config(text_cfg=cfg)
assert _get_full_attention_layer_indices(mc) == [1, 3]
def test_attn_type_list_minimax(self):
from vllm.model_executor.layers.quantization.turboquant.config import (
_get_full_attention_layer_indices,
)
hf = type("C", (), {})()
hf.attn_type_list = [0, 1, 0, 1, 1]
mc = self._fake_model_config(hf_cfg=hf)
assert _get_full_attention_layer_indices(mc) == [1, 3, 4]
def test_no_hybrid_hints_returns_empty(self):
from vllm.model_executor.layers.quantization.turboquant.config import (
_get_full_attention_layer_indices,
)
mc = self._fake_model_config()
assert _get_full_attention_layer_indices(mc) == []
# ============================================================================
# Centroids tests (CPU-only)
# ============================================================================