[Feature] TurboQuant: support hybrid models and uniform quantization (#39931)
Signed-off-by: JartX <sagformas@epdcenter.es> Signed-off-by: Jim Smith <jhsmith0@me.com> Co-authored-by: Jim Smith <jhsmith0@me.com> Co-authored-by: Sandermage <sandermage@users.noreply.github.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Jim Smith
Sandermage
Claude
parent
577b9623e6
commit
4f2af1a7c0
@@ -182,22 +182,100 @@ class TestTurboQuantConfig:
|
||||
|
||||
# ---- Boundary skip layers ----
|
||||
|
||||
@staticmethod
|
||||
def _dense_model_config(num_layers):
|
||||
from types import SimpleNamespace
|
||||
|
||||
return SimpleNamespace(
|
||||
is_hybrid=False,
|
||||
hf_text_config=SimpleNamespace(num_hidden_layers=num_layers),
|
||||
)
|
||||
|
||||
def test_boundary_skip_layers_basic(self):
|
||||
layers = TurboQuantConfig.get_boundary_skip_layers(32)
|
||||
mc = self._dense_model_config(32)
|
||||
layers = TurboQuantConfig.get_boundary_skip_layers(mc)
|
||||
assert layers == ["0", "1", "30", "31"]
|
||||
|
||||
def test_boundary_skip_layers_zero(self):
|
||||
assert TurboQuantConfig.get_boundary_skip_layers(32, 0) == []
|
||||
mc = self._dense_model_config(32)
|
||||
assert TurboQuantConfig.get_boundary_skip_layers(mc, 0) == []
|
||||
|
||||
def test_boundary_skip_layers_small_model(self):
|
||||
layers = TurboQuantConfig.get_boundary_skip_layers(4)
|
||||
mc = self._dense_model_config(4)
|
||||
layers = TurboQuantConfig.get_boundary_skip_layers(mc)
|
||||
assert layers == ["0", "1", "2", "3"]
|
||||
|
||||
def test_boundary_skip_layers_cap_at_half(self):
|
||||
layers = TurboQuantConfig.get_boundary_skip_layers(8, 10)
|
||||
mc = self._dense_model_config(8)
|
||||
layers = TurboQuantConfig.get_boundary_skip_layers(mc, 10)
|
||||
assert len(layers) == 8
|
||||
|
||||
|
||||
class TestHybridAttentionIndices:
|
||||
"""Regression tests for boundary protection on hybrid models.
|
||||
|
||||
Hybrid models (attention + Mamba / linear-attention) identify KV-carrying
|
||||
layers via layer_types / layers_block_type / attn_type_list. The helper
|
||||
must return the *global* layer indices of the full-attention layers so
|
||||
that kv_cache_dtype_skip_layers matches what extract_layer_index(prefix)
|
||||
reports on the Attention layers at runtime.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _fake_model_config(text_cfg=None, hf_cfg=None):
|
||||
from types import SimpleNamespace
|
||||
|
||||
return SimpleNamespace(
|
||||
hf_text_config=text_cfg if text_cfg is not None else SimpleNamespace(),
|
||||
hf_config=hf_cfg if hf_cfg is not None else SimpleNamespace(),
|
||||
)
|
||||
|
||||
def test_layer_types_full_attention(self):
|
||||
from vllm.model_executor.layers.quantization.turboquant.config import (
|
||||
_get_full_attention_layer_indices,
|
||||
)
|
||||
|
||||
cfg = type("C", (), {})()
|
||||
cfg.layer_types = [
|
||||
"linear_attention",
|
||||
"linear_attention",
|
||||
"full_attention",
|
||||
"linear_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
]
|
||||
mc = self._fake_model_config(text_cfg=cfg)
|
||||
assert _get_full_attention_layer_indices(mc) == [2, 4, 5]
|
||||
|
||||
def test_layers_block_type_jamba(self):
|
||||
from vllm.model_executor.layers.quantization.turboquant.config import (
|
||||
_get_full_attention_layer_indices,
|
||||
)
|
||||
|
||||
cfg = type("C", (), {})()
|
||||
cfg.layers_block_type = ["mamba", "attention", "mamba", "attention"]
|
||||
mc = self._fake_model_config(text_cfg=cfg)
|
||||
assert _get_full_attention_layer_indices(mc) == [1, 3]
|
||||
|
||||
def test_attn_type_list_minimax(self):
|
||||
from vllm.model_executor.layers.quantization.turboquant.config import (
|
||||
_get_full_attention_layer_indices,
|
||||
)
|
||||
|
||||
hf = type("C", (), {})()
|
||||
hf.attn_type_list = [0, 1, 0, 1, 1]
|
||||
mc = self._fake_model_config(hf_cfg=hf)
|
||||
assert _get_full_attention_layer_indices(mc) == [1, 3, 4]
|
||||
|
||||
def test_no_hybrid_hints_returns_empty(self):
|
||||
from vllm.model_executor.layers.quantization.turboquant.config import (
|
||||
_get_full_attention_layer_indices,
|
||||
)
|
||||
|
||||
mc = self._fake_model_config()
|
||||
assert _get_full_attention_layer_indices(mc) == []
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Centroids tests (CPU-only)
|
||||
# ============================================================================
|
||||
|
||||
Reference in New Issue
Block a user