[Feature] Triton INT4 per-token-head KV cache quantization (#40835)
Signed-off-by: JartX <sagformas@epdcenter.es> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
d20dbf921b
commit
70749fdcca
@@ -31,7 +31,8 @@ from ..utils import check_logprobs_close
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"kv_cache_dtype", ["int8_per_token_head", "fp8_per_token_head"]
|
||||
"kv_cache_dtype",
|
||||
["int4_per_token_head", "int8_per_token_head", "fp8_per_token_head"],
|
||||
)
|
||||
@pytest.mark.parametrize("max_tokens", [4])
|
||||
@pytest.mark.parametrize("enforce_eager", [True])
|
||||
|
||||
Reference in New Issue
Block a user