[2/N][KV-Cache Layout Refactor] Pack K/V into the content dim across attention backends (#44455)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
This commit is contained in:
co-authored by
Claude
OpenAI Codex
Matthew Bonanni
Nicolò Lucchesi
parent
76fedaa2a5
commit
51878e5b6e
@@ -10,7 +10,7 @@ from tests.kernels.utils import DEFAULT_OPCHECK_TEST_UTILS, opcheck
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import scaled_dequantize
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import nvfp4_kv_cache_split_views, set_random_seed
|
||||
from vllm.utils.torch_utils import nvfp4_split_data_scale, set_random_seed
|
||||
|
||||
COPYING_DIRECTION = [("cuda", "cpu"), ("cuda", "cuda"), ("cpu", "cuda")]
|
||||
DTYPES = [torch.bfloat16, torch.float]
|
||||
@@ -255,10 +255,8 @@ def test_reshape_and_cache_flash(
|
||||
nvfp4_key_data = None
|
||||
nvfp4_value_data = None
|
||||
if kv_cache_dtype == "nvfp4":
|
||||
(nvfp4_key_data,), (key_scale_cache,) = nvfp4_kv_cache_split_views(key_cache)
|
||||
(nvfp4_value_data,), (value_scale_cache,) = nvfp4_kv_cache_split_views(
|
||||
value_cache
|
||||
)
|
||||
nvfp4_key_data, key_scale_cache = nvfp4_split_data_scale(key_cache)
|
||||
nvfp4_value_data, value_scale_cache = nvfp4_split_data_scale(value_cache)
|
||||
|
||||
if kv_cache_dtype == "nvfp4":
|
||||
# Global scale = amax / 448 (per-tensor)
|
||||
|
||||
Reference in New Issue
Block a user