expose flex block size for batch invariant mode (#41252)
Signed-off-by: Angel Li <liangel@meta.com>
This commit is contained in:
@@ -147,8 +147,14 @@ def test_v1_generation_is_deterministic_across_batch_sizes_with_needle(
|
||||
"backend",
|
||||
BACKENDS,
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"block_m,block_n",
|
||||
[(16, 16), (8, 16)],
|
||||
)
|
||||
def test_logprobs_bitwise_batch_invariance_bs1_vs_bsN(
|
||||
backend,
|
||||
block_m,
|
||||
block_n,
|
||||
):
|
||||
seed = int(os.getenv("VLLM_TEST_SEED", "12345"))
|
||||
random.seed(seed)
|
||||
@@ -172,7 +178,11 @@ def test_logprobs_bitwise_batch_invariance_bs1_vs_bsN(
|
||||
max_model_len=8192,
|
||||
dtype="auto", # not everything is supported
|
||||
gpu_memory_utilization=0.9,
|
||||
attention_config={"backend": backend},
|
||||
attention_config={
|
||||
"backend": backend,
|
||||
"flex_attn_block_m": block_m,
|
||||
"flex_attn_block_n": block_n,
|
||||
},
|
||||
)
|
||||
|
||||
# Use more realistic prompts for better token generation
|
||||
|
||||
Reference in New Issue
Block a user