Skip to content

Commit 47fc74c

Browse files
kacperlukawskikacperlukawski
committed
Merge branch 'master' into add-filtering-support
# Conflicts: # poetry.lock # pyproject.toml
2 parents 903e4c7 + 036e4b3 commit 47fc74c

File tree

2 files changed

+136
-6
lines changed

2 files changed

+136
-6
lines changed

‎experiments/configurations/qdrant-single-node-mmap.json

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
[
2+
{
3+
"name": "qdrant-mmap-m-16-ef-128",
4+
"engine": "qdrant",
5+
"connection_params": {},
6+
"collection_params": {
7+
"optimizers_config": { "memmap_threshold": 10000 },
8+
"hnsw_config": { "m": 16, "ef_construct": 128 }
9+
},
10+
"search_params": [
11+
{ "parallel": 1, "search_params": { "hnsw_ef": 64 } }, { "parallel": 1, "search_params": { "hnsw_ef": 128 } }, { "parallel": 1, "search_params": { "hnsw_ef": 256 } }, { "parallel": 1, "search_params": { "hnsw_ef": 512 } },
12+
{ "parallel": 2, "search_params": { "hnsw_ef": 64 } }, { "parallel": 2, "search_params": { "hnsw_ef": 128 } }, { "parallel": 2, "search_params": { "hnsw_ef": 256 } }, { "parallel": 2, "search_params": { "hnsw_ef": 512 } },
13+
{ "parallel": 4, "search_params": { "hnsw_ef": 64 } }, { "parallel": 4, "search_params": { "hnsw_ef": 128 } }, { "parallel": 4, "search_params": { "hnsw_ef": 256 } }, { "parallel": 4, "search_params": { "hnsw_ef": 512 } },
14+
{ "parallel": 8, "search_params": { "hnsw_ef": 64 } }, { "parallel": 8, "search_params": { "hnsw_ef": 128 } }, { "parallel": 8, "search_params": { "hnsw_ef": 256 } }, { "parallel": 8, "search_params": { "hnsw_ef": 512 } },
15+
{ "parallel": 100, "search_params": { "hnsw_ef": 64 } }, { "parallel": 100, "search_params": { "hnsw_ef": 128 } }, { "parallel": 100, "search_params": { "hnsw_ef": 256 } }, { "parallel": 100, "search_params": { "hnsw_ef": 512 } }
16+
],
17+
"upload_params": { "parallel": 16 }
18+
},
19+
{
20+
"name": "qdrant-mmap-m-32-ef-128",
21+
"engine": "qdrant",
22+
"connection_params": {},
23+
"collection_params": {
24+
"optimizers_config": { "memmap_threshold": 10000 },
25+
"hnsw_config": { "m": 32, "ef_construct": 128 }
26+
},
27+
"search_params": [
28+
{ "parallel": 1, "search_params": { "hnsw_ef": 64 } }, { "parallel": 1, "search_params": { "hnsw_ef": 128 } }, { "parallel": 1, "search_params": { "hnsw_ef": 256 } }, { "parallel": 1, "search_params": { "hnsw_ef": 512 } },
29+
{ "parallel": 2, "search_params": { "hnsw_ef": 64 } }, { "parallel": 2, "search_params": { "hnsw_ef": 128 } }, { "parallel": 2, "search_params": { "hnsw_ef": 256 } }, { "parallel": 2, "search_params": { "hnsw_ef": 512 } },
30+
{ "parallel": 4, "search_params": { "hnsw_ef": 64 } }, { "parallel": 4, "search_params": { "hnsw_ef": 128 } }, { "parallel": 4, "search_params": { "hnsw_ef": 256 } }, { "parallel": 4, "search_params": { "hnsw_ef": 512 } },
31+
{ "parallel": 8, "search_params": { "hnsw_ef": 64 } }, { "parallel": 8, "search_params": { "hnsw_ef": 128 } }, { "parallel": 8, "search_params": { "hnsw_ef": 256 } }, { "parallel": 8, "search_params": { "hnsw_ef": 512 } },
32+
{ "parallel": 100, "search_params": { "hnsw_ef": 64 } }, { "parallel": 100, "search_params": { "hnsw_ef": 128 } }, { "parallel": 100, "search_params": { "hnsw_ef": 256 } }, { "parallel": 100, "search_params": { "hnsw_ef": 512 } }
33+
],
34+
"upload_params": { "parallel": 16 }
35+
},
36+
{
37+
"name": "qdrant-mmap-m-32-ef-256",
38+
"engine": "qdrant",
39+
"connection_params": {},
40+
"collection_params": {
41+
"optimizers_config": { "memmap_threshold": 10000 },
42+
"hnsw_config": { "m": 32, "ef_construct": 256 }
43+
},
44+
"search_params": [
45+
{ "parallel": 1, "search_params": { "hnsw_ef": 64 } }, { "parallel": 1, "search_params": { "hnsw_ef": 128 } }, { "parallel": 1, "search_params": { "hnsw_ef": 256 } }, { "parallel": 1, "search_params": { "hnsw_ef": 512 } },
46+
{ "parallel": 2, "search_params": { "hnsw_ef": 64 } }, { "parallel": 2, "search_params": { "hnsw_ef": 128 } }, { "parallel": 2, "search_params": { "hnsw_ef": 256 } }, { "parallel": 2, "search_params": { "hnsw_ef": 512 } },
47+
{ "parallel": 4, "search_params": { "hnsw_ef": 64 } }, { "parallel": 4, "search_params": { "hnsw_ef": 128 } }, { "parallel": 4, "search_params": { "hnsw_ef": 256 } }, { "parallel": 4, "search_params": { "hnsw_ef": 512 } },
48+
{ "parallel": 8, "search_params": { "hnsw_ef": 64 } }, { "parallel": 8, "search_params": { "hnsw_ef": 128 } }, { "parallel": 8, "search_params": { "hnsw_ef": 256 } }, { "parallel": 8, "search_params": { "hnsw_ef": 512 } },
49+
{ "parallel": 100, "search_params": { "hnsw_ef": 64 } }, { "parallel": 100, "search_params": { "hnsw_ef": 128 } }, { "parallel": 100, "search_params": { "hnsw_ef": 256 } }, { "parallel": 100, "search_params": { "hnsw_ef": 512 } }
50+
],
51+
"upload_params": { "parallel": 16 }
52+
},
53+
{
54+
"name": "qdrant-mmap-m-32-ef-512",
55+
"engine": "qdrant",
56+
"connection_params": {},
57+
"collection_params": {
58+
"optimizers_config": { "memmap_threshold": 10000 },
59+
"hnsw_config": { "m": 32, "ef_construct": 512 }
60+
},
61+
"search_params": [
62+
{ "parallel": 1, "search_params": { "hnsw_ef": 64 } }, { "parallel": 1, "search_params": { "hnsw_ef": 128 } }, { "parallel": 1, "search_params": { "hnsw_ef": 256 } }, { "parallel": 1, "search_params": { "hnsw_ef": 512 } },
63+
{ "parallel": 2, "search_params": { "hnsw_ef": 64 } }, { "parallel": 2, "search_params": { "hnsw_ef": 128 } }, { "parallel": 2, "search_params": { "hnsw_ef": 256 } }, { "parallel": 2, "search_params": { "hnsw_ef": 512 } },
64+
{ "parallel": 4, "search_params": { "hnsw_ef": 64 } }, { "parallel": 4, "search_params": { "hnsw_ef": 128 } }, { "parallel": 4, "search_params": { "hnsw_ef": 256 } }, { "parallel": 4, "search_params": { "hnsw_ef": 512 } },
65+
{ "parallel": 8, "search_params": { "hnsw_ef": 64 } }, { "parallel": 8, "search_params": { "hnsw_ef": 128 } }, { "parallel": 8, "search_params": { "hnsw_ef": 256 } }, { "parallel": 8, "search_params": { "hnsw_ef": 512 } },
66+
{ "parallel": 100, "search_params": { "hnsw_ef": 64 } }, { "parallel": 100, "search_params": { "hnsw_ef": 128 } }, { "parallel": 100, "search_params": { "hnsw_ef": 256 } }, { "parallel": 100, "search_params": { "hnsw_ef": 512 } }
67+
],
68+
"upload_params": { "parallel": 16 }
69+
},
70+
{
71+
"name": "qdrant-mmap-m-64-ef-256",
72+
"engine": "qdrant",
73+
"connection_params": {},
74+
"collection_params": {
75+
"optimizers_config": { "memmap_threshold": 10000 },
76+
"hnsw_config": { "m": 64, "ef_construct": 256 }
77+
},
78+
"search_params": [
79+
{ "parallel": 1, "search_params": { "hnsw_ef": 64 } }, { "parallel": 1, "search_params": { "hnsw_ef": 128 } }, { "parallel": 1, "search_params": { "hnsw_ef": 256 } }, { "parallel": 1, "search_params": { "hnsw_ef": 512 } },
80+
{ "parallel": 2, "search_params": { "hnsw_ef": 64 } }, { "parallel": 2, "search_params": { "hnsw_ef": 128 } }, { "parallel": 2, "search_params": { "hnsw_ef": 256 } }, { "parallel": 2, "search_params": { "hnsw_ef": 512 } },
81+
{ "parallel": 4, "search_params": { "hnsw_ef": 64 } }, { "parallel": 4, "search_params": { "hnsw_ef": 128 } }, { "parallel": 4, "search_params": { "hnsw_ef": 256 } }, { "parallel": 4, "search_params": { "hnsw_ef": 512 } },
82+
{ "parallel": 8, "search_params": { "hnsw_ef": 64 } }, { "parallel": 8, "search_params": { "hnsw_ef": 128 } }, { "parallel": 8, "search_params": { "hnsw_ef": 256 } }, { "parallel": 8, "search_params": { "hnsw_ef": 512 } },
83+
{ "parallel": 100, "search_params": { "hnsw_ef": 64 } }, { "parallel": 100, "search_params": { "hnsw_ef": 128 } }, { "parallel": 100, "search_params": { "hnsw_ef": 256 } }, { "parallel": 100, "search_params": { "hnsw_ef": 512 } }
84+
],
85+
"upload_params": { "parallel": 16 }
86+
},
87+
{
88+
"name": "qdrant-mmap-m-64-ef-512",
89+
"engine": "qdrant",
90+
"connection_params": {},
91+
"collection_params": {
92+
"optimizers_config": { "memmap_threshold": 10000 },
93+
"hnsw_config": { "m": 64, "ef_construct": 512 }
94+
},
95+
"search_params": [
96+
{ "parallel": 1, "search_params": { "hnsw_ef": 64 } }, { "parallel": 1, "search_params": { "hnsw_ef": 128 } }, { "parallel": 1, "search_params": { "hnsw_ef": 256 } }, { "parallel": 1, "search_params": { "hnsw_ef": 512 } },
97+
{ "parallel": 2, "search_params": { "hnsw_ef": 64 } }, { "parallel": 2, "search_params": { "hnsw_ef": 128 } }, { "parallel": 2, "search_params": { "hnsw_ef": 256 } }, { "parallel": 2, "search_params": { "hnsw_ef": 512 } },
98+
{ "parallel": 4, "search_params": { "hnsw_ef": 64 } }, { "parallel": 4, "search_params": { "hnsw_ef": 128 } }, { "parallel": 4, "search_params": { "hnsw_ef": 256 } }, { "parallel": 4, "search_params": { "hnsw_ef": 512 } },
99+
{ "parallel": 8, "search_params": { "hnsw_ef": 64 } }, { "parallel": 8, "search_params": { "hnsw_ef": 128 } }, { "parallel": 8, "search_params": { "hnsw_ef": 256 } }, { "parallel": 8, "search_params": { "hnsw_ef": 512 } },
100+
{ "parallel": 100, "search_params": { "hnsw_ef": 64 } }, { "parallel": 100, "search_params": { "hnsw_ef": 128 } }, { "parallel": 100, "search_params": { "hnsw_ef": 256 } }, { "parallel": 100, "search_params": { "hnsw_ef": 512 } }
101+
],
102+
"upload_params": { "parallel": 16 }
103+
}
104+
]

‎run.py

Lines changed: 32 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
import fnmatch
2+
import traceback
3+
from typing import List
24

5+
import stopit
36
import typer
47

58
from benchmark.config_read import read_dataset_config, read_engine_configs
@@ -12,27 +15,29 @@
1215

1316
@app.command()
1417
def run(
15-
engines: str = "*",
16-
datasets: str = "*",
18+
engines: List[str] = typer.Option(["*"]),
19+
datasets: List[str] = typer.Option(["*"]),
1720
host: str = "localhost",
1821
skip_upload: bool = False,
22+
exit_on_error: bool = True,
23+
timeout: float = 86400.0,
1924
):
2025
"""
2126
Example:
22-
python3 run --engines *-m-16-* --datasets glove-*
27+
python3 run.py --engines *-m-16-* --engines qdrant-* --datasets glove-*
2328
"""
2429
all_engines = read_engine_configs()
2530
all_datasets = read_dataset_config()
2631

2732
selected_engines = {
2833
name: config
2934
for name, config in all_engines.items()
30-
if fnmatch.fnmatch(name, engines)
35+
if any(fnmatch.fnmatch(name, engine) for engine in engines)
3136
}
3237
selected_datasets = {
3338
name: config
3439
for name, config in all_datasets.items()
35-
if fnmatch.fnmatch(name, datasets)
40+
if any(fnmatch.fnmatch(name, dataset) for dataset in datasets)
3641
}
3742

3843
for engine_name, engine_config in selected_engines.items():
@@ -42,10 +47,31 @@ def run(
4247
dataset = Dataset(dataset_config)
4348
dataset.download()
4449
try:
45-
client.run_experiment(dataset, skip_upload)
50+
with stopit.ThreadingTimeout(timeout) as tt:
51+
client.run_experiment(dataset, skip_upload)
52+
53+
# If the timeout is reached, the server might be still in the
54+
# middle of some background processing, like creating the index.
55+
# Next experiment should not be launched. It's better to reset
56+
# the server state manually.
57+
if tt.state != stopit.ThreadingTimeout.EXECUTED:
58+
print(
59+
f"Timed out {engine_name} - {dataset_name}, "
60+
f"exceeded {timeout} seconds"
61+
)
62+
exit(2)
4663
except IncompatibilityError as e:
4764
print(f"Skipping {engine_name} - {dataset_name}, incompatible params")
4865
continue
66+
except KeyboardInterrupt as e:
67+
traceback.print_exc()
68+
exit(1)
69+
except Exception as e:
70+
print(f"Experiment {engine_name} - {dataset_name} interrupted")
71+
traceback.print_exc()
72+
if exit_on_error:
73+
raise e
74+
continue
4975

5076

5177
if __name__ == "__main__":

0 commit comments

Comments
 (0)