Skip to content

Commit 455b590

Browse files
tellet-qtellet-q
authored
Automate running benchmarks for all engines (qdrant#134)
* ci: Run *-default benchmarks for all engines * Update poetry.lock
1 parent e6049a4 commit 455b590

File tree

6 files changed

+290
-45
lines changed

6 files changed

+290
-45
lines changed

‎.github/workflows/actions/run-engine-benchmark/action.yaml

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
name: Run Engine Benchmark
2+
description: "Run benchmark with specified params"
3+
inputs:
4+
engine:
5+
description: "engine (i.e qdrant-default)"
6+
required: true
7+
dataset:
8+
description: "dataset (i.e random-100)"
9+
required: true
10+
compose_file:
11+
description: "path to docker compose"
12+
required: true
13+
14+
runs:
15+
using: "composite"
16+
steps:
17+
- name: Install poetry
18+
shell: bash
19+
run: pip install poetry
20+
- uses: actions/setup-python@v5
21+
with:
22+
python-version: "3.10"
23+
cache: "poetry"
24+
- name: Install deps
25+
shell: bash
26+
run: poetry install
27+
- uses: hoverkraft-tech/[email protected]
28+
with:
29+
compose-file: "${{ inputs.compose_file }}"
30+
- name: Execution
31+
shell: bash
32+
run: |
33+
engine="${{ inputs.engine }}"
34+
if [[ "$engine" == *"elasticsearch"* || "$engine" == *"opensearch"* ]]; then
35+
./tools/wait_for_green_status.sh
36+
fi
37+
source $(poetry env info -p)/bin/activate
38+
poetry run python3 run.py --engines "${{ inputs.engine }}" --datasets "${{ inputs.dataset }}"

‎.github/workflows/manual-all-engines-benchmark.yaml

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
name: Manual All Engines Default Benchmarks
2+
3+
on:
4+
push:
5+
branches:
6+
- "master"
7+
pull_request:
8+
types:
9+
- opened
10+
- reopened
11+
workflow_dispatch:
12+
13+
jobs:
14+
elasticsearchBenchmark:
15+
if: >
16+
(
17+
startsWith(github.event.head_commit.modified, 'engine/clients/elasticsearch') ||
18+
startsWith(github.event.head_commit.modified, 'engine/servers/elasticsearch') ||
19+
startsWith(github.event.head_commit.modified, 'engine/base_client/')
20+
)
21+
name: benchmark - elasticsearch-default - random-100 - against elasticsearch-single-node-ci
22+
runs-on: ubuntu-latest
23+
timeout-minutes: 30
24+
steps:
25+
- uses: actions/checkout@v3
26+
- uses: ./.github/workflows/actions/run-engine-benchmark
27+
with:
28+
engine: "elasticsearch-default"
29+
dataset: "random-100"
30+
compose_file: "engine/servers/elasticsearch-single-node-ci/docker-compose.yaml"
31+
32+
milvusBenchmark:
33+
if: >
34+
(
35+
startsWith(github.event.head_commit.modified, 'engine/clients/milvus') ||
36+
startsWith(github.event.head_commit.modified, 'engine/servers/milvus') ||
37+
startsWith(github.event.head_commit.modified, 'engine/base_client/')
38+
)
39+
name: benchmark - milvus-default - random-100 - against milvus-single-node
40+
runs-on: ubuntu-latest
41+
timeout-minutes: 30
42+
steps:
43+
- uses: actions/checkout@v3
44+
- uses: ./.github/workflows/actions/run-engine-benchmark
45+
with:
46+
engine: "milvus-default"
47+
dataset: "random-100"
48+
compose_file: "engine/servers/milvus-single-node/docker-compose.yaml"
49+
50+
opensearchBenchmark:
51+
if: >
52+
(
53+
startsWith(github.event.head_commit.modified, 'engine/clients/opensearch') ||
54+
startsWith(github.event.head_commit.modified, 'engine/servers/opensearch') ||
55+
startsWith(github.event.head_commit.modified, 'engine/base_client/')
56+
)
57+
name: benchmark - opensearch-default - glove-25-angular - against opensearch-single-node-ci
58+
runs-on: ubuntu-latest
59+
timeout-minutes: 30
60+
steps:
61+
- uses: actions/checkout@v3
62+
- uses: ./.github/workflows/actions/run-engine-benchmark
63+
with:
64+
engine: "opensearch-default"
65+
dataset: "glove-25-angular"
66+
compose_file: "engine/servers/opensearch-single-node-ci/docker-compose.yaml"
67+
68+
pgvectorBenchmark:
69+
if: >
70+
(
71+
startsWith(github.event.head_commit.modified, 'engine/clients/pgvector') ||
72+
startsWith(github.event.head_commit.modified, 'engine/servers/pgvector') ||
73+
startsWith(github.event.head_commit.modified, 'engine/base_client/')
74+
)
75+
name: benchmark - pgvector-default - random-100 - against pgvector-single-node
76+
runs-on: ubuntu-latest
77+
timeout-minutes: 30
78+
steps:
79+
- uses: actions/checkout@v3
80+
- uses: ./.github/workflows/actions/run-engine-benchmark
81+
with:
82+
engine: "pgvector-default"
83+
dataset: "random-100"
84+
compose_file: "engine/servers/pgvector-single-node/docker-compose.yaml"
85+
86+
qdrantBenchmark:
87+
if: >
88+
(
89+
startsWith(github.event.head_commit.modified, 'engine/clients/qdrant') ||
90+
startsWith(github.event.head_commit.modified, 'engine/servers/qdrant') ||
91+
startsWith(github.event.head_commit.modified, 'engine/base_client/')
92+
)
93+
name: benchmark - qdrant-default - random-100 - against qdrant-single-node
94+
runs-on: ubuntu-latest
95+
timeout-minutes: 30
96+
steps:
97+
- uses: actions/checkout@v3
98+
- uses: ./.github/workflows/actions/run-engine-benchmark
99+
with:
100+
engine: "qdrant-default"
101+
dataset: "random-100"
102+
compose_file: "engine/servers/qdrant-single-node/docker-compose.yaml"
103+
104+
redisBenchmark:
105+
if: >
106+
(
107+
startsWith(github.event.head_commit.modified, 'engine/clients/redis') ||
108+
startsWith(github.event.head_commit.modified, 'engine/servers/redis') ||
109+
startsWith(github.event.head_commit.modified, 'engine/base_client/')
110+
)
111+
name: benchmark - redis-default - random-100 - against redis-single-node
112+
runs-on: ubuntu-latest
113+
timeout-minutes: 30
114+
steps:
115+
- uses: actions/checkout@v3
116+
- uses: ./.github/workflows/actions/run-engine-benchmark
117+
with:
118+
engine: "redis-default"
119+
dataset: "random-100"
120+
compose_file: "engine/servers/redis-single-node/docker-compose.yaml"
121+
122+
weaviateBenchmark:
123+
if: >
124+
(
125+
startsWith(github.event.head_commit.modified, 'engine/clients/weaviate') ||
126+
startsWith(github.event.head_commit.modified, 'engine/servers/weaviate') ||
127+
startsWith(github.event.head_commit.modified, 'engine/base_client/')
128+
)
129+
name: benchmark - weaviate-default - random-100 - against weaviate-single-node
130+
runs-on: ubuntu-latest
131+
timeout-minutes: 30
132+
steps:
133+
- uses: actions/checkout@v3
134+
- uses: ./.github/workflows/actions/run-engine-benchmark
135+
with:
136+
engine: "weaviate-default"
137+
dataset: "random-100"
138+
compose_file: "engine/servers/weaviate-single-node/docker-compose.yaml"

‎engine/servers/elasticsearch-single-node-ci/docker-compose.yaml

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
version: '3.5'
2+
3+
services:
4+
es:
5+
image: docker.elastic.co/elasticsearch/elasticsearch:8.10.2
6+
environment:
7+
ELASTIC_PASSWORD: "passwd"
8+
KIBANA_PASSWORD: "passwd"
9+
SERVER_SSL_ENABLED: "false"
10+
discovery.type: "single-node"
11+
xpack.security.enabled: "false"
12+
ports:
13+
- "9200:9200"
14+
- "9300:9300"
15+
logging:
16+
driver: "json-file"
17+
options:
18+
max-file: 1
19+
max-size: 10m
20+
deploy:
21+
resources:
22+
limits:
23+
memory: 4Gb

‎engine/servers/opensearch-single-node-ci/docker-compose.yaml

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
version: '3.5'
2+
3+
services:
4+
opensearch:
5+
image: opensearchproject/opensearch:2.10.0
6+
environment:
7+
discovery.type: "single-node"
8+
plugins.security.disabled: true
9+
OPENSEARCH_JAVA_OPTS: "-Xms2g -Xmx2g"
10+
ports:
11+
- "9200:9200"
12+
- "9300:9300"
13+
logging:
14+
driver: "json-file"
15+
options:
16+
max-file: 1
17+
max-size: 10m
18+
deploy:
19+
resources:
20+
limits:
21+
memory: 4Gb

‎poetry.lock

Lines changed: 36 additions & 45 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎tools/wait_for_green_status.sh

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#!/bin/bash
2+
# This scripts helps to wait for Opensearch|Elasticsearch status to become Green
3+
4+
set -e
5+
6+
SEARCH_CLUSTER_HOST=${1:-"localhost:9200"}
7+
8+
# Wait until the search cluster host is available
9+
until $(curl --output /dev/null --silent --head --fail "$SEARCH_CLUSTER_HOST"); do
10+
printf '.'
11+
sleep 1 # Wait for 1 second
12+
done
13+
14+
# Wait for ES/OS to start
15+
response=$(curl --write-out %{http_code} --silent --output /dev/null "$SEARCH_CLUSTER_HOST")
16+
17+
until [ "$response" = "200" ]; do
18+
response=$(curl --write-out %{http_code} --silent --output /dev/null "$SEARCH_CLUSTER_HOST")
19+
>&2 echo "Search cluster is unavailable - sleep 1s"
20+
sleep 1
21+
done
22+
23+
# Wait for ES/OS status to turn Green
24+
health="$(curl -fsSL "$SEARCH_CLUSTER_HOST/_cat/health?h=status")"
25+
health="$(echo "$health" | sed -r 's/^[[:space:]]+|[[:space:]]+$//g')"
26+
27+
until [ "$health" = 'green' ]; do
28+
health="$(curl -fsSL "$SEARCH_CLUSTER_HOST/_cat/health?h=status")"
29+
health="$(echo "$health" | sed -r 's/^[[:space:]]+|[[:space:]]+$//g')"
30+
>&2 echo "Search cluster status is not green yet - sleep 1s"
31+
sleep 1
32+
done
33+
34+
>&2 echo "Search cluster is up"

0 commit comments

Comments
 (0)