Skip to content

Commit 4d82394

Browse files
generallgenerall
committed
CI benchmarks scripts
1 parent df05331 commit 4d82394

9 files changed

+202
-7
lines changed

‎run_all_engines.sh

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,13 @@ SERVER_USERNAME=${SERVER_USERNAME:-"qdrant"}
1111
function run_exp() {
1212
SERVER_PATH=$1
1313
ENGINE_NAME=$2
14-
MONITOR_PATH=$(echo $ENGINE_NAME | sed -e 's/[^A-Za-z0-9._-]/_/g')
15-
ssh ${SERVER_USERNAME}@${SERVER_HOST} "nohup bash -c 'cd ./projects/vector-db-benchmark/monitoring && rm -f docker.stats.jsonl && bash monitor_docker.sh' > /dev/null 2>&1 &"
16-
ssh -t ${SERVER_USERNAME}@${SERVER_HOST} "cd ./projects/vector-db-benchmark/engine/servers/$SERVER_PATH ; docker compose down ; docker compose up -d"
14+
MONITOR_PATH=$(echo "$ENGINE_NAME" | sed -e 's/[^A-Za-z0-9._-]/_/g')
15+
ssh "${SERVER_USERNAME}@${SERVER_HOST}" "nohup bash -c 'cd ./projects/vector-db-benchmark/monitoring && rm -f docker.stats.jsonl && bash monitor_docker.sh' > /dev/null 2>&1 &"
16+
ssh -t "${SERVER_USERNAME}@${SERVER_HOST}" "cd ./projects/vector-db-benchmark/engine/servers/$SERVER_PATH ; docker compose down ; docker compose up -d"
1717
sleep 30
18-
python3 run.py --engines $ENGINE_NAME --datasets "${DATASETS}" --host $SERVER_HOST
19-
ssh -t ${SERVER_USERNAME}@${SERVER_HOST} "cd ./projects/vector-db-benchmark/engine/servers/$SERVER_PATH ; docker compose down"
20-
ssh -t ${SERVER_USERNAME}@${SERVER_HOST} "cd ./projects/vector-db-benchmark/monitoring && mkdir -p results && mv docker.stats.jsonl ./results/${MONITOR_PATH}-docker.stats.jsonl"
18+
python3 run.py --engines "$ENGINE_NAME" --datasets "${DATASETS}" --host "$SERVER_HOST"
19+
ssh -t "${SERVER_USERNAME}@${SERVER_HOST}" "cd ./projects/vector-db-benchmark/engine/servers/$SERVER_PATH ; docker compose down"
20+
ssh -t "${SERVER_USERNAME}@${SERVER_HOST}" "cd ./projects/vector-db-benchmark/monitoring && mkdir -p results && mv docker.stats.jsonl ./results/${MONITOR_PATH}-docker.stats.jsonl"
2121
}
2222

2323

‎tools/hetzner/create_and_install.sh

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
# path relative to the script
6+
7+
SCRIPT=$(realpath "$0")
8+
SCRIPTPATH=$(dirname "$SCRIPT")
9+
10+
11+
# Create server in Hetzner Cloud
12+
13+
SERVER_NAME=${SERVER_NAME:-test-server-1}
14+
15+
SERVER_TYPE=${SERVER_TYPE:-cx41}
16+
17+
SERVER_IMAGE=${SERVER_IMAGE:-ubuntu-22.04}
18+
19+
SERVER_LOCATION=${SERVER_LOCATION:-nbg1}
20+
21+
SERVER_SSH_KEY=${SERVER_SSH_KEY:-'[email protected]'}
22+
23+
SERVER_NETWORK=${SERVER_NETWORK:-'benchmarks'}
24+
25+
26+
hcloud server create \
27+
--name "${SERVER_NAME}" \
28+
--type "${SERVER_TYPE}" \
29+
--image "${SERVER_IMAGE}" \
30+
--location "${SERVER_LOCATION}" \
31+
--ssh-key "${SERVER_SSH_KEY}" \
32+
--network "${SERVER_NETWORK}"
33+
34+
# Get server IP
35+
SERVER_IP=$(hcloud server ip "${SERVER_NAME}")
36+
37+
echo "Server IP: ${SERVER_IP}"
38+
39+
ssh-keygen -f "$HOME/.ssh/known_hosts" -R "${SERVER_IP}"
40+
41+
# Wait for server to be ready
42+
43+
while ! ssh -oStrictHostKeyChecking=no root@${SERVER_IP} echo "Server is ready"; do
44+
sleep 1
45+
done
46+
47+
# Create and install docker
48+
49+
cat "${SCRIPTPATH}/setup_hetzner.sh" | ssh "root@${SERVER_IP}" bash
50+

‎tools/hetzner/get_private_ip.sh

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/bin/bash
2+
3+
set -e
4+
# Get ip of the private network interface of Hetzner server
5+
# Using `hcloud` CLI tool
6+
7+
# Usage: ./get_private_ip.sh <server_name>
8+
9+
# Example: ./get_private_ip.sh benchmark-server-1
10+
11+
hcloud server describe "$1" -o json | jq -r '.private_net[0].ip'

‎tools/hetzner/get_public_ip.sh

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/bin/bash
2+
3+
set -e
4+
# Get public server IP by name
5+
6+
hcloud server ip "$1"

‎tools/hetzner/setup_hetzner.sh

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
mkdir projects
6+
7+
# Install docker
8+
9+
apt-get update
10+
apt-get install -y \
11+
ca-certificates \
12+
curl \
13+
gnupg \
14+
lsb-release
15+
16+
mkdir -p /etc/apt/keyrings
17+
18+
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
19+
20+
echo \
21+
"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
22+
$(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
23+
24+
apt-get update
25+
26+
apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
27+
28+
docker run hello-world
29+
30+
# Optionally change the directory
31+
32+
#DOCKER_CHROOT=/mnt/HC_Volume_23896067
33+
34+
35+
DOCKER_CHROOT=${DOCKER_CHROOT:-}
36+
37+
38+
# check if DOCKER_CHROOT is set
39+
40+
if [ -z "$DOCKER_CHROOT" ]; then
41+
echo "DOCKER_CHROOT is not set"
42+
else
43+
docker rm -f $(docker ps -aq); docker rmi -f $(docker images -q)
44+
systemctl stop docker
45+
rm -rf /var/lib/docker
46+
mkdir /var/lib/docker
47+
mkdir ${DOCKER_CHROOT}/docker
48+
mount --rbind ${DOCKER_CHROOT}/docker /var/lib/docker
49+
systemctl start docker
50+
fi
51+
52+
# Python
53+
54+
apt install -y python3-pip python-is-python3
55+
56+
pip install virtualenv
57+
58+
# jq
59+
60+
apt install -y jq
61+

‎tools/install_dependencies.sh

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
6+
# Install dependencies on the server for running client
7+
cd "$HOME/projects/vector-db-benchmark/"
8+
9+
pip install poetry
10+
poetry install
11+

‎tools/setup_machines.sh

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
# Setup 2 machines in Hetzner Cloud
6+
# One machine will be used as a server, another one as a client
7+
8+
cleanup() {
9+
echo "cleaning up..."
10+
# Our cleanup code goes here
11+
}
12+
13+
trap 'echo signal received!; kill $(jobs -p); wait; cleanup' SIGINT SIGTERM
14+
15+
CLOUD_NAME=${CLOUD_NAME:-"hetzner"}
16+
17+
SCRIPT=$(realpath "$0")
18+
SCRIPT_PATH=$(dirname "$SCRIPT")
19+
20+
21+
BENCH_SERVER_NAME=${SERVER_NAME:-"benchmark-server-1"}
22+
BENCH_CLIENT_NAME=${CLIENT_NAME:-"benchmark-client-1"}
23+
24+
25+
SERVER_NAME=$BENCH_SERVER_NAME bash -x "${SCRIPT_PATH}/${CLOUD_NAME}/create_and_install.sh" &
26+
SERVER_CREATION_PID=$!
27+
SERVER_NAME=$BENCH_CLIENT_NAME bash -x "${SCRIPT_PATH}/${CLOUD_NAME}/create_and_install.sh"
28+
wait $SERVER_CREATION_PID
29+
30+
31+
bash -x "${SCRIPT_PATH}/sync_code_to_machines.sh"
32+

‎sync.sh renamed to ‎tools/sync.sh

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#!/usr/bin/env bash
22

3+
PROJECT_PATH=$(realpath "$(dirname "$0")/..")
34

45
rsync -avP \
56
--exclude="results" \
@@ -11,4 +12,4 @@ rsync -avP \
1112
--exclude='.idea' \
1213
--exclude='.git' \
1314
--exclude='datasets/*/' \
14-
. $1:./projects/vector-db-benchmark/
15+
"$PROJECT_PATH/" $1:./projects/vector-db-benchmark/

‎tools/sync_code_to_machines.sh

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
CLOUD_NAME=${CLOUD_NAME:-"hetzner"}
6+
7+
SCRIPT=$(realpath "$0")
8+
SCRIPT_PATH=$(dirname "$SCRIPT")
9+
10+
11+
BENCH_SERVER_NAME=${SERVER_NAME:-"benchmark-server-1"}
12+
BENCH_CLIENT_NAME=${CLIENT_NAME:-"benchmark-client-1"}
13+
14+
15+
IP_OF_THE_SERVER=$(bash "${SCRIPT_PATH}/${CLOUD_NAME}/get_public_ip.sh" "$BENCH_SERVER_NAME")
16+
IP_OF_THE_CLIENT=$(bash "${SCRIPT_PATH}/${CLOUD_NAME}/get_public_ip.sh" "$BENCH_CLIENT_NAME")
17+
18+
bash -x "${SCRIPT_PATH}/sync.sh" "root@$IP_OF_THE_SERVER"
19+
bash -x "${SCRIPT_PATH}/sync.sh" "root@$IP_OF_THE_CLIENT"
20+
21+
cat "${SCRIPT_PATH}/install_dependencies.sh" | ssh "root@${IP_OF_THE_CLIENT}" bash
22+
23+

0 commit comments

Comments
 (0)