Skip to content

Commit f7d5650

Browse files
fcostaoliveirafilipecosta90
fcostaoliveira
and
filipecosta90
authored
Added OSS weaviate setup with same infra as shown in https://redis.io/blog/benchmarking-results-for-vector-databases/ (#134)
Co-authored-by: filipecosta90 <filipecosta.90@gmail.com>
1 parent cbe85d8 commit f7d5650

File tree

7 files changed

+306
-0
lines changed

7 files changed

+306
-0
lines changed

‎terraform/weaviate-v1.29.1-m6i-2xlarge/Readme.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
2+
apt update
3+
apt install docker.io docker-compose -y
4+
5+
git clone https://github.com/redis-performance/vector-db-benchmark
6+
cd vector-db-benchmark

‎terraform/weaviate-v1.29.1-m6i-2xlarge/common.tf‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
2+
################################################################################
3+
# This is the bucket holding this specific setup tfstate
4+
################################################################################
5+
terraform {
6+
backend "s3" {
7+
bucket = "performance-cto-group"
8+
region = "us-east-1"
9+
key = "weaviate-v1.29.1-m6i-2xlarge.tfstate"
10+
}
11+
}
12+

‎terraform/weaviate-v1.29.1-m6i-2xlarge/db-resources_2a.tf‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
2+
resource "aws_instance" "server_2a" {
3+
count = var.server_instance_count
4+
ami = var.instance_ami
5+
instance_type = var.server_instance_type
6+
subnet_id = data.terraform_remote_state.shared_resources.outputs.subnet_public_id
7+
vpc_security_group_ids = ["${data.terraform_remote_state.shared_resources.outputs.performance_cto_sg_id}"]
8+
key_name = var.key_name
9+
availability_zone = "us-east-2a"
10+
placement_group = data.terraform_remote_state.shared_resources.outputs.perf_cto_pg_name
11+
12+
root_block_device {
13+
volume_size = var.instance_volume_size
14+
volume_type = var.instance_volume_type
15+
encrypted = var.instance_volume_encrypted
16+
delete_on_termination = true
17+
}
18+
19+
volume_tags = {
20+
Environment = "${var.environment}"
21+
Project = "${var.environment}"
22+
Name = "ebs_block_device-${var.setup_name}-DB-us-east-2a-${count.index + 1}"
23+
setup = "${var.setup_name}"
24+
triggering_env = "${var.triggering_env}"
25+
github_actor = "${var.github_actor}"
26+
github_org = "${var.github_org}"
27+
github_repo = "${var.github_repo}"
28+
github_sha = "${var.github_sha}"
29+
}
30+
31+
tags = {
32+
Environment = "${var.environment}"
33+
Project = "${var.environment}"
34+
Name = "${var.setup_name}-DB-us-east-2a-${count.index + 1}"
35+
setup = "${var.setup_name}"
36+
triggering_env = "${var.triggering_env}"
37+
github_actor = "${var.github_actor}"
38+
github_org = "${var.github_org}"
39+
github_repo = "${var.github_repo}"
40+
github_sha = "${var.github_sha}"
41+
}
42+
43+
################################################################################
44+
# Deployment related
45+
################################################################################
46+
47+
################################################################################
48+
# Install docker
49+
################################################################################
50+
provisioner "remote-exec" {
51+
script = "./install_docker.sh"
52+
connection {
53+
host = self.public_ip # The `self` variable is like `this` in many programming languages
54+
type = "ssh" # in this case, `self` is the resource (the server).
55+
user = var.ssh_user
56+
private_key = file(var.private_key)
57+
#need to increase timeout to larger then 5m for metal instances
58+
timeout = "5m"
59+
agent = "false"
60+
}
61+
}
62+
63+
}

‎terraform/weaviate-v1.29.1-m6i-2xlarge/install_docker.sh‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
export DEBIAN_FRONTEND=noninteractive
2+
3+
sudo apt-get update -y
4+
sudo apt-get install ca-certificates curl -y
5+
sudo install -m 0755 -d /etc/apt/keyrings
6+
sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
7+
sudo chmod a+r /etc/apt/keyrings/docker.asc
8+
9+
# Add the repository to Apt sources:
10+
echo \
11+
"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
12+
$(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | \
13+
sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
14+
sudo apt-get update -y
15+
16+
sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y
17+
18+
# check
19+
echo "Printing docker info"
20+
docker --version

‎terraform/weaviate-v1.29.1-m6i-2xlarge/output.tf‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
output "server_public_ip" {
2+
value = aws_instance.server_2a[*].public_ip
3+
}
4+
5+
output "server_private_ip" {
6+
value = aws_instance.server_2a[*].private_ip
7+
}
8+
9+
output "server_instance_type" {
10+
value = var.server_instance_type
11+
}
12+
13+
output "search_threads" {
14+
value = var.search_thread
15+
}
16+
17+
output "setup_name" {
18+
value = var.setup_name
19+
}
20+
21+
output "server_instance_count" {
22+
value = var.server_instance_count
23+
}

‎terraform/weaviate-v1.29.1-m6i-2xlarge/shared_resources.tf‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# provider
2+
provider "aws" {
3+
region = var.region
4+
}
5+
6+
################################################################################
7+
# This is the shared resources bucket key -- you will need it across environments like security rules,etc...
8+
# !! do not change this !!
9+
################################################################################
10+
data "terraform_remote_state" "shared_resources" {
11+
backend = "s3"
12+
config = {
13+
bucket = "performance-cto-group"
14+
key = "benchmarks/infrastructure/shared_resources.tfstate"
15+
region = "us-east-1"
16+
}
17+
}

‎terraform/weaviate-v1.29.1-m6i-2xlarge/variables.tf‎

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
################################################################################
2+
# Variables used for deployment tag
3+
################################################################################
4+
5+
variable "search_thread" {
6+
description = "setup name"
7+
default = "6"
8+
}
9+
10+
variable "setup_name" {
11+
description = "setup name"
12+
default = "weaviate-v1.29.1-m6i-2xlarge"
13+
}
14+
15+
variable "github_actor" {
16+
description = "The name of the person or app that initiated the deployment."
17+
default = "N/A"
18+
}
19+
20+
21+
variable "github_repo" {
22+
description = " The owner and repository name. For example, testing-infrastructure."
23+
default = "N/A"
24+
}
25+
26+
variable "triggering_env" {
27+
description = " The triggering environment. For example circleci."
28+
default = "N/A"
29+
}
30+
31+
variable "environment" {
32+
description = " The cost tag."
33+
default = "VecSim-Competitive"
34+
}
35+
36+
variable "github_org" {
37+
description = " The owner name. For example, RedisModules."
38+
default = "N/A"
39+
}
40+
41+
variable "github_sha" {
42+
description = "The commit SHA that triggered the deployment."
43+
default = "N/A"
44+
}
45+
46+
variable "timeout_secs" {
47+
description = "The maximum time to wait prior destroying the VM via the watchdog."
48+
default = "3600"
49+
}
50+
51+
52+
53+
################################################################################
54+
# Access keys
55+
################################################################################
56+
variable "private_key" {
57+
description = "private key"
58+
default = "/tmp/benchmarks.redislabs.pem"
59+
}
60+
61+
variable "key_name" {
62+
description = "key name"
63+
default = "perf-cto-us-east-2"
64+
}
65+
66+
variable "region" {
67+
default = "us-east-2"
68+
}
69+
70+
# (Ubuntu 20.04)
71+
# us-east-2 Focal Fossa 20.04 LTS amd64 hvm:ebs-ssd 20240205 ami-07b469810a61205a8 hvm
72+
variable "instance_ami" {
73+
description = "AMI for aws EC2 instance - us-east-2 Ubuntu 20.04 - amd64"
74+
default = "ami-07b469810a61205a8"
75+
}
76+
77+
variable "instance_device_name" {
78+
description = "EC2 instance device name"
79+
default = "/dev/sda1"
80+
}
81+
82+
variable "redis_module" {
83+
description = "redis_module"
84+
default = "N/A"
85+
}
86+
87+
variable "instance_volume_size" {
88+
description = "EC2 instance volume_size"
89+
default = "256"
90+
}
91+
92+
variable "instance_volume_type" {
93+
description = "EC2 instance volume_type"
94+
default = "gp3"
95+
}
96+
97+
98+
variable "instance_volume_iops" {
99+
description = "EC2 instance volume_iops"
100+
default = "3000"
101+
}
102+
103+
variable "client_instance_volume_size" {
104+
description = "EC2 instance volume_size"
105+
default = "256"
106+
}
107+
108+
variable "client_instance_volume_type" {
109+
description = "EC2 instance volume_type"
110+
default = "gp3"
111+
}
112+
113+
114+
variable "instance_volume_encrypted" {
115+
description = "EC2 instance instance_volume_encrypted"
116+
default = "false"
117+
}
118+
119+
variable "instance_root_block_device_encrypted" {
120+
description = "EC2 instance instance_root_block_device_encrypted"
121+
default = "false"
122+
}
123+
124+
variable "instance_cpu_threads_per_core" {
125+
description = "CPU threads per core for aws EC2 instance"
126+
default = 1
127+
}
128+
129+
variable "instance_cpu_threads_per_core_hyperthreading" {
130+
description = "CPU threads per core when hyperthreading is enabled for aws EC2 instance"
131+
default = 2
132+
}
133+
134+
variable "instance_network_interface_plus_count" {
135+
description = "number of additional network interfaces to add to aws EC2 instance"
136+
default = 0
137+
}
138+
139+
variable "os" {
140+
description = "os"
141+
default = "ubuntu20.04"
142+
}
143+
144+
variable "ssh_user" {
145+
description = "ssh_user"
146+
default = "ubuntu"
147+
}
148+
149+
################################################################################
150+
# Specific DB machine variables
151+
################################################################################
152+
variable "server_instance_type" {
153+
description = "type for aws EC2 instance"
154+
default = "m6i.2xlarge"
155+
}
156+
157+
variable "server_instance_count" {
158+
description = "count of aws EC2 instances"
159+
default = 1
160+
}
161+
162+
variable "server_instance_cpu_core_count" {
163+
description = "CPU core count for aws EC2 instance"
164+
default = 4
165+
}

0 commit comments

Comments
 (0)