M111-Big-Data-Distributed-SQL-Query-Engine

Distributed SQL Query Engine over a Column Store

Compile

g++ -o minidist src/main.cpp .\src\storage\storage_operations.cpp .\src\storage\storage_parser.cpp .\src\cli\cli_handler.cpp

g++ -g -O0 -std=c++17 -o coordinator.exe src/coordinator/*.cpp -lws2_32

g++ -g -std=c++17 -o worker.exe src/worker/*.cpp src/storage/*.cpp src/coordinator/query_parser.cpp -lws2_32

Run Init

./minidist init data/tags --schema ../data/tags/_schema.ssf
./minidist init data/ratings --schema ../data/ratings/_schema.ssf
./minidist init data/links --schema ../data/links/_schema.ssf
./minidist init data/movies --schema ../data/movies/_schema.ssf
./minidist init data/order --schema ../data/order/_schema.ssf

Run Load

.\minidist load data/links --csv ../data/links/links.csv --sort-key id --segments 2
.\minidist load data/movies --csv ../data/movies/movies.csv --sort-key id --segments 2 
.\minidist load data/ratings --csv ../data/ratings/ratings.csv --sort-key id --segments 2 
.\minidist load data/tags --csv ../data/tags/tags.csv --sort-key id --segments 2
.\minidist load data/order --csv ../data/order/orders.csv --sort-key order_id --segments 2

Run Inspect Schema and Metadata

.\minidist schema show data/links
.\minidist info data/links

Run Coordinator

.\coordinator.exe --port 8080 --workers 9001,9002

Run Workers

.\worker.exe --port 9001 --data data/ --segment seg-000001

.\worker.exe --port 9002 --data data/ --segment seg-000002

Post Query to Coordinator

curl -X POST http://localhost:8080/query --data "SELECT * FROM movies WHERE movieId < 100;"
curl -X POST http://localhost:8080/query --data "SELECT * FROM movies WHERE movieId > 10000;"
curl -X POST http://localhost:8080/query --data "SELECT * FROM movies WHERE movieId = 193609;"
curl -X POST http://localhost:8080/query --data "SELECT * FROM movies WHERE movieId between 100 and 400;"
curl -X POST http://localhost:8080/query --data "SELECT COUNT(*) FROM movies WHERE movieId between 100 and 1000;"
curl -X POST http://localhost:8080/query --data "SELECT max(amount),min(amount),items_bought from order group by items_bought;"
curl -X POST http://localhost:8080/query --data "SELECT sum(amount),avg(amount),items_bought from order group by items_bought;"