From 742dea24cd1e8ab3d60ca8fa7ae53a16288db89d Mon Sep 17 00:00:00 2001
From: helmolill <helmolill@hotmail.com>
Date: Mon, 30 Aug 2021 14:58:20 +0300
Subject: [PATCH] Circular test

---
 Dockerfile                   |   2 +-
 docker-compose.yml           |   8 ++
 tests/circular/cf/aq_a.ini   |   9 ++
 tests/circular/cf/aq_c.ini   |   9 ++
 tests/circular/cf/bq_a.ini   |   9 ++
 tests/circular/cf/bq_b.ini   |   9 ++
 tests/circular/cf/bq_d.ini   |   9 ++
 tests/circular/cf/cq_b.ini   |   9 ++
 tests/circular/cf/cq_c.ini   |   9 ++
 tests/circular/cf/cq_d.ini   |   9 ++
 tests/circular/cf/dq_d.ini   |   9 ++
 tests/circular/docker_run.sh |   9 ++
 tests/circular/init.sh       |  14 +++
 tests/circular/regen.sh      | 162 +++++++++++++++++++++++++++++++++++
 14 files changed, 275 insertions(+), 1 deletion(-)
 create mode 100644 tests/circular/cf/aq_a.ini
 create mode 100644 tests/circular/cf/aq_c.ini
 create mode 100644 tests/circular/cf/bq_a.ini
 create mode 100644 tests/circular/cf/bq_b.ini
 create mode 100644 tests/circular/cf/bq_d.ini
 create mode 100644 tests/circular/cf/cq_b.ini
 create mode 100644 tests/circular/cf/cq_c.ini
 create mode 100644 tests/circular/cf/cq_d.ini
 create mode 100644 tests/circular/cf/dq_d.ini
 create mode 100755 tests/circular/docker_run.sh
 create mode 100755 tests/circular/init.sh
 create mode 100755 tests/circular/regen.sh

diff --git a/Dockerfile b/Dockerfile
index aff8888..5a7e8dd 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -2,7 +2,7 @@ FROM python:3.8-slim-buster
 
 WORKDIR /code
 COPY . .
-RUN pip3 install 'psycopg2-binary==2.8.6' 'pyyaml==5.3.1' 'skytools==3.6.1' 'pgq==3.5'
+RUN pip3 install 'psycopg2-binary==2.8.6' 'pyyaml==5.3.1' 'skytools==3.7.3' 'pgq==3.5.1'
 RUN pip3 install .
 
 ENV DEBIAN_FRONTEND="noninteractive"
diff --git a/docker-compose.yml b/docker-compose.yml
index 05629ad..a5a4c5a 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -17,6 +17,14 @@ services:
       - londiste_test
     command: ["./simple/docker_run.sh"]
 
+  test_circular:
+    build: .
+    volumes:
+      - londiste_test
+    networks:
+      - londiste_test
+    command: ["./circular/docker_run.sh"]
+
 networks:
   londiste_test:
 
diff --git a/tests/circular/cf/aq_a.ini b/tests/circular/cf/aq_a.ini
new file mode 100644
index 0000000..8dab0cb
--- /dev/null
+++ b/tests/circular/cf/aq_a.ini
@@ -0,0 +1,9 @@
+[londiste]
+job_name = aq_a
+db = dbname=a
+public_node_location = dbname=a
+queue_name = aq
+logfile = log/%(job_name)s.log
+pidfile = pid/%(job_name)s.pid
+pgq_autocommit = 1
+pgq_lazy_fetch = 0
diff --git a/tests/circular/cf/aq_c.ini b/tests/circular/cf/aq_c.ini
new file mode 100644
index 0000000..25ddfee
--- /dev/null
+++ b/tests/circular/cf/aq_c.ini
@@ -0,0 +1,9 @@
+[londiste]
+job_name = aq_c
+db = dbname=c
+public_node_location = dbname=c
+queue_name = aq
+logfile = log/%(job_name)s.log
+pidfile = pid/%(job_name)s.pid
+pgq_autocommit = 1
+pgq_lazy_fetch = 0
diff --git a/tests/circular/cf/bq_a.ini b/tests/circular/cf/bq_a.ini
new file mode 100644
index 0000000..78b0feb
--- /dev/null
+++ b/tests/circular/cf/bq_a.ini
@@ -0,0 +1,9 @@
+[londiste]
+job_name = bq_a
+db = dbname=a
+public_node_location = dbname=a
+queue_name = bq
+logfile = log/%(job_name)s.log
+pidfile = pid/%(job_name)s.pid
+pgq_autocommit = 1
+pgq_lazy_fetch = 0
diff --git a/tests/circular/cf/bq_b.ini b/tests/circular/cf/bq_b.ini
new file mode 100644
index 0000000..e9e5581
--- /dev/null
+++ b/tests/circular/cf/bq_b.ini
@@ -0,0 +1,9 @@
+[londiste]
+job_name = bq_b
+db = dbname=b
+public_node_location = dbname=b
+queue_name = bq
+logfile = log/%(job_name)s.log
+pidfile = pid/%(job_name)s.pid
+pgq_autocommit = 1
+pgq_lazy_fetch = 0
diff --git a/tests/circular/cf/bq_d.ini b/tests/circular/cf/bq_d.ini
new file mode 100644
index 0000000..3d278dd
--- /dev/null
+++ b/tests/circular/cf/bq_d.ini
@@ -0,0 +1,9 @@
+[londiste]
+job_name = bq_d
+db = dbname=d
+public_node_location = dbname=d
+queue_name = bq
+logfile = log/%(job_name)s.log
+pidfile = pid/%(job_name)s.pid
+pgq_autocommit = 1
+pgq_lazy_fetch = 0
diff --git a/tests/circular/cf/cq_b.ini b/tests/circular/cf/cq_b.ini
new file mode 100644
index 0000000..2673a49
--- /dev/null
+++ b/tests/circular/cf/cq_b.ini
@@ -0,0 +1,9 @@
+[londiste]
+job_name = cq_b
+db = dbname=b
+public_node_location = dbname=b
+queue_name = cq
+logfile = log/%(job_name)s.log
+pidfile = pid/%(job_name)s.pid
+pgq_autocommit = 1
+pgq_lazy_fetch = 0
diff --git a/tests/circular/cf/cq_c.ini b/tests/circular/cf/cq_c.ini
new file mode 100644
index 0000000..c3c7deb
--- /dev/null
+++ b/tests/circular/cf/cq_c.ini
@@ -0,0 +1,9 @@
+[londiste]
+job_name = cq_c
+db = dbname=c
+public_node_location = dbname=c
+queue_name = cq
+logfile = log/%(job_name)s.log
+pidfile = pid/%(job_name)s.pid
+pgq_autocommit = 1
+pgq_lazy_fetch = 0
diff --git a/tests/circular/cf/cq_d.ini b/tests/circular/cf/cq_d.ini
new file mode 100644
index 0000000..b21f94c
--- /dev/null
+++ b/tests/circular/cf/cq_d.ini
@@ -0,0 +1,9 @@
+[londiste]
+job_name = cq_d
+db = dbname=d
+public_node_location = dbname=d
+queue_name = cq
+logfile = log/%(job_name)s.log
+pidfile = pid/%(job_name)s.pid
+pgq_autocommit = 1
+pgq_lazy_fetch = 0
diff --git a/tests/circular/cf/dq_d.ini b/tests/circular/cf/dq_d.ini
new file mode 100644
index 0000000..53a9272
--- /dev/null
+++ b/tests/circular/cf/dq_d.ini
@@ -0,0 +1,9 @@
+[londiste]
+job_name = dq_d
+db = dbname=d
+public_node_location = dbname=d
+queue_name = dq
+logfile = log/%(job_name)s.log
+pidfile = pid/%(job_name)s.pid
+pgq_autocommit = 1
+pgq_lazy_fetch = 0
diff --git a/tests/circular/docker_run.sh b/tests/circular/docker_run.sh
new file mode 100755
index 0000000..8a2616a
--- /dev/null
+++ b/tests/circular/docker_run.sh
@@ -0,0 +1,9 @@
+#! /bin/bash
+
+set -e
+set -x
+
+cd circular
+
+./init.sh
+./regen.sh
diff --git a/tests/circular/init.sh b/tests/circular/init.sh
new file mode 100755
index 0000000..8e77b26
--- /dev/null
+++ b/tests/circular/init.sh
@@ -0,0 +1,14 @@
+#! /bin/sh
+
+db_list="a b c d"
+
+for db in $db_list; do
+  echo dropdb $db
+  dropdb $db
+done
+
+
+for db in $db_list; do
+  echo createdb $db
+  createdb $db
+done
diff --git a/tests/circular/regen.sh b/tests/circular/regen.sh
new file mode 100755
index 0000000..5e518ac
--- /dev/null
+++ b/tests/circular/regen.sh
@@ -0,0 +1,162 @@
+#! /bin/bash
+
+. ../testlib.sh
+
+v='-q'
+v=''
+nocheck=1
+
+db_list="a b c d"
+
+kdb_list=`echo $db_list | sed 's/ /,/g'`
+
+#( cd ../..; make -s install )
+
+do_check() {
+  test $nocheck = 1 || ../zcheck.sh
+}
+
+title Circular test
+
+# create ticker conf
+cat > conf/pgqd.ini <<EOF
+[pgqd]
+database_list = $kdb_list
+ticker_period = 0.2
+check_period = 1
+syslog = 0
+logfile = log/pgqd.log
+pidfile = pid/pgqd.pid
+#threaded_copy_tables = public.mytable2
+#threaded_copy_pool_size = 1
+EOF
+
+for db in $db_list; do
+  cleardb $db
+done
+
+clearlogs
+
+set -e
+
+msg "Initialize nodes"
+# root
+run londiste $v cf/aq_a.ini create-root a
+run londiste $v cf/bq_b.ini create-root b
+run londiste $v cf/cq_c.ini create-root c
+run londiste $v cf/dq_d.ini create-root d
+
+# merge-leaf
+run londiste $v cf/aq_c.ini create-leaf aq_c --provider='dbname=a' --merge='cq'
+run londiste $v cf/bq_a.ini create-leaf bq_a --provider='dbname=b' --merge='aq'
+run londiste $v cf/bq_d.ini create-leaf bq_d --provider='dbname=b' --merge='dq'
+
+# leaf
+run londiste $v cf/cq_b.ini create-leaf cq_b --provider='dbname=c'
+run londiste $v cf/cq_d.ini create-leaf cq_d --provider='dbname=c'
+
+
+msg "Run ticker"
+run pgqd $v -d conf/pgqd.ini
+run sleep 2
+
+msg "See topology"
+run londiste $v cf/aq_a.ini status
+run londiste $v cf/bq_b.ini status
+run londiste $v cf/cq_c.ini status
+run londiste $v cf/dq_d.ini status
+
+msg "Run londiste daemon for each node"
+run londiste $v -d cf/aq_a.ini worker
+run londiste $v -d cf/bq_b.ini worker
+run londiste $v -d cf/cq_c.ini worker
+run londiste $v -d cf/dq_d.ini worker
+run londiste $v -d cf/aq_c.ini worker
+run londiste $v -d cf/cq_b.ini worker
+run londiste $v -d cf/cq_d.ini worker
+run londiste $v -d cf/bq_a.ini worker
+run londiste $v -d cf/bq_d.ini worker
+
+
+msg "Create tables"
+run_sql a "create table a_to_cbd (data text primary key)"
+run_sql a "create table b_to_acd (data text primary key)"
+
+run_sql b "create table a_to_cbd (data text primary key)"
+run_sql b "create table b_to_acd (data text primary key)"
+run_sql b "create table b_to_d (data text primary key)"
+run_sql b "create table c_to_bd (data text primary key)"
+
+run_sql c "create table a_to_cbd (data text primary key)"
+run_sql c "create table b_to_acd (data text primary key)"
+run_sql c "create table c_to_bd (data text primary key)"
+
+run_sql d "create table a_to_cbd (data text primary key)"
+run_sql d "create table b_to_acd (data text primary key)"
+run_sql d "create table b_to_d (data text primary key)"
+run_sql d "create table c_to_bd (data text primary key)"
+
+
+msg "Register tables"
+# root
+run londiste $v cf/aq_a.ini add-table a_to_cbd
+run londiste $v cf/bq_b.ini add-table b_to_acd b_to_d
+run londiste $v cf/cq_c.ini add-table c_to_bd
+
+# merge-leaf
+run londiste $v cf/bq_a.ini add-table b_to_acd --merge-all
+run londiste $v cf/aq_c.ini add-table a_to_cbd b_to_acd --merge-all
+run londiste $v cf/bq_d.ini add-table b_to_d --merge-all
+
+# leaf
+run londiste $v cf/cq_d.ini add-table a_to_cbd c_to_bd b_to_acd --no-merge
+run londiste $v cf/cq_b.ini add-table a_to_cbd c_to_bd --no-merge
+
+
+msg "Wait until everything is in sync"
+run londiste $v cf/aq_c.ini wait-sync
+run londiste $v cf/bq_a.ini wait-sync
+run londiste $v cf/bq_d.ini wait-sync
+run londiste $v cf/cq_b.ini wait-sync
+run londiste $v cf/cq_d.ini wait-sync
+
+
+msg "Add some data"
+run_sql a "insert into a_to_cbd values ('test')"
+run_sql b "insert into b_to_acd values ('test')"
+run_sql b "insert into b_to_d values ('test')"
+run_sql c "insert into c_to_bd values ('test')"
+
+
+msg "Wait for tables to replicate"
+run sleep 10
+
+
+msg "Check data"
+run_sql a "select * from a_to_cbd"
+run_sql a "select * from b_to_acd"
+
+run_sql b "select * from a_to_cbd"
+run_sql b "select * from b_to_acd"
+run_sql b "select * from b_to_d"
+run_sql b "select * from c_to_bd"
+
+run_sql c "select * from a_to_cbd"
+run_sql c "select * from b_to_acd"
+run_sql c "select * from c_to_bd"
+
+run_sql d "select * from a_to_cbd"
+run_sql d "select * from b_to_acd"
+run_sql d "select * from b_to_d"
+run_sql d "select * from c_to_bd"
+
+
+msg "Check londiste status"
+run londiste $v cf/aq_a.ini status
+run londiste $v cf/bq_b.ini status
+run londiste $v cf/cq_c.ini status
+run londiste $v cf/dq_d.ini status
+
+#run tail -f log/cq_d.log
+
+exit 0