Merge pull request #16 from infocyph/feature/multi-stack

Feature/multi stack

Author: abmmhasan

.gitignore

@@ -1,6 +1,16 @@
-.idea
-.env
-*~
-Local
-*.txt
-!docs/requirements.txt
+*
+!bin
+!configuration
+!data/.gitignore
+!docker/compose
+!docker/conf
+!docker/dockerfiles
+!docs
+!logs
+!.gitignore
+!.gitattributes
+!.readthedocs.yaml
+!lds
+!lds.bat
+!LICENSE
+!README.md

docker/compose/companion.yaml

@@ -23,6 +23,8 @@ services:
     networks:
       backend:
         ipv4_address: 172.29.0.10
+      datastore:
+        ipv4_address: 172.30.0.10
 
   runner:
     container_name: RUNNER
@@ -36,7 +38,17 @@ services:
       - ../../configuration/scheduler/cron-jobs:/etc/cron.d:ro
       - ../../logs/runner:/var/log/supervisor
       - ../../logs/apache:/global/log/apache
+      - ../../logs/cloudbeaver:/global/log/cloudbeaver
+      - ../../logs/elasticsearch:/global/log/elasticsearch
+      - ../../logs/kibana:/global/log/kibana
+      - ../../logs/mariadb:/global/log/mariadb
+      - ../../logs/mongo-express:/global/log/mongo-express
+      - ../../logs/mongodb:/global/log/mongodb
+      - ../../logs/mysql:/global/log/mysql
       - ../../logs/nginx:/global/log/nginx
+      - ../../logs/postgresql:/global/log/postgresql
+      - ../../logs/redis:/global/log/redis
+      - ../../logs/redis-insight:/global/log/redis-insight
       - /var/run/docker.sock:/var/run/docker.sock
     depends_on:
       - server-tools

docker/compose/db-client.yaml

@@ -17,6 +17,7 @@ services:
       - RI_REDIS_HOST=redis
     volumes:
       - ../../data/redis-insight:/data
+      - ../../logs/redis-insight:/var/log/redis-insight
     networks:
       datastore:
         ipv4_address: 172.30.0.150
@@ -33,6 +34,7 @@ services:
       - TZ=${TZ:-}
     volumes:
       - ../../data/cloudbeaver:/opt/cloudbeaver/workspace
+      - ../../logs/cloudbeaver:/opt/cloudbeaver/logs
     networks:
       datastore:
         ipv4_address: 172.30.0.151
@@ -53,6 +55,8 @@ services:
       - ME_CONFIG_MONGODB_ADMINUSERNAME=${MONGODB_ROOT_USERNAME:-root}
       - ME_CONFIG_MONGODB_ADMINPASSWORD=${MONGODB_ROOT_PASSWORD:-12345}
       - ME_CONFIG_MONGODB_URL=mongodb://${MONGODB_ROOT_USERNAME:-root}:${MONGODB_ROOT_PASSWORD:-12345}@mongodb:${MONGODB_PORT:-27017}/admin
+    volumes:
+      - ../../logs/mongo-express:/var/log/mongo-express
     networks:
       datastore:
         ipv4_address: 172.30.0.152
@@ -63,7 +67,7 @@ services:
     <<: *db-client-service
     container_name: KIBANA
     hostname: kibana
-    image: kibana:${ELASTICSEARCH_VERSION:-9.2.4}
+    image: kibana:${ELASTICSEARCH_VERSION:-9.3.0}
     profiles: [elasticsearch]
     depends_on:
       - elasticsearch
@@ -72,8 +76,26 @@ services:
       - "ELASTICSEARCH_HOSTS=http://elasticsearch:9200"
     volumes:
       - ../../data/kibana:/usr/share/kibana/data
+      - ../../logs/kibana:/usr/share/kibana/logs
     networks:
       datastore:
         ipv4_address: 172.30.0.153
       frontend:
         ipv4_address: 172.28.0.153
+
+  filebeat:
+    image: docker.elastic.co/beats/filebeat:${ELASTICSEARCH_VERSION:-9.3.0}
+    container_name: FILEBEAT
+    restart: unless-stopped
+    user: root
+    profiles: [elasticsearch]
+    depends_on:
+      - elasticsearch
+    volumes:
+      - ../conf/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
+      - ../../logs:/global/log:ro
+      - ../../data/filebeat:/usr/share/filebeat/data
+    command: ["--strict.perms=false"]
+    networks:
+      datastore:
+        ipv4_address: 172.30.0.154

docker/compose/db.yaml

@@ -12,6 +12,7 @@ services:
     profiles: [redis]
     volumes:
       - ../../data/redis:/data
+      - ../../logs/redis:/var/log/redis
     environment:
       - TZ=${TZ:-}
       - REDIS_ARGS=--save 10 1 --appendonly yes
@@ -37,6 +38,7 @@ services:
       - POSTGRES_DB=${POSTGRES_DATABASE:-postgres}
     volumes:
       - ../../data/postgresql:/var/lib/postgresql
+      - ../../logs/postgresql:/var/log/postgresql
       - ../conf/pg_hba.conf:/etc/postgresql/pg_hba.conf
 #      - ../conf/postgresql.conf:/etc/postgresql/postgresql.conf
     command:
@@ -94,6 +96,7 @@ services:
       - MONGO_INITDB_ROOT_PASSWORD=${MONGODB_ROOT_PASSWORD:-12345}
     volumes:
       - ../../data/mongo:/data/db
+      - ../../logs/mongodb:/var/log/mongodb
     healthcheck:
       test:
         [
@@ -135,7 +138,7 @@ services:
     <<: *db-service
     container_name: ELASTICSEARCH
     hostname: elasticsearch
-    image: elasticsearch:${ELASTICSEARCH_VERSION:-9.2.4}
+    image: elasticsearch:${ELASTICSEARCH_VERSION:-9.3.0}
     profiles: [elasticsearch]
     environment:
       - TZ=${TZ:-}
@@ -145,6 +148,7 @@ services:
       - "node.name=elasticsearch-node-single"
     volumes:
       - ../../data/elasticsearch:/usr/share/elasticsearch/data
+      - ../../logs/elasticsearch:/usr/share/elasticsearch/logs
     healthcheck:
       test: ["CMD-SHELL", "curl -f http://localhost:9200/_cluster/health || exit 1"]
       interval: 30s

docker/conf/filebeat.yml

@@ -0,0 +1,31 @@
+filebeat.inputs:
+  - type: filestream
+    id: lds-global-logs
+    enabled: true
+    paths:
+      - /global/log/*/*.log
+      - /global/log/*/*/*.log
+    exclude_files: ['-\d{8}(\.gz)?$']
+    processors:
+      - dissect:
+          tokenizer: "/global/log/%{service}/%{file}"
+          field: "log.file.path"
+          target_prefix: ""
+      - convert:
+          fields:
+            - {from: "service", to: "service", type: "string"}
+          ignore_missing: true
+          fail_on_error: false
+output.elasticsearch:
+  hosts: ["http://elasticsearch:9200"]
+  index: "lds-%{[service]}-%{+yyyyMMdd}"
+setup.template.enabled: true
+setup.template.name: "lds"
+setup.template.pattern: "lds-*-*"
+setup.template.overwrite: false
+setup.dashboards.enabled: true
+setup.dashboards.retry.enabled: true
+setup.dashboards.retry.interval: 10s
+setup.dashboards.retry.maximum: 30
+setup.kibana:
+  host: "http://kibana:5601"

lds

@@ -371,7 +371,7 @@ fix_perms() {
   chmod 2777 "$DIR/data"
   mkdir -p "$DIR/data/cloudbeaver" "$DIR/data/mysql" "$DIR/data/postgresql" "$DIR/data/mongo" \
     "$DIR/data/mariadb" "$DIR/data/elasticsearch" "$DIR/data/redis" "$DIR/data/mailpit" \
-    "$DIR/data/redis-insight" "$DIR/data/kibana"
+    "$DIR/data/redis-insight" "$DIR/data/kibana" "$DIR/data/filebeat"
   find "$DIR/data" -mindepth 1 -maxdepth 1 -type d -exec chmod 2777 {} +
   find "$DIR/data" -type f -exec chmod 666 {} +
 
@@ -469,7 +469,7 @@ declare -A SERVICES=(
 declare -a SERVICE_ORDER=(POSTGRESQL MYSQL MARIADB ELASTICSEARCH MONGODB REDIS)
 
 declare -A PROFILE_ENV=(
-  [elasticsearch]="ELASTICSEARCH_VERSION=9.2.4"
+  [elasticsearch]="ELASTICSEARCH_VERSION=9.3.0"
   [mysql]="MYSQL_VERSION=latest MYSQL_ROOT_PASSWORD=12345 MYSQL_USER=infocyph MYSQL_PASSWORD=12345 MYSQL_DATABASE=localdb"
   [mariadb]="MARIADB_VERSION=latest MARIADB_ROOT_PASSWORD=12345 MARIADB_USER=infocyph MARIADB_PASSWORD=12345 MARIADB_DATABASE=localdb"
   [mongodb]="MONGODB_VERSION=latest MONGODB_ROOT_USERNAME=root MONGODB_ROOT_PASSWORD=12345"

logs/.gitignore

@@ -4,5 +4,13 @@
 !nginx
 !mariadb
 !mysql
+!cloudbeaver
+!kibana
+!mongo-express
+!redis-insight
+!elasticsearch
+!mongodb
+!postgresql
+!redis
 !olds
 !.gitignore

logs/cloudbeaver/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

logs/elasticsearch/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

logs/kibana/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore